Add Grafana dashboard

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.24-alpine AS builder
+WORKDIR /build
+COPY go.mod go.sum ./
+RUN go mod download
+COPY main.go ./
+RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o ctlog-uptime-exporter .
+
+FROM scratch
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /build/ctlog-uptime-exporter /ctlog-uptime-exporter
+EXPOSE 9781
+ENTRYPOINT ["/ctlog-uptime-exporter"]

README.md

@@ -1,5 +1,7 @@
 # ctlog-uptime-exporter
 
+![grafana dashboard](grafana.png)
+
 A Prometheus exporter for [Certificate Transparency](https://certificate.transparency.dev/) log uptime data published by Google at:
 
 ```

ctlog-uptime-exporter.default

@@ -3,7 +3,7 @@
 #
 # -listen    address to listen on               (default: :9781)
 # -url       URL of the uptime CSV              (default: https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv)
-# -interval  how often to fetch the CSV         (default: 12h)
+# -interval  how often to fetch the CSV         (default: 25m)
 # -jitter    maximum +/-jitter on the interval  (default: 5m)
 
-ARGS='-listen :9781 -url https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv -interval 12h -jitter 5m'
+ARGS='-listen :9781 -url https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv -interval 25m -jitter 5m'

dashboard.json

@@ -0,0 +1,549 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "title": "CT Log Uptime",
+  "uid": "ctlog-uptime",
+  "description": "Certificate Transparency log 24h uptime sourced from gstatic.com via ctlog-uptime-exporter",
+  "tags": ["certificate-transparency", "ct", "uptime"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "5m",
+  "time": {
+    "from": "now-7d",
+    "to": "now"
+  },
+  "timepicker": {},
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "type": "datasource",
+        "pluginId": "prometheus",
+        "label": "Datasource",
+        "hide": 0,
+        "current": {},
+        "options": []
+      },
+      {
+        "name": "log_url",
+        "type": "query",
+        "label": "Log URL",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(ct_log_uptime_ratio, log_url)",
+        "query": {
+          "query": "label_values(ct_log_uptime_ratio, log_url)",
+          "refId": "StandardVariableQuery"
+        },
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".+",
+        "current": {},
+        "refresh": 2,
+        "sort": 1,
+        "hide": 0
+      },
+      {
+        "name": "endpoint",
+        "type": "query",
+        "label": "Endpoint",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(ct_log_uptime_ratio{log_url=~\"$log_url\"}, endpoint)",
+        "query": {
+          "query": "label_values(ct_log_uptime_ratio{log_url=~\"$log_url\"}, endpoint)",
+          "refId": "StandardVariableQuery"
+        },
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".+",
+        "current": {},
+        "refresh": 2,
+        "sort": 1,
+        "hide": 0
+      },
+      {
+        "name": "topN",
+        "type": "custom",
+        "label": "Top N",
+        "query": "5,10,25,50",
+        "current": {
+          "text": "10",
+          "value": "10",
+          "selected": true
+        },
+        "options": [
+          {"selected": false, "text": "5",  "value": "5"},
+          {"selected": true,  "text": "10", "value": "10"},
+          {"selected": false, "text": "25", "value": "25"},
+          {"selected": false, "text": "50", "value": "50"}
+        ],
+        "hide": 0
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "CT Logs",
+      "description": "Number of unique CT log URLs currently tracked",
+      "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "noValue": "0"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "count(count by (log_url) (ct_log_uptime_ratio))",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Endpoint Types",
+      "description": "Number of distinct endpoint operation types tracked",
+      "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "noValue": "0"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "count(count by (endpoint) (ct_log_uptime_ratio))",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Average Uptime",
+      "description": "Mean 24h uptime ratio across all tracked log/endpoint pairs",
+      "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "noValue": "-",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red",    "value": null},
+              {"color": "yellow", "value": 0.95},
+              {"color": "green",  "value": 0.99}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "avg(ct_log_uptime_ratio)",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Below 100% Uptime",
+      "description": "Number of log/endpoint pairs with uptime below 100% right now",
+      "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green",  "value": null},
+              {"color": "yellow", "value": 1},
+              {"color": "red",    "value": 5}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "count(ct_log_uptime_ratio < 1) or vector(0)",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "type": "stat",
+      "title": "Fetch Status",
+      "description": "Whether the last CSV fetch from gstatic.com succeeded",
+      "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center",
+        "mappings": [
+          {"type": "value", "options": {"0": {"text": "FAILED",  "color": "red"},
+                                        "1": {"text": "OK",      "color": "green"}}}
+        ]
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "noValue": "-",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red",   "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "ct_log_uptime_fetch_success",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Last Fetch",
+      "description": "Timestamp of the most recent CSV fetch attempt",
+      "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value",
+        "justifyMode": "center"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dateTimeFromNow",
+          "noValue": "-"
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "ct_log_uptime_fetch_timestamp_seconds * 1000",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "row",
+      "title": "Uptime Over Time",
+      "gridPos": {"x": 0, "y": 4, "w": 24, "h": 1},
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "24h Rolling Uptime",
+      "description": "How the rolling 24h uptime ratio has changed over the selected time range. Each series is one log/endpoint pair.",
+      "gridPos": {"x": 0, "y": 5, "w": 24, "h": 9},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "tooltip": {"mode": "multi", "sort": "asc"},
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": ["lastNotNull", "min"]
+        }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "custom": {
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "spanNulls": false
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red",    "value": null},
+              {"color": "yellow", "value": 0.95},
+              {"color": "green",  "value": 0.99}
+            ]
+          },
+          "color": {"mode": "palette-classic"}
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "ct_log_uptime_ratio{log_url=~\"$log_url\", endpoint=~\"$endpoint\"}",
+          "legendFormat": "{{log_url}} / {{endpoint}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "type": "row",
+      "title": "Least Available",
+      "gridPos": {"x": 0, "y": 14, "w": 24, "h": 1},
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "id": 10,
+      "type": "table",
+      "title": "Top $topN Least Available Log/Endpoint Pairs",
+      "description": "Current snapshot of the $topN log/endpoint pairs with the lowest 24h uptime ratio, filtered by the selected Log URL and Endpoint variables.",
+      "gridPos": {"x": 0, "y": 15, "w": 12, "h": 10},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "sortBy": [{"displayName": "Uptime", "desc": false}],
+        "footer": {"show": false}
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "left",
+            "displayMode": "auto"
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {"id": "byName", "options": "log_url"},
+            "properties": [
+              {"id": "displayName",   "value": "Log URL"},
+              {"id": "custom.width",  "value": 420}
+            ]
+          },
+          {
+            "matcher": {"id": "byName", "options": "endpoint"},
+            "properties": [
+              {"id": "displayName",  "value": "Endpoint"},
+              {"id": "custom.width", "value": 160}
+            ]
+          },
+          {
+            "matcher": {"id": "byName", "options": "Value #A"},
+            "properties": [
+              {"id": "displayName",          "value": "Uptime"},
+              {"id": "unit",                 "value": "percentunit"},
+              {"id": "custom.width",         "value": 120},
+              {"id": "custom.displayMode",   "value": "color-background"},
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {"color": "red",    "value": null},
+                    {"color": "yellow", "value": 0.95},
+                    {"color": "green",  "value": 0.99}
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "transformations": [
+        {"id": "merge", "options": {}},
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time":     true,
+              "__name__": true,
+              "instance": true,
+              "job":      true
+            },
+            "renameByName": {}
+          }
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "bottomk($topN, ct_log_uptime_ratio{log_url=~\"$log_url\", endpoint=~\"$endpoint\"})",
+          "instant": true,
+          "legendFormat": "{{log_url}} / {{endpoint}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "type": "table",
+      "title": "Logs with any endpoint below 100%",
+      "description": "CT logs where at least one endpoint has a 24h uptime below 100%, showing the worst (minimum) uptime across all endpoints for that log.",
+      "gridPos": {"x": 12, "y": 15, "w": 12, "h": 10},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "options": {
+        "sortBy": [{"displayName": "Uptime", "desc": false}],
+        "footer": {"show": false}
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "left",
+            "displayMode": "auto"
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {"id": "byName", "options": "log_url"},
+            "properties": [
+              {"id": "displayName",  "value": "Log URL"},
+              {"id": "custom.width", "value": 420}
+            ]
+          },
+          {
+            "matcher": {"id": "byName", "options": "Value #A"},
+            "properties": [
+              {"id": "displayName",        "value": "Uptime"},
+              {"id": "unit",               "value": "percentunit"},
+              {"id": "custom.width",       "value": 120},
+              {"id": "custom.displayMode", "value": "color-background"},
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    {"color": "red",    "value": null},
+                    {"color": "yellow", "value": 0.95},
+                    {"color": "green",  "value": 0.99}
+                  ]
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "transformations": [
+        {"id": "merge", "options": {}},
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time":     true,
+              "__name__": true,
+              "endpoint": true,
+              "instance": true,
+              "job":      true
+            },
+            "renameByName": {}
+          }
+        }
+      ],
+      "targets": [
+        {
+          "datasource": {"type": "prometheus", "uid": "${datasource}"},
+          "expr": "sort(min by (log_url) (ct_log_uptime_ratio{log_url=~\"$log_url\"}) < 1)",
+          "instant": true,
+          "legendFormat": "{{log_url}}",
+          "refId": "A"
+        }
+      ]
+    }
+  ]
+}

docker-compose.yml

@@ -0,0 +1,12 @@
+services:
+  ctlog-uptime-exporter:
+    build: .
+    image: git.ipng.ch/ipng/ctlog-uptime-exporter:latest
+    restart: unless-stopped
+    ports:
+      - "9781:9781"
+    environment:
+      LISTEN: ":9781"
+      URL: "https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv"
+      INTERVAL: "25m"
+      JITTER: "5m"

main.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"os"
 	"strconv"
 	"sync"
 	"time"
@@ -114,11 +115,30 @@ func (e *exporter) run(csvURL string, interval, jitter time.Duration) {
 	}
 }
 
+func envOr(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
 func main() {
-	addr := flag.String("listen", ":9781", "address to listen on")
-	csvURL := flag.String("url", "https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv", "URL of the uptime CSV")
-	interval := flag.Duration("interval", 25*time.Minute, "how often to fetch the CSV")
-	jitter := flag.Duration("jitter", 5*time.Minute, "maximum +/-jitter applied to the fetch interval")
+	addr := flag.String("listen", envOr("LISTEN", ":9781"), "address to listen on")
+	csvURL := flag.String("url", envOr("URL", "https://www.gstatic.com/ct/compliance/endpoint_uptime_24h.csv"), "URL of the uptime CSV")
+	interval := flag.Duration("interval", func() time.Duration {
+		d, err := time.ParseDuration(envOr("INTERVAL", "25m"))
+		if err != nil {
+			log.Fatalf("invalid INTERVAL: %v", err)
+		}
+		return d
+	}(), "how often to fetch the CSV")
+	jitter := flag.Duration("jitter", func() time.Duration {
+		d, err := time.ParseDuration(envOr("JITTER", "5m"))
+		if err != nil {
+			log.Fatalf("invalid JITTER: %v", err)
+		}
+		return d
+	}(), "maximum +/-jitter applied to the fetch interval")
 	flag.Parse()
 
 	reg := prometheus.NewRegistry()