From 418e83a83f63776467a3480254baf730e7f30af6 Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Mon, 6 Apr 2026 01:29:38 +0200 Subject: [PATCH] Add ctail, refactor README --- .gitignore | 2 +- README.md | 91 +++-------------- cmd/ctail/main.go | 241 ++++++++++++++++++++++++++++++++++++++++++++++ docs/ctail.md | 57 +++++++++++ docs/ctfetch.md | 75 +++++++++++++++ 5 files changed, 386 insertions(+), 80 deletions(-) create mode 100644 cmd/ctail/main.go create mode 100644 docs/ctail.md create mode 100644 docs/ctfetch.md diff --git a/.gitignore b/.gitignore index 9f89c00..45d19a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /ctfetch -/tiledump +/ctail diff --git a/README.md b/README.md index df08870..42d6155 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,31 @@ -# ctfetch +# Certificate Transparency tools -Tools for working with Certificate Transparency log tiles. +Tools for working with [Static CT log](https://c2sp.org/static-ct-api) tiles. ## Install ```bash -GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctfetch@latest +GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/...@latest ``` -The GOPRIVATE environment variable skips _Go checksum database_ and _Go module proxy_ as these do -not index modules on `git.ipng.ch`. +## Tools -## Usage +### ctfetch -`ctfetch` operates in two modes depending on the arguments given. +Fetch and decode entries from a Static CT log as structured JSON. -### Leaf-index mode - -Fetch a specific entry (or all entries in its tile) by leaf index: - -```bash -ctfetch [flags] [+sct] [+issuer] [+ctlog] [+all] -``` - -**Examples:** - -Dump a specific entry: -```bash -ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 -``` - -Dump with SCTs, issuer chain, and CT log details: ```bash ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all ``` -### Tile-dump mode +→ [Full documentation](docs/ctfetch.md) -Fetch all entries from a tile URL or a local file. Automatically detects data tiles (log entries) and hash tiles (Merkle tree hashes). +### ctail + +Tail a Static CT log, printing a one-liner per new cert/precert as it arrives. ```bash -ctfetch [flags] [+sct] [+issuer] [+ctlog] [+all] +ctail https://halloumi2026h1.mon.ct.ipng.ch ``` -**Examples:** - -Data tile from a URL: -```bash -ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 -``` - -Data tile with SCTs and CT log details: -```bash -ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog -``` - -Hash tile from a URL: -```bash -ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/0/x100/999 -``` - -Data tile from a local file (with issuer resolution): -```bash -ctfetch --monitoring-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer -``` - -## Hash tiles vs data tiles - -A Static CT log stores two kinds of tiles: - -**Data tiles** (`/tile/data/...`) contain the actual log entries — DER-encoded certificates and precertificates along with their metadata (leaf index, timestamp, chain fingerprints, etc.). These are what `ctfetch` parses into structured JSON. The output modifiers `+sct`, `+issuer`, `+ctlog`, and `+all` all operate on data tiles. - -**Hash tiles** (`/tile/N/...`, where N is a tree level ≥ 0) contain the internal nodes of the Merkle tree — rows of raw 32-byte SHA-256 hashes used for inclusion and consistency proofs. There are no certificates in a hash tile; `ctfetch` outputs only the list of hashes. Using `+sct`, `+issuer`, `+ctlog`, or `+all` with a hash tile is an error. - -The tree is organised so that level 0 hashes cover individual leaves (each is `SHA-256(0x00 || MerkleTreeLeaf)`), and each higher level hashes pairs of nodes from the level below. The tile URL encodes the level: `/tile/0/...` is level 0, `/tile/1/...` is level 1, and so on. - -## Output modifiers - -| Modifier | Description | -|---|---| -| `+sct` | Parse and include embedded Signed Certificate Timestamps from final (non-precert) certificates | -| `+issuer` | Fetch and include issuer certificate details from the log's `/issuer/` endpoint | -| `+ctlog` | Look up each SCT's log ID in the CT log list and include operator/state details | -| `+all` | Enable all of `+sct`, `+issuer`, and `+ctlog` at once | - -## Flags - -| Flag | Default | Description | -|---|---|---| -| `--logs-list-url` | `https://www.gstatic.com/ct/log_list/v3/all_logs_list.json` | URL of the CT log list JSON used for `+ctlog` lookups | -| `--monitoring-url` | _(none)_ | Log root URL for issuer lookups when input is a local file | - -## Notes - -- In tile-dump mode with a tile URL, `+issuer` automatically derives the log root by stripping the `/tile/...` path. With a local file, `--monitoring-url` must be provided. -- Partial tiles (`.p/N` suffix) are tried first; on 404 the full tile is fetched automatically. -- The CT log list and issuer certificates are cached in memory, so each unique resource is fetched only once per invocation. +→ [Full documentation](docs/ctail.md) diff --git a/cmd/ctail/main.go b/cmd/ctail/main.go new file mode 100644 index 0000000..f43fc5b --- /dev/null +++ b/cmd/ctail/main.go @@ -0,0 +1,241 @@ +// Command ctail tails a Static CT log, printing a one-liner per new entry. +// +// ctail [flags] +// +// (C) Copyright 2026 Pim van Pelt +package main + +import ( + "bufio" + "bytes" + "crypto/x509" + "flag" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "filippo.io/sunlight" + "git.ipng.ch/certificate-transparency/ctfetch/internal/utils" + "golang.org/x/mod/sumdb/tlog" +) + +const version = "0.1.0" + +var ( + userAgent string + rateLimit time.Duration + lastRequest time.Time +) + +func main() { + interval := flag.Duration("interval", 15*time.Second, "polling interval") + fromLeaf := flag.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)") + rateLimitFlag := flag.Duration("rate-limit", 2*time.Second, "minimum time between HTTP requests") + flag.StringVar(&userAgent, "user-agent", "ctail/"+version+" (https://git.ipng.ch/certificate-transparency/)", "User-Agent header for HTTP requests") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: ctail [flags] \n") + fmt.Fprintf(os.Stderr, "\nPrints a one-liner per cert/pre-cert as new entries arrive in a Static CT log.\n") + fmt.Fprintf(os.Stderr, "\nExamples:\n") + fmt.Fprintf(os.Stderr, " ctail https://halloumi2026h1.mon.ct.ipng.ch\n") + fmt.Fprintf(os.Stderr, " ctail --from-leaf 0 --interval 10s https://halloumi2026h1.mon.ct.ipng.ch\n") + fmt.Fprintf(os.Stderr, "\nFlags:\n") + flag.PrintDefaults() + } + flag.Parse() + + if *interval < time.Second { + fmt.Fprintf(os.Stderr, "Error: --interval must be at least 1s\n") + os.Exit(1) + } + if *rateLimitFlag < 100*time.Millisecond { + fmt.Fprintf(os.Stderr, "Error: --rate-limit must be at least 100ms\n") + os.Exit(1) + } + rateLimit = *rateLimitFlag + + if flag.NArg() != 1 { + flag.Usage() + os.Exit(1) + } + + logURL := strings.TrimSuffix(flag.Arg(0), "/") + var nextLeaf int64 = -1 + + for { + checkpointAt := time.Now() + treeSize, err := fetchCheckpointSize(logURL) + if err != nil { + fmt.Fprintf(os.Stderr, "checkpoint error: %v\n", err) + time.Sleep(*interval) + continue + } + + if nextLeaf < 0 { + if *fromLeaf >= 0 { + fmt.Fprintf(os.Stderr, "Starting from leaf %d, tree size %d, polling every %s\n", *fromLeaf, treeSize, *interval) + nextLeaf = *fromLeaf + } else { + fmt.Fprintf(os.Stderr, "Starting at tree size %d, polling every %s\n", treeSize, *interval) + nextLeaf = treeSize + } + } + + if err := processNewEntries(logURL, &nextLeaf, treeSize); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + + if wait := time.Until(checkpointAt.Add(*interval)); wait > 0 { + time.Sleep(wait) + } + } +} + +func processNewEntries(logURL string, nextLeaf *int64, treeSize int64) error { + for *nextLeaf < treeSize { + tileStart := (*nextLeaf / sunlight.TileWidth) * sunlight.TileWidth + if treeSize < tileStart+sunlight.TileWidth { + break // tile not yet complete; wait for next poll + } + + tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, *nextLeaf)) + tile.L = -1 + tilePath := sunlight.TilePath(tile) + // Always fetch the full tile; strip any .p/N partial suffix. + if idx := strings.Index(tilePath, ".p/"); idx != -1 { + tilePath = tilePath[:idx] + } + + tileData, err := fetchURL(logURL + "/" + tilePath) + if err != nil { + return fmt.Errorf("fetch tile at leaf %d: %w", *nextLeaf, err) + } + tileData, err = utils.Decompress(tileData) + if err != nil { + return fmt.Errorf("decompress tile: %w", err) + } + + prevNext := *nextLeaf + remaining := tileData + for len(remaining) > 0 { + e, rest, err := sunlight.ReadTileLeaf(remaining) + if err != nil { + return fmt.Errorf("read tile leaf: %w", err) + } + remaining = rest + if e.LeafIndex >= *nextLeaf { + fmt.Println(formatEntry(e)) + *nextLeaf = e.LeafIndex + 1 + } + } + if *nextLeaf == prevNext { + return fmt.Errorf("no progress reading tile at leaf %d", *nextLeaf) + } + } + return nil +} + +func fetchURL(url string) ([]byte, error) { + if wait := time.Until(lastRequest.Add(rateLimit)); wait > 0 { + time.Sleep(wait) + } + lastRequest = time.Now() + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +func fetchCheckpointSize(logURL string) (int64, error) { + data, err := fetchURL(logURL + "/checkpoint") + if err != nil { + return 0, err + } + // Checkpoint format (tlog/sunlight): + // line 1: origin + // line 2: tree size (decimal) + // line 3: root hash (base64) + // (blank line + signature lines) + scanner := bufio.NewScanner(bytes.NewReader(data)) + for i := 0; scanner.Scan(); i++ { + if i == 1 { + return strconv.ParseInt(scanner.Text(), 10, 64) + } + } + return 0, fmt.Errorf("checkpoint too short") +} + +// formatEntry formats a log entry as a single ~100-char line. +func formatEntry(e *sunlight.LogEntry) string { + certType := "cert" + if e.IsPrecert { + certType = "pre " + } + + certDER := e.Certificate + if e.IsPrecert && len(e.PreCertificate) > 0 { + certDER = e.PreCertificate + } + + name := "(unknown)" + issuer := "" + validRange := "" + + // x509.ParseCertificate may return a non-nil cert even on error (e.g. precerts + // with the CT poison critical extension), so we use the cert regardless. + cert, _ := x509.ParseCertificate(certDER) + if cert != nil { + if len(cert.DNSNames) > 0 { + name = cert.DNSNames[0] + } else if cert.Subject.CommonName != "" { + name = cert.Subject.CommonName + } + issuer = issuerLabel(cert) + notBefore := cert.NotBefore.UTC().Format("2006-01-02") + notAfter := cert.NotAfter.UTC().Format("2006-01-02") + validRange = notBefore + ".." + notAfter + } + + return fmt.Sprintf("%9d %-4s %-21s %-40s %s", + e.LeafIndex, certType, + validRange, trunc(issuer, 40), name) +} + +// issuerLabel builds a human-readable issuer string. When the CN is terse +// (e.g. "R13") and the org name isn't already embedded in it, we prepend the +// org so the result reads as "Let's Encrypt R13" instead of just "R13". +func issuerLabel(cert *x509.Certificate) string { + cn := cert.Issuer.CommonName + if len(cert.Issuer.Organization) == 0 { + return cn + } + org := cert.Issuer.Organization[0] + // Only prepend if the CN doesn't already contain the org's first word. + firstWord := strings.Fields(org)[0] + if strings.Contains(cn, firstWord) { + return cn + } + return org + " " + cn +} + +// trunc truncates s to at most n bytes, appending "..." if truncated. +func trunc(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-3] + "..." +} diff --git a/docs/ctail.md b/docs/ctail.md new file mode 100644 index 0000000..bb03260 --- /dev/null +++ b/docs/ctail.md @@ -0,0 +1,57 @@ +# ctail + +Tail a Static CT log, printing a one-liner per new certificate or precertificate as it arrives. + +## Install + +```bash +GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctail@latest +``` + +## Usage + +```bash +ctail [flags] +``` + +Example: + +```bash +ctail https://halloumi2026h1.mon.ct.ipng.ch +``` + +By default `ctail` starts at the current tree tip and prints new entries as they appear. Use `--from-leaf 0` to replay from the beginning. + +## Output format + +One line per entry: + +``` + leaf-index type validity-range issuer (up to 40 chars) subject name +``` + +Example: + +``` +1440154358 cert 2026-03-31..2026-06-29 Let's Encrypt R13 bereavementcounselling.uk +1440154359 pre 2026-03-31..2026-06-29 ZeroSSL ECC Domain Secured Ce... alpenglowforeverfilms.com +``` + +- **type**: `cert` for a final certificate, `pre` for a precertificate +- **issuer**: CommonName, prefixed with the organisation name when the CN alone is terse (e.g. `R13` → `Let's Encrypt R13`) +- **subject name**: first DNS SAN, falling back to the certificate's CommonName + +## Flags + +| Flag | Default | Description | +|---|---|---| +| `--interval` | `15s` | How often to poll the checkpoint (minimum 1s) | +| `--from-leaf` | `-1` | Start from this leaf index; `-1` means current tree tip | +| `--rate-limit` | `2s` | Minimum time between HTTP requests (minimum 100ms) | +| `--user-agent` | `ctail/VERSION (https://git.ipng.ch/certificate-transparency/)` | User-Agent header sent with every request | + +## Notes + +- The interval timer starts when the checkpoint is fetched, so tile-fetch time counts against the interval and the next poll stays on schedule. +- A tile is only fetched once the checkpoint confirms it is complete (256 entries). This avoids unnecessary 404s at the tree tip. +- Status and error messages go to stderr; the entry one-liners go to stdout. diff --git a/docs/ctfetch.md b/docs/ctfetch.md new file mode 100644 index 0000000..123c34a --- /dev/null +++ b/docs/ctfetch.md @@ -0,0 +1,75 @@ +# ctfetch + +Fetch and decode entries from a Static CT log, outputting structured JSON. + +## Install + +```bash +GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctfetch@latest +``` + +The `GOPRIVATE` variable skips the Go checksum database and module proxy, which do not index modules on `git.ipng.ch`. + +## Modes + +`ctfetch` operates in two modes depending on the arguments given. + +### Leaf-index mode + +Fetch the entry at a specific leaf index: + +```bash +ctfetch [flags] [+sct] [+issuer] [+ctlog] [+all] +``` + +Examples: + +```bash +ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all +``` + +### Tile-dump mode + +Fetch all entries from a tile URL or local file. Automatically detects data tiles (log entries) and hash tiles (Merkle tree hashes). + +```bash +ctfetch [flags] [+sct] [+issuer] [+ctlog] [+all] +``` + +Examples: + +```bash +ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog +ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/0/x100/999 +ctfetch --monitoring-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer +``` + +## Output modifiers + +| Modifier | Description | +|---|---| +| `+sct` | Parse embedded Signed Certificate Timestamps from final (non-precert) certificates | +| `+issuer` | Fetch issuer certificate details from the log's `/issuer/` endpoint | +| `+ctlog` | Look up each SCT's log ID in the CT log list and include operator/state details | +| `+all` | Enable all of `+sct`, `+issuer`, and `+ctlog` | + +## Flags + +| Flag | Default | Description | +|---|---|---| +| `--logs-list-url` | `https://www.gstatic.com/ct/log_list/v3/all_logs_list.json` | CT log list URL for `+ctlog` lookups | +| `--monitoring-url` | _(none)_ | Log root URL for issuer lookups when input is a local file | + +## Hash tiles vs data tiles + +**Data tiles** (`/tile/data/...`) contain DER-encoded certificates and precertificates with metadata (leaf index, timestamp, chain fingerprints). Output modifiers `+sct`, `+issuer`, `+ctlog`, and `+all` only apply here. + +**Hash tiles** (`/tile/N/...`, N ≥ 0) contain raw 32-byte SHA-256 hashes — the internal nodes of the Merkle tree used for inclusion and consistency proofs. `ctfetch` outputs only the list of hashes; using output modifiers with a hash tile is an error. + +## Notes + +- With a tile URL, `+issuer` derives the log root by stripping the `/tile/...` path. With a local file, `--monitoring-url` must be provided. +- Partial tiles (`.p/N` suffix) are tried first; on 404 the full tile is fetched automatically. +- The CT log list and issuer certificates are cached in memory per invocation.