certificate-transparency/ctool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor ctail+ctfetch into a common ctool

Browse Source
This commit is contained in:
Pim van Pelt
2026-04-06 01:36:21 +02:00
parent 418e83a83f
commit ba7f0dcb9f
7 changed files with 114 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -1,2 +1 @@
/ctfetch /ctool
/ctail

14
README.md
View File

@@ -5,27 +5,29 @@ Tools for working with [Static CT log](https://c2sp.org/static-ct-api) tiles.
## Install ## Install
```bash ```bash
GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/...@latest GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctool@latest
``` ```
## Tools The `GOPRIVATE` variable skips the Go checksum database and module proxy, which do not index modules on `git.ipng.ch`.
### ctfetch ## Commands
### ctool fetch
Fetch and decode entries from a Static CT log as structured JSON. Fetch and decode entries from a Static CT log as structured JSON.
```bash ```bash
ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all
``` ```
→ [Full documentation](docs/ctfetch.md) → [Full documentation](docs/ctfetch.md)
### ctail ### ctool tail
Tail a Static CT log, printing a one-liner per new cert/precert as it arrives. Tail a Static CT log, printing a one-liner per new cert/precert as it arrives.
```bash ```bash
ctail https://halloumi2026h1.mon.ct.ipng.ch ctool tail https://halloumi2026h1.mon.ct.ipng.ch
``` ```
→ [Full documentation](docs/ctail.md) → [Full documentation](docs/ctail.md)

65
cmd/ctfetch/main.go → cmd/ctool/cmd_fetch.go
View File

@@ -1,11 +1,3 @@
// Command ctfetch fetches and dumps entries from a Static CT log.
//
// Two modes:
//
// ctfetch [flags] <log-url> <leaf-index> [+sct] [+issuer] [+ctlog] [+all] fetch one entry by leaf index
// ctfetch [flags] <tile-url-or-file> [+sct] [+issuer] [+ctlog] [+all] dump all entries in a tile
//
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
package main package main
import ( import (
@@ -16,47 +8,52 @@ import (
"strconv" "strconv"
"strings" "strings"
"git.ipng.ch/certificate-transparency/ctfetch/internal/utils"
"filippo.io/sunlight" "filippo.io/sunlight"
"git.ipng.ch/certificate-transparency/ctfetch/internal/utils"
"golang.org/x/mod/sumdb/tlog" "golang.org/x/mod/sumdb/tlog"
) )
func main() { func runFetch(args []string) {
logsListURL := flag.String("logs-list-url", "https://www.gstatic.com/ct/log_list/v3/all_logs_list.json", "URL of the CT log list JSON") fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
monitoringURL := flag.String("monitoring-url", "", "log root URL for issuer lookups when input is a file") logsListURL := fs.String("logs-list-url", "https://www.gstatic.com/ct/log_list/v3/all_logs_list.json", "URL of the CT log list JSON")
flag.Usage = func() { monitoringURL := fs.String("monitoring-url", "", "log root URL for issuer lookups when input is a file")
fs.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "Usage:\n")
fmt.Fprintf(os.Stderr, " %s [flags] <log-url> <leaf-index> [+sct] [+issuer] [+ctlog] [+all] fetch one entry\n", os.Args[0]) fmt.Fprintf(os.Stderr, " ctool fetch [flags] <log-url> <leaf-index> [+sct] [+issuer] [+ctlog] [+all] fetch one entry\n")
fmt.Fprintf(os.Stderr, " %s [flags] <tile-url-or-file> [+sct] [+issuer] [+ctlog] [+all] dump all entries in a tile\n", os.Args[0]) fmt.Fprintf(os.Stderr, " ctool fetch [flags] <tile-url-or-file> [+sct] [+issuer] [+ctlog] [+all] dump all entries in a tile\n")
fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, "\nExamples:\n")
fmt.Fprintf(os.Stderr, " %s https://halloumi2026h1.mon.ct.ipng.ch 457683896 +sct +issuer +ctlog\n", os.Args[0]) fmt.Fprintf(os.Stderr, " ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all\n")
fmt.Fprintf(os.Stderr, " %s https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct\n", os.Args[0]) fmt.Fprintf(os.Stderr, " ctool fetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct\n")
fmt.Fprintf(os.Stderr, " %s --monitoring-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nFlags:\n") fmt.Fprintf(os.Stderr, "\nFlags:\n")
flag.PrintDefaults() fs.PrintDefaults()
} }
flag.Parse()
if flag.NArg() < 1 { if err := fs.Parse(args); err != nil {
flag.Usage() if err == flag.ErrHelp {
os.Exit(0)
}
os.Exit(1)
}
if fs.NArg() < 1 {
fs.Usage()
os.Exit(1) os.Exit(1)
} }
// Determine mode: if second positional arg parses as an integer → leaf-index mode. // Determine mode: if second positional arg parses as an integer → leaf-index mode.
_, secondIsInt := func() (int64, bool) { _, secondIsInt := func() (int64, bool) {
if flag.NArg() < 2 { if fs.NArg() < 2 {
return 0, false return 0, false
} }
v, err := strconv.ParseInt(flag.Arg(1), 10, 64) v, err := strconv.ParseInt(fs.Arg(1), 10, 64)
return v, err == nil return v, err == nil
}() }()
var modifiers []string var modifiers []string
if secondIsInt { if secondIsInt {
modifiers = flag.Args()[2:] modifiers = fs.Args()[2:]
} else { } else {
modifiers = flag.Args()[1:] modifiers = fs.Args()[1:]
} }
opts := utils.Options{} opts := utils.Options{}
@@ -87,9 +84,9 @@ func main() {
} }
if secondIsInt { if secondIsInt {
runLeafIndex(flag.Arg(0), flag.Arg(1), opts) runLeafIndex(fs.Arg(0), fs.Arg(1), opts)
} else { } else {
runTileDump(flag.Arg(0), *monitoringURL, opts) runTileDump(fs.Arg(0), *monitoringURL, opts)
} }
} }
@@ -104,16 +101,13 @@ func runLeafIndex(logURL, indexStr string, opts utils.Options) {
tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, leafIndex)) tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, leafIndex))
tile.L = -1 tile.L = -1
partialPath := sunlight.TilePath(tile) partialPath := sunlight.TilePath(tile)
positionInTile := leafIndex % sunlight.TileWidth positionInTile := leafIndex % sunlight.TileWidth
tileData, err := utils.FetchTile(logURL + "/" + partialPath) tileData, err := utils.FetchTile(logURL + "/" + partialPath)
if err != nil { if err != nil {
fatal("failed to fetch tile: %v", err) fatal("failed to fetch tile: %v", err)
} }
tileData, err = utils.Decompress(tileData) tileData, err = utils.Decompress(tileData)
if err != nil { if err != nil {
fatal("failed to decompress tile: %v", err) fatal("failed to decompress tile: %v", err)
@@ -131,7 +125,6 @@ func runTileDump(arg, monitoringURL string, opts utils.Options) {
var err error var err error
if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") { if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
// Derive log root from tile URL for issuer lookups.
if opts.ShowIssuer { if opts.ShowIssuer {
if idx := strings.Index(arg, "/tile/"); idx != -1 { if idx := strings.Index(arg, "/tile/"); idx != -1 {
opts.LogURL = strings.TrimSuffix(arg[:idx], "/") opts.LogURL = strings.TrimSuffix(arg[:idx], "/")
@@ -146,7 +139,6 @@ func runTileDump(arg, monitoringURL string, opts utils.Options) {
fatal("failed to fetch tile: %v", err) fatal("failed to fetch tile: %v", err)
} }
} else { } else {
// File input.
if opts.ShowIssuer { if opts.ShowIssuer {
if monitoringURL != "" { if monitoringURL != "" {
opts.LogURL = strings.TrimSuffix(monitoringURL, "/") opts.LogURL = strings.TrimSuffix(monitoringURL, "/")
@@ -179,8 +171,3 @@ func printJSON(v interface{}) {
} }
fmt.Println(string(data)) fmt.Println(string(data))
} }
func fatal(format string, args ...any) {
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...)
os.Exit(1)
}

42
cmd/ctail/main.go → cmd/ctool/cmd_tail.go
View File

@@ -1,8 +1,3 @@
// Command ctail tails a Static CT log, printing a one-liner per new entry.
//
// ctail [flags] <log-url>
//
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
package main package main
import ( import (
@@ -23,29 +18,34 @@ import (
"golang.org/x/mod/sumdb/tlog" "golang.org/x/mod/sumdb/tlog"
) )
const version = "0.1.0"
var ( var (
userAgent string userAgent string
rateLimit time.Duration rateLimit time.Duration
lastRequest time.Time lastRequest time.Time
) )
func main() { func runTail(args []string) {
interval := flag.Duration("interval", 15*time.Second, "polling interval") fs := flag.NewFlagSet("tail", flag.ContinueOnError)
fromLeaf := flag.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)") interval := fs.Duration("interval", 15*time.Second, "polling interval (minimum 1s)")
rateLimitFlag := flag.Duration("rate-limit", 2*time.Second, "minimum time between HTTP requests") fromLeaf := fs.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)")
flag.StringVar(&userAgent, "user-agent", "ctail/"+version+" (https://git.ipng.ch/certificate-transparency/)", "User-Agent header for HTTP requests") rateLimitFlag := fs.Duration("rate-limit", 2*time.Second, "minimum time between HTTP requests (minimum 100ms)")
flag.Usage = func() { fs.StringVar(&userAgent, "user-agent", "ctool/"+version+" (https://git.ipng.ch/certificate-transparency/)", "User-Agent header for HTTP requests")
fmt.Fprintf(os.Stderr, "Usage: ctail [flags] <log-url>\n") fs.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: ctool tail [flags] <log-url>\n")
fmt.Fprintf(os.Stderr, "\nPrints a one-liner per cert/pre-cert as new entries arrive in a Static CT log.\n") fmt.Fprintf(os.Stderr, "\nPrints a one-liner per cert/pre-cert as new entries arrive in a Static CT log.\n")
fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, "\nExamples:\n")
fmt.Fprintf(os.Stderr, " ctail https://halloumi2026h1.mon.ct.ipng.ch\n") fmt.Fprintf(os.Stderr, " ctool tail https://halloumi2026h2.mon.ct.ipng.ch\n")
fmt.Fprintf(os.Stderr, " ctail --from-leaf 0 --interval 10s https://halloumi2026h1.mon.ct.ipng.ch\n") fmt.Fprintf(os.Stderr, " ctool tail --from-leaf 0 --interval 10s https://halloumi2026h2.mon.ct.ipng.ch\n")
fmt.Fprintf(os.Stderr, "\nFlags:\n") fmt.Fprintf(os.Stderr, "\nFlags:\n")
flag.PrintDefaults() fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
os.Exit(0)
}
os.Exit(1)
} }
flag.Parse()
if *interval < time.Second { if *interval < time.Second {
fmt.Fprintf(os.Stderr, "Error: --interval must be at least 1s\n") fmt.Fprintf(os.Stderr, "Error: --interval must be at least 1s\n")
@@ -57,12 +57,12 @@ func main() {
} }
rateLimit = *rateLimitFlag rateLimit = *rateLimitFlag
if flag.NArg() != 1 { if fs.NArg() != 1 {
flag.Usage() fs.Usage()
os.Exit(1) os.Exit(1)
} }
logURL := strings.TrimSuffix(flag.Arg(0), "/") logURL := strings.TrimSuffix(fs.Arg(0), "/")
var nextLeaf int64 = -1 var nextLeaf int64 = -1
for { for {

44
cmd/ctool/main.go Normal file
View File

@@ -0,0 +1,44 @@
// Command ctool provides tools for working with Static CT log tiles.
//
// ctool <command> [flags] ...
//
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
package main
import (
"fmt"
"os"
)
const version = "0.1.0"
func main() {
if len(os.Args) < 2 {
usage()
os.Exit(1)
}
switch os.Args[1] {
case "fetch":
runFetch(os.Args[2:])
case "tail":
runTail(os.Args[2:])
default:
fmt.Fprintf(os.Stderr, "Error: unknown command %q\n\n", os.Args[1])
usage()
os.Exit(1)
}
}
func usage() {
fmt.Fprintf(os.Stderr, "Usage: ctool <command> [flags] ...\n\n")
fmt.Fprintf(os.Stderr, "Commands:\n")
fmt.Fprintf(os.Stderr, " fetch fetch and decode CT log entries as JSON\n")
fmt.Fprintf(os.Stderr, " tail tail a CT log, printing one-liners for new entries\n")
fmt.Fprintf(os.Stderr, "\nRun 'ctool <command> --help' for command-specific flags.\n")
}
func fatal(format string, args ...any) {
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...)
os.Exit(1)
}

8
docs/ctail.md
View File

@@ -5,22 +5,22 @@ Tail a Static CT log, printing a one-liner per new certificate or precertificate
## Install ## Install
```bash ```bash
GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctail@latest GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctool@latest
``` ```
## Usage ## Usage
```bash ```bash
ctail [flags] <log-url> ctool tail [flags] <log-url>
``` ```
Example: Example:
```bash ```bash
ctail https://halloumi2026h1.mon.ct.ipng.ch ctool tail https://halloumi2026h2.mon.ct.ipng.ch
``` ```
By default `ctail` starts at the current tree tip and prints new entries as they appear. Use `--from-leaf 0` to replay from the beginning. By default `ctool tail` starts at the current tree tip and prints new entries as they appear. Use `--from-leaf 0` to replay from the beginning.
## Output format ## Output format

22
docs/ctfetch.md
View File

@@ -5,28 +5,26 @@ Fetch and decode entries from a Static CT log, outputting structured JSON.
## Install ## Install
```bash ```bash
GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctfetch@latest GOPRIVATE=git.ipng.ch go install git.ipng.ch/certificate-transparency/ctfetch/cmd/ctool@latest
``` ```
The `GOPRIVATE` variable skips the Go checksum database and module proxy, which do not index modules on `git.ipng.ch`.
## Modes ## Modes
`ctfetch` operates in two modes depending on the arguments given. `ctool fetch` operates in two modes depending on the arguments given.
### Leaf-index mode ### Leaf-index mode
Fetch the entry at a specific leaf index: Fetch the entry at a specific leaf index:
```bash ```bash
ctfetch [flags] <log-url> <leaf-index> [+sct] [+issuer] [+ctlog] [+all] ctool fetch [flags] <log-url> <leaf-index> [+sct] [+issuer] [+ctlog] [+all]
``` ```
Examples: Examples:
```bash ```bash
ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635
ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all
``` ```
### Tile-dump mode ### Tile-dump mode
@@ -34,16 +32,16 @@ ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all
Fetch all entries from a tile URL or local file. Automatically detects data tiles (log entries) and hash tiles (Merkle tree hashes). Fetch all entries from a tile URL or local file. Automatically detects data tiles (log entries) and hash tiles (Merkle tree hashes).
```bash ```bash
ctfetch [flags] <tile-url-or-file> [+sct] [+issuer] [+ctlog] [+all] ctool fetch [flags] <tile-url-or-file> [+sct] [+issuer] [+ctlog] [+all]
``` ```
Examples: Examples:
```bash ```bash
ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 ctool fetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135
ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog ctool fetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog
ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/0/x100/999 ctool fetch https://halloumi2026h1.mon.ct.ipng.ch/tile/0/x100/999
ctfetch --monitoring-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer ctool fetch --monitoring-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer
``` ```
## Output modifiers ## Output modifiers