diff --git a/.gitignore b/.gitignore index 9afef4e..2c7d4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /ctool /ctfetch /ctail +/build diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89d5e02 --- /dev/null +++ b/Makefile @@ -0,0 +1,160 @@ +BINARIES := ctool +MODULE := git.ipng.ch/certificate-transparency/ctfetch + +NATIVE_ARCH := $(shell go env GOARCH) +VERSION := 0.1.0 +COMMIT_HASH := $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -X 'main.version=$(VERSION)' \ + -X 'main.commit=$(COMMIT_HASH)' \ + -X 'main.date=$(DATE)' + +# CGO_ENABLED=0 produces fully static binaries: no libc dependency, so the +# debian package runs on any Linux-amd64/arm64 host regardless of glibc +# version (or musl). The only behavioural change is that Go's net package +# uses the pure-Go resolver instead of libc's getaddrinfo, which skips +# /etc/nsswitch.conf and NSS modules — fine for DNS-only lookups, which +# is all ctool does when resolving log hostnames. +export CGO_ENABLED := 0 + +# GO_VERSION is what install-deps-go downloads from go.dev when the +# system Go is missing or older than this. Matches the go.mod floor. +# Override on the command line to pull a specific patch release: +# make install-deps GO_VERSION=1.25.0 +GO_VERSION ?= 1.24.6 + +# GOLANGCI_LINT_VERSION is the minimum golangci-lint version that +# install-deps-go-tools accepts. install-deps-go-tools always `go +# install`s @latest, then asserts the resulting binary reports a +# version >= this floor as a sanity check. +GOLANGCI_LINT_VERSION ?= 1.64.0 + +.PHONY: all build build-amd64 build-arm64 test lint fixstyle pkg-deb clean install-deps install-deps-apt install-deps-go install-deps-go-tools + +all: build + +build: + mkdir -p build/$(NATIVE_ARCH) + go build -ldflags "$(LDFLAGS)" -o build/$(NATIVE_ARCH)/ctool ./cmd/ctool/ + +build-amd64: + mkdir -p build/amd64 + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o build/amd64/ctool ./cmd/ctool/ + +build-arm64: + mkdir -p build/arm64 + GOOS=linux GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o build/arm64/ctool ./cmd/ctool/ + +# pkg-deb produces one .deb per architecture. Each package contains a +# single statically-linked ctool binary plus ctfetch and ctail symlinks, +# and the three section-1 manpages. +pkg-deb: build-amd64 build-arm64 + debian/build-deb.sh ctool amd64 $(VERSION) + debian/build-deb.sh ctool arm64 $(VERSION) + +test: + go test ./... + +lint: + golangci-lint run ./... + +fixstyle: + gofmt -w . + +# install-deps is an opt-in "set up a fresh developer box" target. Tested +# on Debian Trixie; the apt half should also work on Bookworm and recent +# Ubuntu LTS. Splits into three sub-targets so they can be run individually: +# +# install-deps-apt — Debian-packaged build-time deps (git, make, +# dpkg-dev, curl, tar). +# install-deps-go — ensure a Go toolchain >= $(GO_VERSION) is on +# the system. Downloads the upstream tarball +# into /usr/local/go when the system Go is +# missing or older than the go.mod floor. +# install-deps-go-tools — `go install` the helpers this repo needs +# (golangci-lint) and assert it is new enough +# to understand current Go syntax. +# +# Each sub-target is idempotent and safe to re-run. +install-deps: install-deps-apt install-deps-go install-deps-go-tools + @echo "" + @echo "==> All build dependencies installed." + @echo " Make sure these are on PATH:" + @echo " /usr/local/go/bin (Go toolchain)" + @echo " \$$(go env GOPATH)/bin (golangci-lint, ...)" + +install-deps-apt: + @set -eu; \ + if [ "$$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo"; fi; \ + echo "==> Installing apt packages (git, make, dpkg-dev, curl, tar)"; \ + $$SUDO apt-get update; \ + $$SUDO apt-get install -y --no-install-recommends \ + git make dpkg-dev ca-certificates curl tar + +# install-deps-go short-circuits when go env GOVERSION already reports a +# version >= GO_VERSION. Otherwise it downloads the official upstream +# tarball (https://go.dev/dl/) and extracts it to /usr/local/go, matching +# the layout that go.dev recommends and that most Debian setups use for +# "Go newer than apt provides". +install-deps-go: + @set -eu; \ + if [ "$$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo"; fi; \ + echo "==> Checking Go toolchain (required: $(GO_VERSION)+)"; \ + if command -v go >/dev/null 2>&1; then \ + CURRENT=$$(go env GOVERSION 2>/dev/null | sed 's/^go//'); \ + OLDEST=$$(printf '%s\n%s\n' "$(GO_VERSION)" "$$CURRENT" | sort -V | head -n1); \ + if [ "$$OLDEST" = "$(GO_VERSION)" ] && [ -n "$$CURRENT" ]; then \ + echo " go$$CURRENT already installed (>= $(GO_VERSION)), skipping."; \ + exit 0; \ + fi; \ + echo " go$$CURRENT is older than $(GO_VERSION), upgrading."; \ + else \ + echo " no Go toolchain on PATH, installing."; \ + fi; \ + DEB_ARCH=$$(dpkg --print-architecture); \ + case "$$DEB_ARCH" in \ + amd64) GOARCH=amd64 ;; \ + arm64) GOARCH=arm64 ;; \ + armhf) GOARCH=armv6l ;; \ + *) echo " unsupported architecture: $$DEB_ARCH" >&2; exit 1 ;; \ + esac; \ + TARBALL="go$(GO_VERSION).linux-$$GOARCH.tar.gz"; \ + URL="https://go.dev/dl/$$TARBALL"; \ + echo " downloading $$URL"; \ + curl -fsSL -o "/tmp/$$TARBALL" "$$URL"; \ + echo " installing to /usr/local/go"; \ + $$SUDO rm -rf /usr/local/go; \ + $$SUDO tar -C /usr/local -xzf "/tmp/$$TARBALL"; \ + rm -f "/tmp/$$TARBALL"; \ + echo " installed $$(/usr/local/go/bin/go version)" + +install-deps-go-tools: + @set -eu; \ + if ! command -v go >/dev/null 2>&1; then \ + export PATH="/usr/local/go/bin:$$PATH"; \ + fi; \ + echo "==> Installing Go tools via 'go install'"; \ + echo " github.com/golangci/golangci-lint/v2/cmd/golangci-lint"; \ + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest; \ + GOBIN="$$(go env GOBIN)"; \ + if [ -z "$$GOBIN" ]; then GOBIN="$$(go env GOPATH)/bin"; fi; \ + echo "==> Asserting golangci-lint version >= $(GOLANGCI_LINT_VERSION)"; \ + if ! "$$GOBIN/golangci-lint" version >/dev/null 2>&1; then \ + echo " ERROR: $$GOBIN/golangci-lint is not executable" >&2; \ + exit 1; \ + fi; \ + INSTALLED=$$("$$GOBIN/golangci-lint" version 2>&1 | sed -En 's/.*has version v?([0-9][0-9.]*).*/\1/p' | head -n1); \ + if [ -z "$$INSTALLED" ]; then \ + echo " ERROR: could not parse golangci-lint version output" >&2; \ + "$$GOBIN/golangci-lint" version >&2; \ + exit 1; \ + fi; \ + OLDEST=$$(printf '%s\n%s\n' "$(GOLANGCI_LINT_VERSION)" "$$INSTALLED" | sort -V | head -n1); \ + if [ "$$OLDEST" != "$(GOLANGCI_LINT_VERSION)" ]; then \ + echo " ERROR: golangci-lint $$INSTALLED is older than the required $(GOLANGCI_LINT_VERSION)" >&2; \ + exit 1; \ + fi; \ + echo " golangci-lint $$INSTALLED (>= $(GOLANGCI_LINT_VERSION)) OK" + +clean: + rm -rf build/ diff --git a/cmd/ctool/cmd_fetch.go b/cmd/ctool/cmd_fetch.go index 9e62ebf..e3dd286 100644 --- a/cmd/ctool/cmd_fetch.go +++ b/cmd/ctool/cmd_fetch.go @@ -14,6 +14,10 @@ import ( ) func runFetch(args []string) { + if len(args) > 0 && args[0] == "version" { + printVersion() + return + } fs := flag.NewFlagSet("fetch", flag.ContinueOnError) logsListURL := fs.String("logs-list-url", "https://www.gstatic.com/ct/log_list/v3/all_logs_list.json", "URL of the CT log list JSON") monitoringURL := fs.String("monitoring-url", "", "log root URL for issuer lookups when input is a file") diff --git a/cmd/ctool/cmd_fetch_test.go b/cmd/ctool/cmd_fetch_test.go new file mode 100644 index 0000000..40900d8 --- /dev/null +++ b/cmd/ctool/cmd_fetch_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "strings" + "testing" +) + +func TestFetchNoArgsShowsUsage(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "fetch") + if exit == 0 { + t.Errorf("expected non-zero exit, got 0 (stderr=%q)", stderr) + } + if !strings.Contains(stderr, "Usage:") { + t.Errorf("stderr missing usage header: %q", stderr) + } + if !strings.Contains(stderr, "ctool fetch") { + t.Errorf("stderr missing qualified command name: %q", stderr) + } +} + +// When invoked via the ctfetch symlink, the usage banner must use the +// symlink name ("ctfetch"), not "ctool fetch". This exercises cmdName's +// symlink branch through the real CLI. +func TestFetchUsageUsesSymlinkName(t *testing.T) { + _, stderr, exit := runCLI(t, "ctfetch") + if exit == 0 { + t.Errorf("expected non-zero exit, got 0") + } + if !strings.Contains(stderr, "ctfetch") { + t.Errorf("stderr should mention ctfetch: %q", stderr) + } + if strings.Contains(stderr, "ctool fetch") { + t.Errorf("stderr should NOT say 'ctool fetch' when invoked via symlink: %q", stderr) + } +} + +func TestFetchHelpExitsZero(t *testing.T) { + _, _, exit := runCLI(t, "ctool", "fetch", "-h") + if exit != 0 { + t.Errorf("expected exit 0 for -h, got %d", exit) + } +} + +// An unknown +modifier must be rejected before any network request is made. +// The fetch dispatcher validates the modifier enum (sct/issuer/ctlog/all) +// up-front; we pass a junk URL to prove the error happens during parse, +// not during HTTP. +func TestFetchRejectsUnknownModifier(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "fetch", "http://invalid.invalid", "1", "+bogus") + if exit == 0 { + t.Errorf("expected non-zero exit") + } + if !strings.Contains(stderr, "unknown argument") { + t.Errorf("stderr should mention 'unknown argument', got %q", stderr) + } + if !strings.Contains(stderr, "+bogus") { + t.Errorf("stderr should echo the bad modifier: %q", stderr) + } +} + +// Leaf-index mode is distinguished from tile-dump mode by whether the +// second positional parses as an integer. The +modifier-rejection path +// runs through different code for each mode; verify both reach the +// same validation. +func TestFetchRejectsUnknownModifierTileMode(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "fetch", "http://invalid.invalid/tile/data/x000/000", "+bogus") + if exit == 0 { + t.Errorf("expected non-zero exit") + } + if !strings.Contains(stderr, "unknown argument") { + t.Errorf("stderr should mention 'unknown argument', got %q", stderr) + } +} diff --git a/cmd/ctool/cmd_tail.go b/cmd/ctool/cmd_tail.go index 2bdf8d2..90cde33 100644 --- a/cmd/ctool/cmd_tail.go +++ b/cmd/ctool/cmd_tail.go @@ -25,6 +25,10 @@ var ( ) func runTail(args []string) { + if len(args) > 0 && args[0] == "version" { + printVersion() + return + } fs := flag.NewFlagSet("tail", flag.ContinueOnError) interval := fs.Duration("interval", 15*time.Second, "polling interval (minimum 1s)") fromLeaf := fs.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)") @@ -154,7 +158,7 @@ func fetchURL(url string) ([]byte, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } diff --git a/cmd/ctool/cmd_tail_test.go b/cmd/ctool/cmd_tail_test.go new file mode 100644 index 0000000..35c49da --- /dev/null +++ b/cmd/ctool/cmd_tail_test.go @@ -0,0 +1,164 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "strings" + "testing" + "time" + + "filippo.io/sunlight" +) + +func TestTailNoArgsShowsUsage(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "tail") + if exit == 0 { + t.Errorf("expected non-zero exit, got 0") + } + if !strings.Contains(stderr, "Usage:") { + t.Errorf("stderr missing usage header: %q", stderr) + } +} + +func TestTailRejectsShortInterval(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "tail", "--interval", "500ms", "http://invalid.invalid") + if exit == 0 { + t.Errorf("expected non-zero exit") + } + if !strings.Contains(stderr, "interval must be at least 1s") { + t.Errorf("stderr=%q", stderr) + } +} + +func TestTailRejectsShortRateLimit(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "tail", "--rate-limit", "50ms", "http://invalid.invalid") + if exit == 0 { + t.Errorf("expected non-zero exit") + } + if !strings.Contains(stderr, "rate-limit must be at least 100ms") { + t.Errorf("stderr=%q", stderr) + } +} + +func TestTrunc(t *testing.T) { + cases := []struct { + in string + n int + want string + }{ + {"hello", 10, "hello"}, + {"hello", 5, "hello"}, + {"abcdefghij", 6, "abc..."}, + {"abcdefghij", 5, "ab..."}, + } + for _, c := range cases { + if got := trunc(c.in, c.n); got != c.want { + t.Errorf("trunc(%q,%d)=%q, want %q", c.in, c.n, got, c.want) + } + } +} + +func TestIssuerLabel(t *testing.T) { + terse := &x509.Certificate{Issuer: pkix.Name{ + CommonName: "R13", + Organization: []string{"Let's Encrypt"}, + }} + if got := issuerLabel(terse); got != "Let's Encrypt R13" { + t.Errorf("terse CN: got %q", got) + } + + self := &x509.Certificate{Issuer: pkix.Name{ + CommonName: "Let's Encrypt Authority X3", + Organization: []string{"Let's Encrypt"}, + }} + if got := issuerLabel(self); got != "Let's Encrypt Authority X3" { + t.Errorf("CN already mentions org: got %q", got) + } + + noOrg := &x509.Certificate{Issuer: pkix.Name{CommonName: "ipng root"}} + if got := issuerLabel(noOrg); got != "ipng root" { + t.Errorf("no org: got %q", got) + } +} + +// makeDERCert mints a short-lived, self-signed ECDSA cert for tests. Using +// P-256 keeps generation to single-digit milliseconds so the test stays fast. +func makeDERCert(t *testing.T, cn, org, dnsName string, notBefore, notAfter time.Time) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("genkey: %v", err) + } + // In a self-signed cert (parent == template), the issuer of the + // resulting cert is taken from the template's Subject, not its Issuer. + // So we put the issuer identity in Subject and let x509 mirror it. + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: cn, + Organization: []string{org}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + DNSNames: []string{dnsName}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("CreateCertificate: %v", err) + } + return der +} + +func TestFormatEntry(t *testing.T) { + notBefore := time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC) + notAfter := time.Date(2026, 6, 29, 0, 0, 0, 0, time.UTC) + der := makeDERCert(t, "R13", "Let's Encrypt", "example.com", notBefore, notAfter) + + e := &sunlight.LogEntry{ + LeafIndex: 1234567, + IsPrecert: false, + Certificate: der, + } + line := formatEntry(e) + for _, want := range []string{"1234567", "cert", "2026-03-31..2026-06-29", "Let's Encrypt R13", "example.com"} { + if !strings.Contains(line, want) { + t.Errorf("formatEntry output missing %q: %q", want, line) + } + } +} + +func TestFormatEntryPrecert(t *testing.T) { + notBefore := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + notAfter := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + der := makeDERCert(t, "IPng CA", "IPng Networks", "pre.example.com", notBefore, notAfter) + + e := &sunlight.LogEntry{ + LeafIndex: 42, + IsPrecert: true, + PreCertificate: der, + } + line := formatEntry(e) + if !strings.Contains(line, "pre ") { + t.Errorf("expected 'pre ' marker for precert: %q", line) + } + if !strings.Contains(line, "pre.example.com") { + t.Errorf("expected precert DNS name: %q", line) + } +} + +// An entry with no certificate bytes must still render safely, with the +// documented "(unknown)" placeholder rather than panicking or skipping. +func TestFormatEntryUnknownCert(t *testing.T) { + e := &sunlight.LogEntry{LeafIndex: 7} + line := formatEntry(e) + if !strings.Contains(line, "(unknown)") { + t.Errorf("expected '(unknown)' placeholder, got %q", line) + } + if !strings.Contains(line, "cert") { + t.Errorf("expected type 'cert': %q", line) + } +} diff --git a/cmd/ctool/main.go b/cmd/ctool/main.go index dc5e806..d739941 100644 --- a/cmd/ctool/main.go +++ b/cmd/ctool/main.go @@ -12,7 +12,16 @@ import ( "strings" ) -const version = "0.1.0" +// version, commit, and date are stamped at link time via -ldflags "-X ...": +// +// -X main.version=X.Y.Z -X main.commit= -X main.date= +// +// Defaults keep `go run` / `go build` without ldflags usable. +var ( + version = "0.1.0" + commit = "unknown" + date = "unknown" +) func main() { // Busybox-style: if invoked as "ctfetch" or "ctail", shortcut to that command. @@ -35,6 +44,8 @@ func main() { runFetch(os.Args[2:]) case "tail": runTail(os.Args[2:]) + case "version", "-version", "--version": + printVersion() default: fmt.Fprintf(os.Stderr, "Error: unknown command %q\n\n", os.Args[1]) usage() @@ -42,11 +53,20 @@ func main() { } } +// printVersion writes the build version, commit hash, and build date to +// stdout, using the binary's invocation name so the output reads naturally +// whether called as `ctool version`, `ctfetch version`, or `ctail version`. +func printVersion() { + base := strings.TrimSuffix(filepath.Base(os.Args[0]), filepath.Ext(os.Args[0])) + fmt.Printf("%s version %s (commit %s, built %s)\n", base, version, commit, date) +} + func usage() { fmt.Fprintf(os.Stderr, "Usage: ctool [flags] ...\n\n") fmt.Fprintf(os.Stderr, "Commands:\n") - fmt.Fprintf(os.Stderr, " fetch fetch and decode CT log entries as JSON\n") - fmt.Fprintf(os.Stderr, " tail tail a CT log, printing one-liners for new entries\n") + fmt.Fprintf(os.Stderr, " fetch fetch and decode CT log entries as JSON\n") + fmt.Fprintf(os.Stderr, " tail tail a CT log, printing one-liners for new entries\n") + fmt.Fprintf(os.Stderr, " version print version, commit hash, and build date\n") fmt.Fprintf(os.Stderr, "\nRun 'ctool --help' for command-specific flags.\n") } diff --git a/cmd/ctool/main_test.go b/cmd/ctool/main_test.go new file mode 100644 index 0000000..246e486 --- /dev/null +++ b/cmd/ctool/main_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "bytes" + "io" + "os" + "os/exec" + "strings" + "testing" +) + +// TestMain lets a subprocess re-enter main() so we can drive the real CLI +// dispatch from inside the test binary. When CTOOL_TEST_MAIN=1 is set, +// the test binary replaces os.Args from CTOOL_TEST_ARGV0/CTOOL_TEST_ARGS +// and calls main() directly; m.Run() is skipped in that child. +func TestMain(m *testing.M) { + if os.Getenv("CTOOL_TEST_MAIN") == "1" { + argv := []string{os.Getenv("CTOOL_TEST_ARGV0")} + if rest := os.Getenv("CTOOL_TEST_ARGS"); rest != "" { + argv = append(argv, strings.Split(rest, "\t")...) + } + os.Args = argv + main() + os.Exit(0) + } + os.Exit(m.Run()) +} + +// runCLI re-execs the test binary as the ctool CLI with the given argv[0] +// and argv[1:], capturing stdout, stderr, and the exit code. +func runCLI(t *testing.T, argv0 string, args ...string) (stdout, stderr string, exit int) { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), + "CTOOL_TEST_MAIN=1", + "CTOOL_TEST_ARGV0="+argv0, + "CTOOL_TEST_ARGS="+strings.Join(args, "\t"), + ) + var o, e bytes.Buffer + cmd.Stdout = &o + cmd.Stderr = &e + err := cmd.Run() + switch v := err.(type) { + case nil: + exit = 0 + case *exec.ExitError: + exit = v.ExitCode() + default: + t.Fatalf("runCLI: exec failed: %v", err) + } + return o.String(), e.String(), exit +} + +func TestCmdName(t *testing.T) { + save := os.Args + t.Cleanup(func() { os.Args = save }) + + cases := []struct { + argv0 string + sub string + want string + }{ + {"/usr/bin/ctool", "fetch", "ctool fetch"}, + {"/usr/bin/ctool", "tail", "ctool tail"}, + {"ctool", "fetch", "ctool fetch"}, + {"ctool.exe", "fetch", "ctool fetch"}, + {"/usr/bin/ctfetch", "fetch", "ctfetch"}, + {"ctail", "tail", "ctail"}, + } + for _, c := range cases { + os.Args = []string{c.argv0} + if got := cmdName(c.sub); got != c.want { + t.Errorf("argv0=%q sub=%q: cmdName()=%q, want %q", c.argv0, c.sub, got, c.want) + } + } +} + +// capturePrintVersion captures os.Stdout while calling printVersion(). We +// assert on the format rather than the exact version string so the test +// stays stable whether or not -ldflags stamped a commit/date. +func capturePrintVersion(t *testing.T, argv0 string) string { + t.Helper() + saveArgs, saveStdout := os.Args, os.Stdout + t.Cleanup(func() { os.Args, os.Stdout = saveArgs, saveStdout }) + os.Args = []string{argv0} + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + printVersion() + _ = w.Close() + out, _ := io.ReadAll(r) + return string(out) +} + +func TestPrintVersion(t *testing.T) { + cases := []struct { + argv0 string + prefix string + }{ + {"ctool", "ctool version "}, + {"ctfetch", "ctfetch version "}, + {"/usr/bin/ctail", "ctail version "}, + } + for _, c := range cases { + out := capturePrintVersion(t, c.argv0) + if !strings.HasPrefix(out, c.prefix) { + t.Errorf("argv0=%q: printVersion()=%q, want prefix %q", c.argv0, out, c.prefix) + } + if !strings.Contains(out, "commit ") || !strings.Contains(out, "built ") { + t.Errorf("argv0=%q: printVersion()=%q, missing commit/built fields", c.argv0, out) + } + } +} + +func TestDispatchVersion(t *testing.T) { + cases := []struct { + argv0 string + args []string + prefix string + }{ + {"ctool", []string{"version"}, "ctool version "}, + {"ctool", []string{"-version"}, "ctool version "}, + {"ctool", []string{"--version"}, "ctool version "}, + {"ctool", []string{"fetch", "version"}, "ctool version "}, + {"ctool", []string{"tail", "version"}, "ctool version "}, + {"ctfetch", []string{"version"}, "ctfetch version "}, + {"ctail", []string{"version"}, "ctail version "}, + } + for _, c := range cases { + stdout, _, exit := runCLI(t, c.argv0, c.args...) + if exit != 0 { + t.Errorf("argv=%v %v: exit %d, want 0", c.argv0, c.args, exit) + } + if !strings.HasPrefix(stdout, c.prefix) { + t.Errorf("argv=%v %v: stdout=%q, want prefix %q", c.argv0, c.args, stdout, c.prefix) + } + } +} + +func TestDispatchNoArgsShowsUsage(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool") + if exit == 0 { + t.Errorf("expected non-zero exit, got 0 (stderr=%q)", stderr) + } + if !strings.Contains(stderr, "Usage: ctool") { + t.Errorf("stderr missing usage header: %q", stderr) + } + if !strings.Contains(stderr, "fetch") || !strings.Contains(stderr, "tail") || !strings.Contains(stderr, "version") { + t.Errorf("stderr missing subcommand list: %q", stderr) + } +} + +func TestDispatchUnknownSubcommand(t *testing.T) { + _, stderr, exit := runCLI(t, "ctool", "bogus") + if exit == 0 { + t.Errorf("expected non-zero exit, got 0") + } + if !strings.Contains(stderr, "unknown command") { + t.Errorf("stderr=%q", stderr) + } +} diff --git a/debian/build-deb.sh b/debian/build-deb.sh new file mode 100755 index 0000000..62a3727 --- /dev/null +++ b/debian/build-deb.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Build the ctool Debian package for one architecture. +# Usage: build-deb.sh ctool +# +# The commit hash and build date are baked into the binary at link time +# via -ldflags in the Makefile, so `ctool version` is the source of truth +# for "which build". The .deb itself carries only the release version. +set -euo pipefail + +PACKAGE="${1:?usage: build-deb.sh ctool }" +ARCH="${2:?usage: build-deb.sh ctool }" +VERSION="${3:?usage: build-deb.sh ctool }" + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PKG="${PACKAGE}_${VERSION}_${ARCH}" +STAGING="$(mktemp -d)" +trap 'rm -rf "$STAGING"' EXIT + +echo "Building ${PKG}.deb" + +if [ "$PACKAGE" != "ctool" ]; then + echo "error: unknown package '${PACKAGE}' (expected ctool)" >&2 + exit 2 +fi + +install -d "$STAGING/DEBIAN" +install -d "$STAGING/usr/bin" +install -d "$STAGING/usr/share/man/man1" + +# Binary. ctool is an interactive tool, not a daemon, so it lives in +# /usr/bin and is on every login shell's PATH by default. +install -m 755 "$REPO_ROOT/build/${ARCH}/ctool" "$STAGING/usr/bin/ctool" + +# ctfetch and ctail are busybox-style aliases: the binary dispatches on +# argv[0], so a symlink is all it takes for `ctfetch ...` to mean +# `ctool fetch ...` and `ctail ...` to mean `ctool tail ...`. +ln -s ctool "$STAGING/usr/bin/ctfetch" +ln -s ctool "$STAGING/usr/bin/ctail" + +# Manpages — one overview (ctool) plus one detailed page per subcommand. +gzip -9 -c "$REPO_ROOT/docs/ctool.1" > "$STAGING/usr/share/man/man1/ctool.1.gz" +gzip -9 -c "$REPO_ROOT/docs/ctfetch.1" > "$STAGING/usr/share/man/man1/ctfetch.1.gz" +gzip -9 -c "$REPO_ROOT/docs/ctail.1" > "$STAGING/usr/share/man/man1/ctail.1.gz" + +# DEBIAN metadata. No conffiles and no maintainer scripts — the package +# ships a single binary, two symlinks, and three manpages. +sed "s/@VERSION@/${VERSION}/;s/@ARCH@/${ARCH}/" \ + "$REPO_ROOT/debian/${PACKAGE}.control.in" > "$STAGING/DEBIAN/control" + +# Emit package into build/ +mkdir -p "$REPO_ROOT/build" +OUT="$REPO_ROOT/build/${PKG}.deb" +dpkg-deb --build --root-owner-group "$STAGING" "$OUT" +echo "Built: $OUT" diff --git a/debian/ctool.control.in b/debian/ctool.control.in new file mode 100644 index 0000000..6c1e3dc --- /dev/null +++ b/debian/ctool.control.in @@ -0,0 +1,24 @@ +Package: ctool +Version: @VERSION@ +Architecture: @ARCH@ +Maintainer: Pim van Pelt +Section: net +Priority: optional +Description: Tools for working with Static CT log tiles + ctool is a busybox-style binary that fetches and decodes entries + from Static CT API logs (c2sp.org/static-ct-api). + . + ctool fetch reads one or more log entries from a data tile — + either by leaf index or by dumping a whole tile URL / local file — + and prints them as structured JSON. Optional modifiers decode + embedded SCTs, fetch the issuer certificate from the log, and + enrich each SCT with operator and state information from the + Chrome CT log list. + . + ctool tail follows a log's /checkpoint endpoint and prints a + one-line summary per certificate or precertificate as new data + tiles complete. Useful for live monitoring of a log's growth. + . + The package installs ctool under /usr/bin along with ctfetch + and ctail symlinks, which invoke the matching subcommand + directly for scripting convenience. diff --git a/docs/ctail.1 b/docs/ctail.1 new file mode 100644 index 0000000..b13895a --- /dev/null +++ b/docs/ctail.1 @@ -0,0 +1,124 @@ +.TH CTAIL 1 "April 2026" "ctool" "User Commands" +.SH NAME +ctail \- tail a Static CT log, printing one line per new entry +.SH SYNOPSIS +.B ctail +[\fIflags\fR] \fIlog\-url\fR +.PP +.B ctail version +.SH DESCRIPTION +.B ctail +polls a Static CT API log (c2sp.org/static\-ct\-api) and prints a +one\-liner to stdout for every certificate or precertificate that +appears in a newly\-completed data tile. +It is also available as the +.B tail +subcommand of +.BR ctool (1); +the two invocations are equivalent. +.PP +Status messages (startup banner, HTTP errors, progress complaints) go +to stderr; only the per\-entry one\-liners go to stdout, so the +output is safe to pipe into +.BR grep , +.BR awk , +or a log collector without mixing the two streams. +.SH OUTPUT FORMAT +One line per log entry, fixed\-width columns separated by spaces: +.PP +.RS +.EX + +.EE +.RE +.TP +.B leaf\-index +Decimal index of the entry within the log's Merkle tree. +.TP +.B type +.B cert +for a final certificate, +.B pre +for a precertificate. +.TP +.B validity\-range +.IR YYYY\-MM\-DD .. YYYY\-MM\-DD , +taken from the certificate's +.I NotBefore +and +.I NotAfter +fields. +.TP +.B issuer +Issuer common name, prefixed with the issuer organisation when the +CN alone is terse (so +.I R13 +is shown as +.IR "Let's Encrypt R13" ); +truncated to 40 characters with a trailing ellipsis if necessary. +.TP +.B subject +First DNS Subject Alternative Name, falling back to the certificate's +subject common name. Shown as +.I (unknown) +when neither is populated. +.SH FLAGS +.TP +.BI \-\-interval " duration" +How often to poll the log's +.B /checkpoint +endpoint. Minimum +.IR 1s ; +default +.IR 15s . +The interval timer starts when the checkpoint is fetched, so time +spent fetching the ensuing data tiles counts against the interval +and the next poll stays on schedule. +.TP +.BI \-\-from\-leaf " n" +Start at leaf index +.IR n . +The default, +.IR \-1 , +means "start at the current tree tip" \(em no backfill. +.TP +.BI \-\-rate\-limit " duration" +Minimum time between outgoing HTTP requests to the log. Minimum +.IR 100ms ; +default +.IR 2s . +Lets the client stay a polite distance below any per\-IP rate limit +the log operator enforces. +.TP +.BI \-\-user\-agent " string" +User\-Agent header sent with every request. Default: +\fIctail/VERSION (https://git.ipng.ch/certificate\-transparency/)\fR. +.SH SUBCOMMANDS +.TP +.B version +Print the binary's version, git commit hash, and build date, then +exit. +.SH EXAMPLES +Follow a log from the current tip: +.PP +.RS +.EX +$ ctail https://halloumi2026h2.mon.ct.ipng.ch +.EE +.RE +.PP +Replay a log from the very first entry, polling every ten seconds: +.PP +.RS +.EX +$ ctail \-\-from\-leaf 0 \-\-interval 10s https://halloumi2026h2.mon.ct.ipng.ch +.EE +.RE +.SH NOTES +A data tile is only fetched once the checkpoint confirms it is +complete (256 entries). This avoids unnecessary 404s at the tree tip. +.SH SEE ALSO +.BR ctool (1), +.BR ctfetch (1) +.SH AUTHOR +Pim van Pelt diff --git a/docs/ctfetch.1 b/docs/ctfetch.1 new file mode 100644 index 0000000..aaaaad6 --- /dev/null +++ b/docs/ctfetch.1 @@ -0,0 +1,130 @@ +.TH CTFETCH 1 "April 2026" "ctool" "User Commands" +.SH NAME +ctfetch \- fetch and decode Static CT log entries as JSON +.SH SYNOPSIS +.B ctfetch +[\fIflags\fR] \fIlog\-url\fR \fIleaf\-index\fR [\fB+sct\fR] [\fB+issuer\fR] [\fB+ctlog\fR] [\fB+all\fR] +.PP +.B ctfetch +[\fIflags\fR] \fItile\-url\-or\-file\fR [\fB+sct\fR] [\fB+issuer\fR] [\fB+ctlog\fR] [\fB+all\fR] +.PP +.B ctfetch version +.SH DESCRIPTION +.B ctfetch +reads entries from a Static CT API log (c2sp.org/static\-ct\-api) and +writes them to stdout as pretty\-printed JSON. +It is also available as the +.B fetch +subcommand of +.BR ctool (1); +the two invocations are equivalent. +.PP +Two modes are distinguished by whether the second positional argument +parses as an integer. +.SS Leaf\-index mode +.PP +.RS +.EX +ctfetch [modifiers...] +.EE +.RE +.PP +Fetches the data tile that contains +.IR leaf\-index , +decompresses it, and decodes the single entry at that position. +.SS Tile\-dump mode +.PP +.RS +.EX +ctfetch [modifiers...] +.EE +.RE +.PP +Fetches (or reads from disk) one tile and decodes every entry in it. +Hash tiles (\fB/tile/N/\fR..., N \(>= 0) produce the list of 32\-byte +SHA\-256 node hashes; output modifiers are an error in this case. +Data tiles (\fB/tile/data/\fR...) produce the full decoded +entry list. +.SH OUTPUT MODIFIERS +The modifiers are positional tokens beginning with +.BR + . +They control which optional fields are computed and included in the +JSON output. +.TP +.B +sct +Parse the embedded Signed Certificate Timestamp list from final +(non\-precert) certificates and include it alongside the entry. +.TP +.B +issuer +Fetch the issuer certificate from the log's +.B /issuer/ +endpoint and include parsed issuer details. +.TP +.B +ctlog +Look up each SCT's log ID in the CT log list (see +.BR \-\-logs\-list\-url ) +and enrich it with operator and state information. +.TP +.B +all +Shorthand for +.BR +sct " " +issuer " " +ctlog . +.SH FLAGS +.TP +.BI \-\-logs\-list\-url " url" +CT log list JSON used for +.B +ctlog +enrichment. +Default: +.IR https://www.gstatic.com/ct/log_list/v3/all_logs_list.json . +.TP +.BI \-\-monitoring\-url " url" +Log root URL used for +.B +issuer +lookups when the input is a local tile file. Ignored when the input +is already an HTTP(S) URL; in that case the root is derived by +stripping +.I /tile/... +from the path. +.SH SUBCOMMANDS +.TP +.B version +Print the binary's version, git commit hash, and build date, then +exit. +.SH EXAMPLES +Fetch one entry with all enrichments: +.PP +.RS +.EX +$ ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all +.EE +.RE +.PP +Dump a data tile straight off the web: +.PP +.RS +.EX +$ ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog +.EE +.RE +.PP +Dump a tile from disk, pointing at a monitoring URL so +.B +issuer +lookups can find the issuer endpoint: +.PP +.RS +.EX +$ ctfetch \-\-monitoring\-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer +.EE +.RE +.SH NOTES +Partial tiles (the +.I .p/N +suffix) are tried first; on 404 the full tile is fetched +automatically. +The CT log list and any fetched issuer certificates are cached in +memory for the lifetime of a single invocation. +.SH SEE ALSO +.BR ctool (1), +.BR ctail (1) +.SH AUTHOR +Pim van Pelt diff --git a/docs/ctool.1 b/docs/ctool.1 new file mode 100644 index 0000000..1137c46 --- /dev/null +++ b/docs/ctool.1 @@ -0,0 +1,83 @@ +.TH CTOOL 1 "April 2026" "ctool" "User Commands" +.SH NAME +ctool \- tools for working with Static CT log tiles +.SH SYNOPSIS +.B ctool +.I command +[\fIflags\fR] [\fIargs\fR...] +.PP +.B ctool +.B version +.SH DESCRIPTION +.B ctool +is a busybox\-style front\-end for a set of tools that read tiles from +a Static CT API log (c2sp.org/static\-ct\-api). +It dispatches on the first argument to one of the subcommands below. +The same binary can also be invoked through the +.B ctfetch +or +.B ctail +symlinks, in which case the subcommand name is implied by the link +name; flags and arguments are unchanged. +.SH COMMANDS +.TP +.B fetch +Fetch and decode one or more entries from a Static CT log and print +them as structured JSON. Supports both leaf\-index lookups and +full\-tile dumps, with optional SCT decoding, issuer\-certificate +lookup, and CT log\-list enrichment. See +.BR ctfetch (1) +for the full reference. +.TP +.B tail +Follow a Static CT log. Polls the log's checkpoint, walks new data +tiles as they complete, and prints a one\-line summary per certificate +or precertificate \(em leaf index, type, validity range, issuer, and +subject name. See +.BR ctail (1) +for the full reference. +.TP +.B version +Print the binary's version, git commit hash, and build date, then +exit. Equivalent forms +.B ctool fetch version +and +.B ctool tail version +also work and print the same information. +.SH EXAMPLES +Fetch a single leaf entry with all optional enrichments: +.PP +.RS +.EX +$ ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all +.EE +.RE +.PP +Tail a log from the current tree tip: +.PP +.RS +.EX +$ ctool tail https://halloumi2026h2.mon.ct.ipng.ch +.EE +.RE +.PP +Print the build version: +.PP +.RS +.EX +$ ctool version +ctool version 0.1.0 (commit abc1234, built 2026\-04\-21T12:00:00Z) +.EE +.RE +.SH "FULL DOCUMENTATION" +This manpage is an overview only. For the complete reference of each +subcommand \(em every flag, every output modifier, and the format of +the JSON and tail\-line output \(em see the per\-command manpages: +.BR ctfetch (1) +and +.BR ctail (1). +.SH SEE ALSO +.BR ctfetch (1), +.BR ctail (1) +.SH AUTHOR +Pim van Pelt diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..ff30d27 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,621 @@ +# ctool Design Document + +## Metadata + +| | | +| --- | --- | +| **Status** | Retrofit — describes shipped behavior as of `v0.1.0` | +| **Author** | Pim van Pelt `` | +| **Last updated** | 2026-04-21 | +| **Audience** | Operators and contributors who will read the source tree next | + +The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and +**MAY** are used as described in +[RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119), and are +reserved in this document for requirements that are actually enforced +in code or by an external dependency. Plain-language descriptions of +what the system or an operator can do are written in lowercase — +"can", "will", "does" — and should not be read as normative. + +## Summary + +`ctool` is a small collection of command-line tools for working with +[Static CT API](https://c2sp.org/static-ct-api) logs. It ships as a +single Go binary that dispatches to one of several subcommands. Today +there are two: `fetch`, which decodes one or more entries from a log +tile as structured JSON, and `tail`, which follows a log's checkpoint +and prints a one-liner per new certificate. The binary is also +exported as two busybox-style symlinks, `ctfetch` and `ctail`, so that +scripts can call the subcommand directly without the outer `ctool` +prefix. + +## Background + +The Static CT API defines a stateless on-disk layout for Certificate +Transparency logs: each leaf lives in a 256-entry gzip-compressed +*data tile* fetched by path, and a Merkle tree over those leaves is +published as a parallel set of *hash tiles*. Because the layout is +static, a log is just a tree of files on a web server — anyone with +an HTTP client and a decoder can read one. The CT ecosystem is +shifting from the old RFC 6962 "logs are a database" model to this +static-tile model; operators running or watching such logs need +ad-hoc tools to poke at tiles by hand and to watch a log grow in real +time. + +`ctool` is those tools, gathered under one roof so that packaging, +versioning, and future shared utilities live in a single place. The +project deliberately stays small: each subcommand is a thin wrapper +over [`filippo.io/sunlight`](https://pkg.go.dev/filippo.io/sunlight) +and Go's standard `crypto/x509`, with a small shared helper package +for the fiddly parts (issuer fetching, SCT decoding, CT log list +enrichment). + +## Goals and Non-Goals + +### Product Goals + +1. **Operator utility first.** The tools exist to make Static CT + logs inspectable by a human with a shell. Scriptable output + (JSON for `fetch`, fixed-column lines for `tail`) is a + first-class concern. +2. **No state.** Every invocation is independent; there are no + databases, no lock files, no background processes. +3. **Portable packaging.** A single statically-linked binary that + works on any Linux amd64/arm64 host without touching system + libraries. +4. **Extensible dispatcher.** `ctool` will grow new subcommands + over time. Adding one MUST NOT require a new binary, a new + Debian package, or any changes outside the `cmd/ctool` tree. + +### Non-Goals + +- `ctool` is not a CT log itself. It does not host, sign, or + distribute tiles. +- It is not a CT monitor in the Google-Chrome-policy sense. + Detecting misissuance is left to whatever the operator feeds + `ctool tail`'s output into. +- It is not a general x509 pretty-printer. The decoded fields + are the ones CT operators typically care about (SCTs, issuer + chain, validity range); a general "dump every extension" + mode is out of scope. +- It does not secure its own outbound traffic. Plain HTTP is + accepted when the operator passes an `http://` URL; it is the + operator's responsibility to prefer HTTPS for production logs. +- It is not a long-running service. `tail` runs in a loop, but it + is a foreground process — there is no daemon, no systemd unit, + no PID file. + +## Requirements + +Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) +so that later sections can cite it. + +### Functional Requirements + +**FR-1 Dispatcher (`ctool`)** + +- **FR-1.1** A single binary MUST dispatch its first positional + argument to a named subcommand. Unknown subcommands MUST exit + non-zero with a usage banner on stderr. +- **FR-1.2** The binary MUST recognize busybox-style invocation: + when `os.Args[0]`'s basename is `ctfetch` or `ctail` (with or + without a platform-native extension such as `.exe`), the + corresponding subcommand is invoked directly and the + remaining argv is passed through unchanged. +- **FR-1.3** The dispatcher MUST expose a `version` subcommand + that prints the build version, git commit hash, and build + date on stdout and exits zero. The same information MUST also + be reachable as `ctool -version`, `ctool --version`, + `ctool fetch version`, and `ctool tail version`, so that + "find out what you have" works regardless of which form the + operator guesses first. +- **FR-1.4** Usage banners emitted by a subcommand MUST refer to + the tool by its effective invocation name — `ctfetch ...` when + invoked through the symlink, `ctool fetch ...` when invoked + through the outer dispatcher — so that copy/paste from an + error message yields a command that works. +- **FR-1.5** Adding a new subcommand MUST require only: a new + `cmd_.go` file in `cmd/ctool/`, a new `case` in the + dispatcher's switch, and (optionally) a new busybox symlink + in the package layout. No changes to the Makefile's build + graph and no new binaries are required. + +**FR-2 `fetch` subcommand** + +- **FR-2.1** `ctool fetch` MUST operate in two modes, + distinguished by whether the second positional argument parses + as a signed decimal integer: + - **Leaf-index mode**: ` [modifiers]` + fetches the data tile containing `leaf-index` and decodes + the single entry at that position. + - **Tile-dump mode**: ` [modifiers]` + fetches or reads one tile and decodes every entry in it. +- **FR-2.2** Tile-dump mode MUST auto-detect hash tiles versus + data tiles from the tile contents. For hash tiles, the output + is the list of 32-byte SHA-256 node hashes and the `+sct`, + `+issuer`, `+ctlog`, and `+all` modifiers MUST be rejected as + an error. +- **FR-2.3** The output format MUST be pretty-printed JSON on + stdout with a trailing newline. +- **FR-2.4** The following positional modifiers MUST be + accepted, and any other `+`-prefixed token MUST be rejected as + an unknown argument: + - `+sct` — decode embedded SCTs on final certificates. + - `+issuer` — fetch and parse the issuer certificate from + the log's `/issuer/` endpoint. + - `+ctlog` — enrich each SCT with operator and state data + from the CT log list. + - `+all` — shorthand for all three of the above. +- **FR-2.5** When `+issuer` is used with tile-dump mode on a + local file, the operator MUST provide `--monitoring-url`; the + subcommand MUST otherwise derive the log root from the tile + URL by stripping the `/tile/...` path component. +- **FR-2.6** Partial tile suffixes (`.p/N`) MUST be tried first + when the path advertises one; on HTTP 404 the full tile MUST + be fetched by stripping the suffix. + +**FR-3 `tail` subcommand** + +- **FR-3.1** `ctool tail` MUST poll the log's `/checkpoint` + endpoint at a configurable interval (default 15s, minimum 1s), + and for each completed data tile that appears after the + current cursor, MUST decode every entry and print a one-line + summary to stdout. +- **FR-3.2** By default, the cursor starts at the current tree + tip so that only *new* entries are printed. `--from-leaf N` + MUST override this so that operators can replay from a known + index (including `--from-leaf 0` for a full backfill). +- **FR-3.3** The one-liner format MUST be fixed-column and + space-separated: leaf index (right-aligned, nine columns), + type (`cert` or `pre `), validity range + (`YYYY-MM-DD..YYYY-MM-DD`), issuer label (up to 40 chars, + truncated with `...`), subject name. +- **FR-3.4** The subject name MUST be the first DNS SAN; + falling back to the certificate's subject common name; falling + back to the literal `(unknown)` if neither is present. +- **FR-3.5** The issuer label MUST prepend the issuer + organisation to the issuer CN when the CN's first word is not + already contained in the organisation name (e.g. `R13` is + shown as `Let's Encrypt R13`, but `Let's Encrypt Authority X3` + is shown verbatim). +- **FR-3.6** Status and error messages MUST go to stderr; + one-liners MUST go to stdout, so that the output stream is + safe to pipe into `grep`, `awk`, or a log shipper without + interleaving diagnostic noise. +- **FR-3.7** A tile MUST NOT be fetched until the checkpoint + confirms it is complete (256 entries), to avoid unnecessary + 404s at the tree tip. +- **FR-3.8** A configurable minimum delay between outgoing HTTP + requests MUST be enforced (default 2s, minimum 100ms), to stay + a polite distance below any per-IP rate limit a log operator + might enforce. + +**FR-4 Packaging and versioning** + +- **FR-4.1** The build MUST produce statically linked binaries + for `linux/amd64` and `linux/arm64`. `CGO_ENABLED=0` MUST be + the default so that the resulting binary has no libc + dependency and runs on any Linux host of the matching + architecture. +- **FR-4.2** The version, commit hash, and build date MUST be + injected at link time via `-ldflags -X`, so that `ctool + version` on a shipped binary identifies the exact build + without requiring access to the source tree. The defaults + baked into the source MUST keep `go run` / `go build` without + ldflags usable. +- **FR-4.3** The Debian package MUST install `ctool` under + `/usr/bin`, MUST provide `/usr/bin/ctfetch` and + `/usr/bin/ctail` as symlinks to it, and MUST install the + three section-1 manpages (`ctool.1`, `ctfetch.1`, `ctail.1`) + gzipped under `/usr/share/man/man1`. + +### Non-Functional Requirements + +**NFR-1 Availability and reliability** + +- **NFR-1.1** A failing HTTP request MUST NOT crash `tail`. The + error MUST be logged to stderr and the next poll MUST happen + on schedule. +- **NFR-1.2** A malformed tile MUST NOT be silently skipped. If + a tile cannot be read, the subcommand MUST return an error + that names the offending leaf index. + +**NFR-2 Determinism and correctness** + +- **NFR-2.1** Given the same tile input, `fetch` MUST produce + byte-identical JSON output. No timestamps, random IDs, or + map iteration order MAY leak into the output. +- **NFR-2.2** `tail`'s polling interval timer MUST start when + the checkpoint is fetched, not when the previous loop + finished, so that time spent fetching data tiles counts + against the interval and the next poll stays on schedule. + +**NFR-3 Performance and scalability** + +- **NFR-3.1** `fetch` MUST fetch at most one tile per + invocation. Leaf-index mode derives the one tile that + contains the requested index; tile-dump mode operates on + exactly the tile the operator named. +- **NFR-3.2** Caches (CT log list, issuer certificates) MUST + be scoped to a single invocation. No on-disk cache, no + cross-invocation state. + +**NFR-4 Security** + +- **NFR-4.1** Tiles MUST be decompressed with a bounded + expansion ratio (currently 100×) to prevent a malicious log + from driving the client out of memory via a zip bomb. +- **NFR-4.2** The tools MUST NOT require any Linux + capabilities or root privileges. They are plain HTTP clients + and run unprivileged. +- **NFR-4.3** `ctool` MUST NOT write anywhere on the + filesystem except when the operator explicitly asks it to + (e.g. by redirecting stdout). There is no cache directory, + no log file, no state directory. + +**NFR-5 Operability** + +- **NFR-5.1** Every CLI flag SHOULD have a sensible default so + that the most common invocation is a single URL plus maybe a + leaf index. +- **NFR-5.2** The JSON output of `fetch` SHOULD be stable + across patch releases within a minor version. Field additions + are allowed; renames and removals are not. +- **NFR-5.3** Subcommand usage banners SHOULD include at least + one concrete example against `mon.ct.ipng.ch`, which is the + reference log we test against. + +## Architecture Overview + +### Process Model + +`ctool` is a short-lived foreground process. There are no daemons, +no sockets, no persistent state. Each invocation: + +1. Parses argv and dispatches to a subcommand. +2. Makes some number of HTTP requests (or reads a local file). +3. Writes JSON or one-liners to stdout. +4. Exits. + +`tail` is the one exception to step 3 being a single write: it +loops, making one checkpoint request per interval and zero or more +tile requests per loop iteration. It still exits on Ctrl-C or on an +unrecoverable error; there is no restart policy because there is no +supervisor. + +### Data Flow + +Configuration flows **in** as command-line flags and positional +arguments (there is no config file, no environment variables, no +`/etc/` anything). Data flows **in** from the Static CT log's HTTP +surface: the `/checkpoint` endpoint, the `/tile/...` tree, and the +`/issuer/` endpoint. The CT log list JSON (default: +`gstatic.com/ct/log_list/v3/all_logs_list.json`) is consumed +optionally when `+ctlog` is requested. Data flows **out** as JSON or +fixed-column lines on stdout, and as status/error messages on +stderr. + +## Components + +### `ctool` (dispatcher) + +`ctool` is the outer binary: a Go `main` package under `cmd/ctool/` +whose job is to decide which subcommand to run and hand argv off to +it. It is deliberately tiny — today about forty lines of code — +because every new subcommand adds a `case` to its `switch` and +nothing else. + +#### Responsibilities + +- Examine `os.Args[0]` and, if it matches the basename of a + known busybox symlink (`ctfetch`, `ctail`), route directly + to that subcommand (FR-1.2). +- Otherwise, examine `os.Args[1]` and route to the named + subcommand, print usage and exit non-zero on unknown names + (FR-1.1). +- Handle the `version` / `-version` / `--version` top-level + alias by printing the build version and exiting zero + (FR-1.3). +- Provide a single `cmdName()` helper that subcommands use + when building usage banners, so that every banner refers to + the effective invocation name (FR-1.4). + +#### Extension Model + +Adding a new subcommand `foo` is a three-line change: + +1. Add `cmd/ctool/cmd_foo.go` with an exported `runFoo(args + []string)` function. +2. Add `case "foo": runFoo(os.Args[2:])` to `main()`'s switch + and a matching line to `usage()`. +3. (Optional, if busybox-style invocation is wanted) Add + `case "ctfoo": runFoo(os.Args[1:])` to the symlink switch, + add the symlink to `debian/build-deb.sh`, and add a + `ctfoo(1)` manpage. + +No Makefile change is required (FR-1.5) because the build target +globs over `./cmd/ctool/`. No new Debian package is required; the +existing `ctool` package gets the new subcommand for free, and +optionally a new symlink. + +`version` is specifically *not* a subcommand file — it lives in the +dispatcher because every subcommand reuses the same `printVersion()` +helper for its own `version` alias (`ctool fetch version`, `ctool +tail version`). + +#### Interfaces + +**Presents.** + +- **A command-line interface** driven by `os.Args`. The + exit-code contract is 0 on success, 1 on unknown subcommand + or subcommand-internal fatal error, and whatever the + subcommand chooses otherwise. +- **`stdout` and `stderr` streams**. Structured output (JSON, + one-liners) goes to stdout; banners, progress messages, and + errors go to stderr (FR-3.6, NFR-5.3). + +**Consumes.** + +- **`os.Args`** — the sole input to the dispatcher. + +### `fetch` + +`fetch` (reachable as `ctool fetch` or `ctfetch`) decodes one or +more entries from a Static CT log tile. + +#### Responsibilities + +- Distinguish leaf-index mode from tile-dump mode by whether + `args[1]` parses as a decimal integer (FR-2.1). +- Fetch the relevant tile (or read it from disk), decompress it + with a bounded ratio (NFR-4.1), and decode each leaf. +- Optionally enrich each entry with decoded SCTs, the issuer + certificate, and CT-log-list metadata per the `+sct`, + `+issuer`, `+ctlog`, and `+all` modifiers (FR-2.4). +- In tile-dump mode, auto-detect hash tiles versus data tiles + and refuse the cert-oriented modifiers on hash tiles + (FR-2.2). +- Emit pretty-printed JSON on stdout (FR-2.3). + +#### Tile Fetching + +The tile URL is derived by [`filippo.io/sunlight`]' `TilePath` +function from the requested leaf index; `fetch` tries the partial +tile suffix first (`.p/N`) and falls back to the full tile on a 404 +(FR-2.6). Both paths return gzipped bytes, which are then +decompressed with `io.LimitReader` capping expansion at 100× the +input size (NFR-4.1). Tile-dump mode on a local file skips the URL +derivation and simply reads the file. + +#### Enrichment + +The optional modifiers trigger network calls outside the main tile +fetch: + +- `+sct` parses the SCT list extension from the DER cert + in-memory; no extra network calls. +- `+issuer` fetches `/issuer/` from the log root for each + referenced issuer fingerprint. Results are cached for the + lifetime of the invocation (NFR-3.2). +- `+ctlog` fetches the CT log list JSON once per invocation + and looks up each SCT's log ID against it. + +`+all` is syntactic sugar for all three. + +#### Interfaces + +**Presents.** + +- **Pretty-printed JSON** on stdout, with a trailing newline. + The top-level object is either a single `Entry` (leaf-index + mode), a `DumpResult` with an `entries` array (data tile), + or a `DumpResult` with a `hash_tile` field (hash tile). + +**Consumes.** + +- **Positional arguments** (log URL, leaf index, modifiers) and + two flags (`--logs-list-url`, `--monitoring-url`) from argv. +- **The Static CT log's HTTP surface** — checkpoint is not + read by `fetch`, but the tile tree and the issuer endpoint + are. +- **The outbound network** directly. No proxy handling, no + capability requirements (NFR-4.2). + +### `tail` + +`tail` (reachable as `ctool tail` or `ctail`) follows a Static CT +log's checkpoint and prints a one-line summary per new entry. + +#### Responsibilities + +- Poll the log's `/checkpoint` on a configurable interval and + track the current tree size (FR-3.1). +- For each completed data tile whose entries have not yet been + printed, fetch it, decompress it, decode each leaf, and print + a fixed-column one-liner to stdout (FR-3.1, FR-3.3). +- Throttle outbound HTTP requests with a configurable minimum + inter-request delay (FR-3.8). +- Respect a configurable starting leaf index so that operators + can replay from any point in the log (FR-3.2). + +#### Poll Loop + +Each iteration fetches the checkpoint first and records the +timestamp; then walks forward from the cursor tile by tile, stopping +at the first tile that the checkpoint does not yet confirm complete +(FR-3.7). The next poll sleeps until `checkpoint_time + interval`, +so that the time spent fetching data tiles counts against the +interval and consecutive polls stay on a stable cadence (NFR-2.2). + +A tile fetch failure is logged to stderr and the iteration is +retried on the next poll (NFR-1.1); the cursor does not advance +past a leaf that was never printed, so no entries are lost even +across transient log outages. + +#### One-Liner Format + +Columns are separated by a single space, each of fixed width. The +format string is documented in `docs/ctail.md`, reproduced here: + +``` +%9d %-4s %-21s %-40s %s +``` + +Fields: leaf index (decimal), type (`cert` or `pre `), validity +range (`YYYY-MM-DD..YYYY-MM-DD`), issuer label (truncated to 40 +with `...`), subject name. The truncation rule and the +issuer-with-org prepending rule are both tested (FR-3.3, FR-3.4, +FR-3.5). + +#### Interfaces + +**Presents.** + +- **Fixed-column one-liners** on stdout, one per log entry. +- **Status and error messages** on stderr (FR-3.6). + +**Consumes.** + +- **Positional ``** plus flags `--interval`, + `--from-leaf`, `--rate-limit`, `--user-agent`. +- **The log's HTTP surface** (`/checkpoint` and the tile tree). +- **The outbound network** directly. + +### Shared Helpers (`internal/utils`) + +`internal/utils` holds the code that both `fetch` and `tail` +call into but neither one should own: tile fetching with partial +fallback, bounded gzip decompression, SCT decoding, issuer fetching +with per-invocation caching, and CT-log-list enrichment. The +package is deliberately import-only from `cmd/ctool/`; nothing +outside the binary imports it. + +The helpers are not tested in isolation today beyond what the +binary-level tests exercise; if the package grows substantially, +per-package unit tests SHOULD be added before it becomes difficult +to reason about. + +## Operational Concerns + +### Packaging + +The repository ships a single Debian package, `ctool`, built for +`amd64` and `arm64`. The package contains `/usr/bin/ctool` plus two +symlinks (`ctfetch`, `ctail`) and three manpages under +`/usr/share/man/man1`. There are no conffiles, no maintainer scripts, +and no systemd units; the package is a pure binary-plus-docs bundle +(FR-4.3). + +Binaries are produced with `CGO_ENABLED=0` (FR-4.1). The effect is +that the binary has no libc dependency and runs on any Linux +amd64/arm64 host regardless of glibc version, at the cost of Go's +`net` package using the pure-Go resolver instead of `getaddrinfo` +(i.e. `/etc/nsswitch.conf` and NSS modules are ignored). `ctool` +only does DNS-over-UDP lookups for log hostnames, so this is fine. + +### Versioning + +Versioning follows [Semantic Versioning](https://semver.org/). The +authoritative version string lives in `Makefile`'s `VERSION` +variable; on each build, the Makefile stamps `main.version`, +`main.commit`, and `main.date` with `-ldflags -X` (FR-4.2). The +commit and date are derived from `git rev-parse --short HEAD` and +an ISO-8601 UTC timestamp at build time. `ctool version` prints +all three; the Debian package's `Version:` field carries only the +release version. + +### Failure Modes + +- **Log returns 404 on a tile we expected.** Treat as a + transient error; `tail` logs to stderr and retries on the + next poll (NFR-1.1). `fetch` exits non-zero with the HTTP + status in the error message. +- **Log returns gzip bomb.** The decompress helper's + `LimitReader` caps expansion at 100× the input size + (NFR-4.1); an attempt to exceed it surfaces as a decode + error, not an out-of-memory crash. +- **Checkpoint is malformed.** `tail` logs a "checkpoint + error" on stderr and sleeps for one interval before + retrying. +- **CT log list is unreachable.** `+ctlog` degrades: a warning + is printed to stderr and entries go out without the + `ctlog` enrichment rather than failing the whole + invocation. +- **Operator passes a hash tile URL with `+sct` / `+issuer` / + `+ctlog`.** The subcommand detects the tile type and exits + with an error explaining that cert-oriented modifiers do not + apply to hash tiles (FR-2.2). + +### Observability + +`ctool` has no metrics and no structured logging surface of its +own. Observability in practice is whatever the operator builds +around the output streams: + +- Pipe `ctool tail` into a log shipper or `awk` to watch for + specific issuers or subjects. +- Pipe `ctool fetch ... +all` into `jq` for ad-hoc inspection. +- Exit codes (0 / non-zero) are the only machine-readable + signal of success or failure. + +If a future operational need calls for structured logs or metrics +(e.g. a `ctool watch` subcommand that is meant to be run under +systemd), it can be added without disturbing `fetch` or `tail`. + +### Security + +The tools run unprivileged (NFR-4.2) and write nothing to disk +outside stdout (NFR-4.3). They do not handle credentials, do not +parse user-supplied certificates for signature verification, and +do not act as a TLS server. + +TLS verification on outbound requests uses Go's default roots; +there is no flag to disable it. Operators probing a log over plain +HTTP accept the risk of in-flight modification. + +## Alternatives Considered + +- **Separate `ctfetch` and `ctail` binaries, no dispatcher.** + Rejected in favor of one binary with symlinks. A single + binary is simpler to package, ship, and version, and + scales better as new subcommands land (FR-1.5). Operators + who prefer the separate-binary UX still get it through the + `ctfetch` / `ctail` symlinks. +- **A full CLI framework (cobra, urfave/cli).** Rejected + because the dispatcher is under fifty lines and the + subcommand parsers use the standard `flag` package. A + framework would add a dependency and a build surface + without improving the operator experience at this scale. + If the subcommand count grows past ~five, this decision + should be revisited. +- **Embedding a local cache for issuer certificates across + invocations.** Rejected for now (NFR-3.2). Caching introduces + a state directory, a cache-invalidation policy, and a new + failure mode (stale issuer); since `fetch` is typically run + ad-hoc rather than in a tight loop, the per-invocation cache + is sufficient. +- **Making `tail` a daemon with a systemd unit.** Rejected in + favor of a plain foreground loop. Operators who want a daemon + wrap it in `systemd-run`, `tmux`, or their orchestration tool + of choice; the tool itself stays stateless and exit-code + honest. + +## Open Questions + +- **Output schema stability.** `fetch`'s JSON output is currently + whatever `internal/utils` emits. NFR-5.2 commits to additive + compatibility across patch releases, but a minor-version bump + may still reshape fields. A more explicit schema document (or + a `--schema-version` flag) would help downstream consumers. +- **More subcommands.** The dispatcher is built for growth — + candidate ideas include `ctool verify` (checkpoint signature + + consistency proof) and `ctool stats` (issuer/validity + distribution over a time window). These are out of scope for + v0.1 but the FR-1.x requirements exist precisely so the + additions stay cheap. +- **Proxy and retry policy.** There is no HTTP retry today. A + log that 502s intermittently will surface those as visible + errors on stderr (NFR-1.1) but the cursor still stalls until + the log recovers. A configurable retry/backoff policy may be + worth adding once we see a deployment hit this in practice. diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 8668672..e0ad07d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -167,7 +167,7 @@ func FetchURL(url string) ([]byte, error) { if err != nil { return nil, err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) @@ -506,9 +506,7 @@ func parseCertDetails(certDER []byte) *CertDetails { IsCA: cert.IsCA, } - for _, e := range cert.EmailAddresses { - d.EmailSANs = append(d.EmailSANs, e) - } + d.EmailSANs = append(d.EmailSANs, cert.EmailAddresses...) for _, u := range cert.URIs { d.URISANs = append(d.URISANs, u.String()) }