Add Debian packaging, Makefile, manpages, tests, and design doc
Introduces a static-binary build and Debian package (amd64/arm64) with version/commit/date stamped via -ldflags. Ships section-1 manpages for ctool, ctfetch, and ctail. Adds a `version` subcommand reachable as `ctool version`, `ctool -version`, `ctool --version`, `ctool fetch version`, `ctool tail version`, and via the ctfetch/ctail symlinks. Adds tests covering the dispatcher, fetch/tail argument parsing, and the formatter/helper functions. Adds a retrofit design document modelled on the vpp-maglev one, with FRs and NFRs for each tool and the dispatcher. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
/ctool
|
||||
/ctfetch
|
||||
/ctail
|
||||
/build
|
||||
|
||||
160
Makefile
Normal file
160
Makefile
Normal file
@@ -0,0 +1,160 @@
|
||||
BINARIES := ctool
|
||||
MODULE := git.ipng.ch/certificate-transparency/ctfetch
|
||||
|
||||
NATIVE_ARCH := $(shell go env GOARCH)
|
||||
VERSION := 0.1.0
|
||||
COMMIT_HASH := $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
||||
DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
LDFLAGS := -X 'main.version=$(VERSION)' \
|
||||
-X 'main.commit=$(COMMIT_HASH)' \
|
||||
-X 'main.date=$(DATE)'
|
||||
|
||||
# CGO_ENABLED=0 produces fully static binaries: no libc dependency, so the
|
||||
# debian package runs on any Linux-amd64/arm64 host regardless of glibc
|
||||
# version (or musl). The only behavioural change is that Go's net package
|
||||
# uses the pure-Go resolver instead of libc's getaddrinfo, which skips
|
||||
# /etc/nsswitch.conf and NSS modules — fine for DNS-only lookups, which
|
||||
# is all ctool does when resolving log hostnames.
|
||||
export CGO_ENABLED := 0
|
||||
|
||||
# GO_VERSION is what install-deps-go downloads from go.dev when the
|
||||
# system Go is missing or older than this. Matches the go.mod floor.
|
||||
# Override on the command line to pull a specific patch release:
|
||||
# make install-deps GO_VERSION=1.25.0
|
||||
GO_VERSION ?= 1.24.6
|
||||
|
||||
# GOLANGCI_LINT_VERSION is the minimum golangci-lint version that
|
||||
# install-deps-go-tools accepts. install-deps-go-tools always `go
|
||||
# install`s @latest, then asserts the resulting binary reports a
|
||||
# version >= this floor as a sanity check.
|
||||
GOLANGCI_LINT_VERSION ?= 1.64.0
|
||||
|
||||
.PHONY: all build build-amd64 build-arm64 test lint fixstyle pkg-deb clean install-deps install-deps-apt install-deps-go install-deps-go-tools
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
mkdir -p build/$(NATIVE_ARCH)
|
||||
go build -ldflags "$(LDFLAGS)" -o build/$(NATIVE_ARCH)/ctool ./cmd/ctool/
|
||||
|
||||
build-amd64:
|
||||
mkdir -p build/amd64
|
||||
GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o build/amd64/ctool ./cmd/ctool/
|
||||
|
||||
build-arm64:
|
||||
mkdir -p build/arm64
|
||||
GOOS=linux GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o build/arm64/ctool ./cmd/ctool/
|
||||
|
||||
# pkg-deb produces one .deb per architecture. Each package contains a
|
||||
# single statically-linked ctool binary plus ctfetch and ctail symlinks,
|
||||
# and the three section-1 manpages.
|
||||
pkg-deb: build-amd64 build-arm64
|
||||
debian/build-deb.sh ctool amd64 $(VERSION)
|
||||
debian/build-deb.sh ctool arm64 $(VERSION)
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
|
||||
fixstyle:
|
||||
gofmt -w .
|
||||
|
||||
# install-deps is an opt-in "set up a fresh developer box" target. Tested
|
||||
# on Debian Trixie; the apt half should also work on Bookworm and recent
|
||||
# Ubuntu LTS. Splits into three sub-targets so they can be run individually:
|
||||
#
|
||||
# install-deps-apt — Debian-packaged build-time deps (git, make,
|
||||
# dpkg-dev, curl, tar).
|
||||
# install-deps-go — ensure a Go toolchain >= $(GO_VERSION) is on
|
||||
# the system. Downloads the upstream tarball
|
||||
# into /usr/local/go when the system Go is
|
||||
# missing or older than the go.mod floor.
|
||||
# install-deps-go-tools — `go install` the helpers this repo needs
|
||||
# (golangci-lint) and assert it is new enough
|
||||
# to understand current Go syntax.
|
||||
#
|
||||
# Each sub-target is idempotent and safe to re-run.
|
||||
install-deps: install-deps-apt install-deps-go install-deps-go-tools
|
||||
@echo ""
|
||||
@echo "==> All build dependencies installed."
|
||||
@echo " Make sure these are on PATH:"
|
||||
@echo " /usr/local/go/bin (Go toolchain)"
|
||||
@echo " \$$(go env GOPATH)/bin (golangci-lint, ...)"
|
||||
|
||||
install-deps-apt:
|
||||
@set -eu; \
|
||||
if [ "$$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo"; fi; \
|
||||
echo "==> Installing apt packages (git, make, dpkg-dev, curl, tar)"; \
|
||||
$$SUDO apt-get update; \
|
||||
$$SUDO apt-get install -y --no-install-recommends \
|
||||
git make dpkg-dev ca-certificates curl tar
|
||||
|
||||
# install-deps-go short-circuits when go env GOVERSION already reports a
|
||||
# version >= GO_VERSION. Otherwise it downloads the official upstream
|
||||
# tarball (https://go.dev/dl/) and extracts it to /usr/local/go, matching
|
||||
# the layout that go.dev recommends and that most Debian setups use for
|
||||
# "Go newer than apt provides".
|
||||
install-deps-go:
|
||||
@set -eu; \
|
||||
if [ "$$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo"; fi; \
|
||||
echo "==> Checking Go toolchain (required: $(GO_VERSION)+)"; \
|
||||
if command -v go >/dev/null 2>&1; then \
|
||||
CURRENT=$$(go env GOVERSION 2>/dev/null | sed 's/^go//'); \
|
||||
OLDEST=$$(printf '%s\n%s\n' "$(GO_VERSION)" "$$CURRENT" | sort -V | head -n1); \
|
||||
if [ "$$OLDEST" = "$(GO_VERSION)" ] && [ -n "$$CURRENT" ]; then \
|
||||
echo " go$$CURRENT already installed (>= $(GO_VERSION)), skipping."; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
echo " go$$CURRENT is older than $(GO_VERSION), upgrading."; \
|
||||
else \
|
||||
echo " no Go toolchain on PATH, installing."; \
|
||||
fi; \
|
||||
DEB_ARCH=$$(dpkg --print-architecture); \
|
||||
case "$$DEB_ARCH" in \
|
||||
amd64) GOARCH=amd64 ;; \
|
||||
arm64) GOARCH=arm64 ;; \
|
||||
armhf) GOARCH=armv6l ;; \
|
||||
*) echo " unsupported architecture: $$DEB_ARCH" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
TARBALL="go$(GO_VERSION).linux-$$GOARCH.tar.gz"; \
|
||||
URL="https://go.dev/dl/$$TARBALL"; \
|
||||
echo " downloading $$URL"; \
|
||||
curl -fsSL -o "/tmp/$$TARBALL" "$$URL"; \
|
||||
echo " installing to /usr/local/go"; \
|
||||
$$SUDO rm -rf /usr/local/go; \
|
||||
$$SUDO tar -C /usr/local -xzf "/tmp/$$TARBALL"; \
|
||||
rm -f "/tmp/$$TARBALL"; \
|
||||
echo " installed $$(/usr/local/go/bin/go version)"
|
||||
|
||||
install-deps-go-tools:
|
||||
@set -eu; \
|
||||
if ! command -v go >/dev/null 2>&1; then \
|
||||
export PATH="/usr/local/go/bin:$$PATH"; \
|
||||
fi; \
|
||||
echo "==> Installing Go tools via 'go install'"; \
|
||||
echo " github.com/golangci/golangci-lint/v2/cmd/golangci-lint"; \
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest; \
|
||||
GOBIN="$$(go env GOBIN)"; \
|
||||
if [ -z "$$GOBIN" ]; then GOBIN="$$(go env GOPATH)/bin"; fi; \
|
||||
echo "==> Asserting golangci-lint version >= $(GOLANGCI_LINT_VERSION)"; \
|
||||
if ! "$$GOBIN/golangci-lint" version >/dev/null 2>&1; then \
|
||||
echo " ERROR: $$GOBIN/golangci-lint is not executable" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
INSTALLED=$$("$$GOBIN/golangci-lint" version 2>&1 | sed -En 's/.*has version v?([0-9][0-9.]*).*/\1/p' | head -n1); \
|
||||
if [ -z "$$INSTALLED" ]; then \
|
||||
echo " ERROR: could not parse golangci-lint version output" >&2; \
|
||||
"$$GOBIN/golangci-lint" version >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
OLDEST=$$(printf '%s\n%s\n' "$(GOLANGCI_LINT_VERSION)" "$$INSTALLED" | sort -V | head -n1); \
|
||||
if [ "$$OLDEST" != "$(GOLANGCI_LINT_VERSION)" ]; then \
|
||||
echo " ERROR: golangci-lint $$INSTALLED is older than the required $(GOLANGCI_LINT_VERSION)" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
echo " golangci-lint $$INSTALLED (>= $(GOLANGCI_LINT_VERSION)) OK"
|
||||
|
||||
clean:
|
||||
rm -rf build/
|
||||
@@ -14,6 +14,10 @@ import (
|
||||
)
|
||||
|
||||
func runFetch(args []string) {
|
||||
if len(args) > 0 && args[0] == "version" {
|
||||
printVersion()
|
||||
return
|
||||
}
|
||||
fs := flag.NewFlagSet("fetch", flag.ContinueOnError)
|
||||
logsListURL := fs.String("logs-list-url", "https://www.gstatic.com/ct/log_list/v3/all_logs_list.json", "URL of the CT log list JSON")
|
||||
monitoringURL := fs.String("monitoring-url", "", "log root URL for issuer lookups when input is a file")
|
||||
|
||||
73
cmd/ctool/cmd_fetch_test.go
Normal file
73
cmd/ctool/cmd_fetch_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchNoArgsShowsUsage(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "fetch")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit, got 0 (stderr=%q)", stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "Usage:") {
|
||||
t.Errorf("stderr missing usage header: %q", stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "ctool fetch") {
|
||||
t.Errorf("stderr missing qualified command name: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// When invoked via the ctfetch symlink, the usage banner must use the
|
||||
// symlink name ("ctfetch"), not "ctool fetch". This exercises cmdName's
|
||||
// symlink branch through the real CLI.
|
||||
func TestFetchUsageUsesSymlinkName(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctfetch")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit, got 0")
|
||||
}
|
||||
if !strings.Contains(stderr, "ctfetch") {
|
||||
t.Errorf("stderr should mention ctfetch: %q", stderr)
|
||||
}
|
||||
if strings.Contains(stderr, "ctool fetch") {
|
||||
t.Errorf("stderr should NOT say 'ctool fetch' when invoked via symlink: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchHelpExitsZero(t *testing.T) {
|
||||
_, _, exit := runCLI(t, "ctool", "fetch", "-h")
|
||||
if exit != 0 {
|
||||
t.Errorf("expected exit 0 for -h, got %d", exit)
|
||||
}
|
||||
}
|
||||
|
||||
// An unknown +modifier must be rejected before any network request is made.
|
||||
// The fetch dispatcher validates the modifier enum (sct/issuer/ctlog/all)
|
||||
// up-front; we pass a junk URL to prove the error happens during parse,
|
||||
// not during HTTP.
|
||||
func TestFetchRejectsUnknownModifier(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "fetch", "http://invalid.invalid", "1", "+bogus")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit")
|
||||
}
|
||||
if !strings.Contains(stderr, "unknown argument") {
|
||||
t.Errorf("stderr should mention 'unknown argument', got %q", stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "+bogus") {
|
||||
t.Errorf("stderr should echo the bad modifier: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// Leaf-index mode is distinguished from tile-dump mode by whether the
|
||||
// second positional parses as an integer. The +modifier-rejection path
|
||||
// runs through different code for each mode; verify both reach the
|
||||
// same validation.
|
||||
func TestFetchRejectsUnknownModifierTileMode(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "fetch", "http://invalid.invalid/tile/data/x000/000", "+bogus")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit")
|
||||
}
|
||||
if !strings.Contains(stderr, "unknown argument") {
|
||||
t.Errorf("stderr should mention 'unknown argument', got %q", stderr)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,10 @@ var (
|
||||
)
|
||||
|
||||
func runTail(args []string) {
|
||||
if len(args) > 0 && args[0] == "version" {
|
||||
printVersion()
|
||||
return
|
||||
}
|
||||
fs := flag.NewFlagSet("tail", flag.ContinueOnError)
|
||||
interval := fs.Duration("interval", 15*time.Second, "polling interval (minimum 1s)")
|
||||
fromLeaf := fs.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)")
|
||||
@@ -154,7 +158,7 @@ func fetchURL(url string) ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
164
cmd/ctool/cmd_tail_test.go
Normal file
164
cmd/ctool/cmd_tail_test.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"math/big"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"filippo.io/sunlight"
|
||||
)
|
||||
|
||||
func TestTailNoArgsShowsUsage(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "tail")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit, got 0")
|
||||
}
|
||||
if !strings.Contains(stderr, "Usage:") {
|
||||
t.Errorf("stderr missing usage header: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTailRejectsShortInterval(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "tail", "--interval", "500ms", "http://invalid.invalid")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit")
|
||||
}
|
||||
if !strings.Contains(stderr, "interval must be at least 1s") {
|
||||
t.Errorf("stderr=%q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTailRejectsShortRateLimit(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "tail", "--rate-limit", "50ms", "http://invalid.invalid")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit")
|
||||
}
|
||||
if !strings.Contains(stderr, "rate-limit must be at least 100ms") {
|
||||
t.Errorf("stderr=%q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrunc(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
n int
|
||||
want string
|
||||
}{
|
||||
{"hello", 10, "hello"},
|
||||
{"hello", 5, "hello"},
|
||||
{"abcdefghij", 6, "abc..."},
|
||||
{"abcdefghij", 5, "ab..."},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := trunc(c.in, c.n); got != c.want {
|
||||
t.Errorf("trunc(%q,%d)=%q, want %q", c.in, c.n, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIssuerLabel(t *testing.T) {
|
||||
terse := &x509.Certificate{Issuer: pkix.Name{
|
||||
CommonName: "R13",
|
||||
Organization: []string{"Let's Encrypt"},
|
||||
}}
|
||||
if got := issuerLabel(terse); got != "Let's Encrypt R13" {
|
||||
t.Errorf("terse CN: got %q", got)
|
||||
}
|
||||
|
||||
self := &x509.Certificate{Issuer: pkix.Name{
|
||||
CommonName: "Let's Encrypt Authority X3",
|
||||
Organization: []string{"Let's Encrypt"},
|
||||
}}
|
||||
if got := issuerLabel(self); got != "Let's Encrypt Authority X3" {
|
||||
t.Errorf("CN already mentions org: got %q", got)
|
||||
}
|
||||
|
||||
noOrg := &x509.Certificate{Issuer: pkix.Name{CommonName: "ipng root"}}
|
||||
if got := issuerLabel(noOrg); got != "ipng root" {
|
||||
t.Errorf("no org: got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// makeDERCert mints a short-lived, self-signed ECDSA cert for tests. Using
|
||||
// P-256 keeps generation to single-digit milliseconds so the test stays fast.
|
||||
func makeDERCert(t *testing.T, cn, org, dnsName string, notBefore, notAfter time.Time) []byte {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("genkey: %v", err)
|
||||
}
|
||||
// In a self-signed cert (parent == template), the issuer of the
|
||||
// resulting cert is taken from the template's Subject, not its Issuer.
|
||||
// So we put the issuer identity in Subject and let x509 mirror it.
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{
|
||||
CommonName: cn,
|
||||
Organization: []string{org},
|
||||
},
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
DNSNames: []string{dnsName},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateCertificate: %v", err)
|
||||
}
|
||||
return der
|
||||
}
|
||||
|
||||
func TestFormatEntry(t *testing.T) {
|
||||
notBefore := time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC)
|
||||
notAfter := time.Date(2026, 6, 29, 0, 0, 0, 0, time.UTC)
|
||||
der := makeDERCert(t, "R13", "Let's Encrypt", "example.com", notBefore, notAfter)
|
||||
|
||||
e := &sunlight.LogEntry{
|
||||
LeafIndex: 1234567,
|
||||
IsPrecert: false,
|
||||
Certificate: der,
|
||||
}
|
||||
line := formatEntry(e)
|
||||
for _, want := range []string{"1234567", "cert", "2026-03-31..2026-06-29", "Let's Encrypt R13", "example.com"} {
|
||||
if !strings.Contains(line, want) {
|
||||
t.Errorf("formatEntry output missing %q: %q", want, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatEntryPrecert(t *testing.T) {
|
||||
notBefore := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
notAfter := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||
der := makeDERCert(t, "IPng CA", "IPng Networks", "pre.example.com", notBefore, notAfter)
|
||||
|
||||
e := &sunlight.LogEntry{
|
||||
LeafIndex: 42,
|
||||
IsPrecert: true,
|
||||
PreCertificate: der,
|
||||
}
|
||||
line := formatEntry(e)
|
||||
if !strings.Contains(line, "pre ") {
|
||||
t.Errorf("expected 'pre ' marker for precert: %q", line)
|
||||
}
|
||||
if !strings.Contains(line, "pre.example.com") {
|
||||
t.Errorf("expected precert DNS name: %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// An entry with no certificate bytes must still render safely, with the
|
||||
// documented "(unknown)" placeholder rather than panicking or skipping.
|
||||
func TestFormatEntryUnknownCert(t *testing.T) {
|
||||
e := &sunlight.LogEntry{LeafIndex: 7}
|
||||
line := formatEntry(e)
|
||||
if !strings.Contains(line, "(unknown)") {
|
||||
t.Errorf("expected '(unknown)' placeholder, got %q", line)
|
||||
}
|
||||
if !strings.Contains(line, "cert") {
|
||||
t.Errorf("expected type 'cert': %q", line)
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,16 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const version = "0.1.0"
|
||||
// version, commit, and date are stamped at link time via -ldflags "-X ...":
|
||||
//
|
||||
// -X main.version=X.Y.Z -X main.commit=<sha> -X main.date=<iso8601>
|
||||
//
|
||||
// Defaults keep `go run` / `go build` without ldflags usable.
|
||||
var (
|
||||
version = "0.1.0"
|
||||
commit = "unknown"
|
||||
date = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Busybox-style: if invoked as "ctfetch" or "ctail", shortcut to that command.
|
||||
@@ -35,6 +44,8 @@ func main() {
|
||||
runFetch(os.Args[2:])
|
||||
case "tail":
|
||||
runTail(os.Args[2:])
|
||||
case "version", "-version", "--version":
|
||||
printVersion()
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "Error: unknown command %q\n\n", os.Args[1])
|
||||
usage()
|
||||
@@ -42,11 +53,20 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// printVersion writes the build version, commit hash, and build date to
|
||||
// stdout, using the binary's invocation name so the output reads naturally
|
||||
// whether called as `ctool version`, `ctfetch version`, or `ctail version`.
|
||||
func printVersion() {
|
||||
base := strings.TrimSuffix(filepath.Base(os.Args[0]), filepath.Ext(os.Args[0]))
|
||||
fmt.Printf("%s version %s (commit %s, built %s)\n", base, version, commit, date)
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage: ctool <command> [flags] ...\n\n")
|
||||
fmt.Fprintf(os.Stderr, "Commands:\n")
|
||||
fmt.Fprintf(os.Stderr, " fetch fetch and decode CT log entries as JSON\n")
|
||||
fmt.Fprintf(os.Stderr, " tail tail a CT log, printing one-liners for new entries\n")
|
||||
fmt.Fprintf(os.Stderr, " fetch fetch and decode CT log entries as JSON\n")
|
||||
fmt.Fprintf(os.Stderr, " tail tail a CT log, printing one-liners for new entries\n")
|
||||
fmt.Fprintf(os.Stderr, " version print version, commit hash, and build date\n")
|
||||
fmt.Fprintf(os.Stderr, "\nRun 'ctool <command> --help' for command-specific flags.\n")
|
||||
}
|
||||
|
||||
|
||||
163
cmd/ctool/main_test.go
Normal file
163
cmd/ctool/main_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMain lets a subprocess re-enter main() so we can drive the real CLI
|
||||
// dispatch from inside the test binary. When CTOOL_TEST_MAIN=1 is set,
|
||||
// the test binary replaces os.Args from CTOOL_TEST_ARGV0/CTOOL_TEST_ARGS
|
||||
// and calls main() directly; m.Run() is skipped in that child.
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("CTOOL_TEST_MAIN") == "1" {
|
||||
argv := []string{os.Getenv("CTOOL_TEST_ARGV0")}
|
||||
if rest := os.Getenv("CTOOL_TEST_ARGS"); rest != "" {
|
||||
argv = append(argv, strings.Split(rest, "\t")...)
|
||||
}
|
||||
os.Args = argv
|
||||
main()
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// runCLI re-execs the test binary as the ctool CLI with the given argv[0]
|
||||
// and argv[1:], capturing stdout, stderr, and the exit code.
|
||||
func runCLI(t *testing.T, argv0 string, args ...string) (stdout, stderr string, exit int) {
|
||||
t.Helper()
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"CTOOL_TEST_MAIN=1",
|
||||
"CTOOL_TEST_ARGV0="+argv0,
|
||||
"CTOOL_TEST_ARGS="+strings.Join(args, "\t"),
|
||||
)
|
||||
var o, e bytes.Buffer
|
||||
cmd.Stdout = &o
|
||||
cmd.Stderr = &e
|
||||
err := cmd.Run()
|
||||
switch v := err.(type) {
|
||||
case nil:
|
||||
exit = 0
|
||||
case *exec.ExitError:
|
||||
exit = v.ExitCode()
|
||||
default:
|
||||
t.Fatalf("runCLI: exec failed: %v", err)
|
||||
}
|
||||
return o.String(), e.String(), exit
|
||||
}
|
||||
|
||||
func TestCmdName(t *testing.T) {
|
||||
save := os.Args
|
||||
t.Cleanup(func() { os.Args = save })
|
||||
|
||||
cases := []struct {
|
||||
argv0 string
|
||||
sub string
|
||||
want string
|
||||
}{
|
||||
{"/usr/bin/ctool", "fetch", "ctool fetch"},
|
||||
{"/usr/bin/ctool", "tail", "ctool tail"},
|
||||
{"ctool", "fetch", "ctool fetch"},
|
||||
{"ctool.exe", "fetch", "ctool fetch"},
|
||||
{"/usr/bin/ctfetch", "fetch", "ctfetch"},
|
||||
{"ctail", "tail", "ctail"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
os.Args = []string{c.argv0}
|
||||
if got := cmdName(c.sub); got != c.want {
|
||||
t.Errorf("argv0=%q sub=%q: cmdName()=%q, want %q", c.argv0, c.sub, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// capturePrintVersion captures os.Stdout while calling printVersion(). We
|
||||
// assert on the format rather than the exact version string so the test
|
||||
// stays stable whether or not -ldflags stamped a commit/date.
|
||||
func capturePrintVersion(t *testing.T, argv0 string) string {
|
||||
t.Helper()
|
||||
saveArgs, saveStdout := os.Args, os.Stdout
|
||||
t.Cleanup(func() { os.Args, os.Stdout = saveArgs, saveStdout })
|
||||
os.Args = []string{argv0}
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("pipe: %v", err)
|
||||
}
|
||||
os.Stdout = w
|
||||
printVersion()
|
||||
_ = w.Close()
|
||||
out, _ := io.ReadAll(r)
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func TestPrintVersion(t *testing.T) {
|
||||
cases := []struct {
|
||||
argv0 string
|
||||
prefix string
|
||||
}{
|
||||
{"ctool", "ctool version "},
|
||||
{"ctfetch", "ctfetch version "},
|
||||
{"/usr/bin/ctail", "ctail version "},
|
||||
}
|
||||
for _, c := range cases {
|
||||
out := capturePrintVersion(t, c.argv0)
|
||||
if !strings.HasPrefix(out, c.prefix) {
|
||||
t.Errorf("argv0=%q: printVersion()=%q, want prefix %q", c.argv0, out, c.prefix)
|
||||
}
|
||||
if !strings.Contains(out, "commit ") || !strings.Contains(out, "built ") {
|
||||
t.Errorf("argv0=%q: printVersion()=%q, missing commit/built fields", c.argv0, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchVersion(t *testing.T) {
|
||||
cases := []struct {
|
||||
argv0 string
|
||||
args []string
|
||||
prefix string
|
||||
}{
|
||||
{"ctool", []string{"version"}, "ctool version "},
|
||||
{"ctool", []string{"-version"}, "ctool version "},
|
||||
{"ctool", []string{"--version"}, "ctool version "},
|
||||
{"ctool", []string{"fetch", "version"}, "ctool version "},
|
||||
{"ctool", []string{"tail", "version"}, "ctool version "},
|
||||
{"ctfetch", []string{"version"}, "ctfetch version "},
|
||||
{"ctail", []string{"version"}, "ctail version "},
|
||||
}
|
||||
for _, c := range cases {
|
||||
stdout, _, exit := runCLI(t, c.argv0, c.args...)
|
||||
if exit != 0 {
|
||||
t.Errorf("argv=%v %v: exit %d, want 0", c.argv0, c.args, exit)
|
||||
}
|
||||
if !strings.HasPrefix(stdout, c.prefix) {
|
||||
t.Errorf("argv=%v %v: stdout=%q, want prefix %q", c.argv0, c.args, stdout, c.prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchNoArgsShowsUsage(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit, got 0 (stderr=%q)", stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "Usage: ctool") {
|
||||
t.Errorf("stderr missing usage header: %q", stderr)
|
||||
}
|
||||
if !strings.Contains(stderr, "fetch") || !strings.Contains(stderr, "tail") || !strings.Contains(stderr, "version") {
|
||||
t.Errorf("stderr missing subcommand list: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchUnknownSubcommand(t *testing.T) {
|
||||
_, stderr, exit := runCLI(t, "ctool", "bogus")
|
||||
if exit == 0 {
|
||||
t.Errorf("expected non-zero exit, got 0")
|
||||
}
|
||||
if !strings.Contains(stderr, "unknown command") {
|
||||
t.Errorf("stderr=%q", stderr)
|
||||
}
|
||||
}
|
||||
55
debian/build-deb.sh
vendored
Executable file
55
debian/build-deb.sh
vendored
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Build the ctool Debian package for one architecture.
|
||||
# Usage: build-deb.sh ctool <amd64|arm64> <version>
|
||||
#
|
||||
# The commit hash and build date are baked into the binary at link time
|
||||
# via -ldflags in the Makefile, so `ctool version` is the source of truth
|
||||
# for "which build". The .deb itself carries only the release version.
|
||||
set -euo pipefail
|
||||
|
||||
PACKAGE="${1:?usage: build-deb.sh ctool <amd64|arm64> <version>}"
|
||||
ARCH="${2:?usage: build-deb.sh ctool <amd64|arm64> <version>}"
|
||||
VERSION="${3:?usage: build-deb.sh ctool <amd64|arm64> <version>}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PKG="${PACKAGE}_${VERSION}_${ARCH}"
|
||||
STAGING="$(mktemp -d)"
|
||||
trap 'rm -rf "$STAGING"' EXIT
|
||||
|
||||
echo "Building ${PKG}.deb"
|
||||
|
||||
if [ "$PACKAGE" != "ctool" ]; then
|
||||
echo "error: unknown package '${PACKAGE}' (expected ctool)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
install -d "$STAGING/DEBIAN"
|
||||
install -d "$STAGING/usr/bin"
|
||||
install -d "$STAGING/usr/share/man/man1"
|
||||
|
||||
# Binary. ctool is an interactive tool, not a daemon, so it lives in
|
||||
# /usr/bin and is on every login shell's PATH by default.
|
||||
install -m 755 "$REPO_ROOT/build/${ARCH}/ctool" "$STAGING/usr/bin/ctool"
|
||||
|
||||
# ctfetch and ctail are busybox-style aliases: the binary dispatches on
|
||||
# argv[0], so a symlink is all it takes for `ctfetch ...` to mean
|
||||
# `ctool fetch ...` and `ctail ...` to mean `ctool tail ...`.
|
||||
ln -s ctool "$STAGING/usr/bin/ctfetch"
|
||||
ln -s ctool "$STAGING/usr/bin/ctail"
|
||||
|
||||
# Manpages — one overview (ctool) plus one detailed page per subcommand.
|
||||
gzip -9 -c "$REPO_ROOT/docs/ctool.1" > "$STAGING/usr/share/man/man1/ctool.1.gz"
|
||||
gzip -9 -c "$REPO_ROOT/docs/ctfetch.1" > "$STAGING/usr/share/man/man1/ctfetch.1.gz"
|
||||
gzip -9 -c "$REPO_ROOT/docs/ctail.1" > "$STAGING/usr/share/man/man1/ctail.1.gz"
|
||||
|
||||
# DEBIAN metadata. No conffiles and no maintainer scripts — the package
|
||||
# ships a single binary, two symlinks, and three manpages.
|
||||
sed "s/@VERSION@/${VERSION}/;s/@ARCH@/${ARCH}/" \
|
||||
"$REPO_ROOT/debian/${PACKAGE}.control.in" > "$STAGING/DEBIAN/control"
|
||||
|
||||
# Emit package into build/
|
||||
mkdir -p "$REPO_ROOT/build"
|
||||
OUT="$REPO_ROOT/build/${PKG}.deb"
|
||||
dpkg-deb --build --root-owner-group "$STAGING" "$OUT"
|
||||
echo "Built: $OUT"
|
||||
24
debian/ctool.control.in
vendored
Normal file
24
debian/ctool.control.in
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
Package: ctool
|
||||
Version: @VERSION@
|
||||
Architecture: @ARCH@
|
||||
Maintainer: Pim van Pelt <pim@ipng.ch>
|
||||
Section: net
|
||||
Priority: optional
|
||||
Description: Tools for working with Static CT log tiles
|
||||
ctool is a busybox-style binary that fetches and decodes entries
|
||||
from Static CT API logs (c2sp.org/static-ct-api).
|
||||
.
|
||||
ctool fetch reads one or more log entries from a data tile —
|
||||
either by leaf index or by dumping a whole tile URL / local file —
|
||||
and prints them as structured JSON. Optional modifiers decode
|
||||
embedded SCTs, fetch the issuer certificate from the log, and
|
||||
enrich each SCT with operator and state information from the
|
||||
Chrome CT log list.
|
||||
.
|
||||
ctool tail follows a log's /checkpoint endpoint and prints a
|
||||
one-line summary per certificate or precertificate as new data
|
||||
tiles complete. Useful for live monitoring of a log's growth.
|
||||
.
|
||||
The package installs ctool under /usr/bin along with ctfetch
|
||||
and ctail symlinks, which invoke the matching subcommand
|
||||
directly for scripting convenience.
|
||||
124
docs/ctail.1
Normal file
124
docs/ctail.1
Normal file
@@ -0,0 +1,124 @@
|
||||
.TH CTAIL 1 "April 2026" "ctool" "User Commands"
|
||||
.SH NAME
|
||||
ctail \- tail a Static CT log, printing one line per new entry
|
||||
.SH SYNOPSIS
|
||||
.B ctail
|
||||
[\fIflags\fR] \fIlog\-url\fR
|
||||
.PP
|
||||
.B ctail version
|
||||
.SH DESCRIPTION
|
||||
.B ctail
|
||||
polls a Static CT API log (c2sp.org/static\-ct\-api) and prints a
|
||||
one\-liner to stdout for every certificate or precertificate that
|
||||
appears in a newly\-completed data tile.
|
||||
It is also available as the
|
||||
.B tail
|
||||
subcommand of
|
||||
.BR ctool (1);
|
||||
the two invocations are equivalent.
|
||||
.PP
|
||||
Status messages (startup banner, HTTP errors, progress complaints) go
|
||||
to stderr; only the per\-entry one\-liners go to stdout, so the
|
||||
output is safe to pipe into
|
||||
.BR grep ,
|
||||
.BR awk ,
|
||||
or a log collector without mixing the two streams.
|
||||
.SH OUTPUT FORMAT
|
||||
One line per log entry, fixed\-width columns separated by spaces:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
<leaf-index> <type> <validity-range> <issuer> <subject>
|
||||
.EE
|
||||
.RE
|
||||
.TP
|
||||
.B leaf\-index
|
||||
Decimal index of the entry within the log's Merkle tree.
|
||||
.TP
|
||||
.B type
|
||||
.B cert
|
||||
for a final certificate,
|
||||
.B pre
|
||||
for a precertificate.
|
||||
.TP
|
||||
.B validity\-range
|
||||
.IR YYYY\-MM\-DD .. YYYY\-MM\-DD ,
|
||||
taken from the certificate's
|
||||
.I NotBefore
|
||||
and
|
||||
.I NotAfter
|
||||
fields.
|
||||
.TP
|
||||
.B issuer
|
||||
Issuer common name, prefixed with the issuer organisation when the
|
||||
CN alone is terse (so
|
||||
.I R13
|
||||
is shown as
|
||||
.IR "Let's Encrypt R13" );
|
||||
truncated to 40 characters with a trailing ellipsis if necessary.
|
||||
.TP
|
||||
.B subject
|
||||
First DNS Subject Alternative Name, falling back to the certificate's
|
||||
subject common name. Shown as
|
||||
.I (unknown)
|
||||
when neither is populated.
|
||||
.SH FLAGS
|
||||
.TP
|
||||
.BI \-\-interval " duration"
|
||||
How often to poll the log's
|
||||
.B /checkpoint
|
||||
endpoint. Minimum
|
||||
.IR 1s ;
|
||||
default
|
||||
.IR 15s .
|
||||
The interval timer starts when the checkpoint is fetched, so time
|
||||
spent fetching the ensuing data tiles counts against the interval
|
||||
and the next poll stays on schedule.
|
||||
.TP
|
||||
.BI \-\-from\-leaf " n"
|
||||
Start at leaf index
|
||||
.IR n .
|
||||
The default,
|
||||
.IR \-1 ,
|
||||
means "start at the current tree tip" \(em no backfill.
|
||||
.TP
|
||||
.BI \-\-rate\-limit " duration"
|
||||
Minimum time between outgoing HTTP requests to the log. Minimum
|
||||
.IR 100ms ;
|
||||
default
|
||||
.IR 2s .
|
||||
Lets the client stay a polite distance below any per\-IP rate limit
|
||||
the log operator enforces.
|
||||
.TP
|
||||
.BI \-\-user\-agent " string"
|
||||
User\-Agent header sent with every request. Default:
|
||||
\fIctail/VERSION (https://git.ipng.ch/certificate\-transparency/)\fR.
|
||||
.SH SUBCOMMANDS
|
||||
.TP
|
||||
.B version
|
||||
Print the binary's version, git commit hash, and build date, then
|
||||
exit.
|
||||
.SH EXAMPLES
|
||||
Follow a log from the current tip:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctail https://halloumi2026h2.mon.ct.ipng.ch
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Replay a log from the very first entry, polling every ten seconds:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctail \-\-from\-leaf 0 \-\-interval 10s https://halloumi2026h2.mon.ct.ipng.ch
|
||||
.EE
|
||||
.RE
|
||||
.SH NOTES
|
||||
A data tile is only fetched once the checkpoint confirms it is
|
||||
complete (256 entries). This avoids unnecessary 404s at the tree tip.
|
||||
.SH SEE ALSO
|
||||
.BR ctool (1),
|
||||
.BR ctfetch (1)
|
||||
.SH AUTHOR
|
||||
Pim van Pelt <pim@ipng.ch>
|
||||
130
docs/ctfetch.1
Normal file
130
docs/ctfetch.1
Normal file
@@ -0,0 +1,130 @@
|
||||
.TH CTFETCH 1 "April 2026" "ctool" "User Commands"
|
||||
.SH NAME
|
||||
ctfetch \- fetch and decode Static CT log entries as JSON
|
||||
.SH SYNOPSIS
|
||||
.B ctfetch
|
||||
[\fIflags\fR] \fIlog\-url\fR \fIleaf\-index\fR [\fB+sct\fR] [\fB+issuer\fR] [\fB+ctlog\fR] [\fB+all\fR]
|
||||
.PP
|
||||
.B ctfetch
|
||||
[\fIflags\fR] \fItile\-url\-or\-file\fR [\fB+sct\fR] [\fB+issuer\fR] [\fB+ctlog\fR] [\fB+all\fR]
|
||||
.PP
|
||||
.B ctfetch version
|
||||
.SH DESCRIPTION
|
||||
.B ctfetch
|
||||
reads entries from a Static CT API log (c2sp.org/static\-ct\-api) and
|
||||
writes them to stdout as pretty\-printed JSON.
|
||||
It is also available as the
|
||||
.B fetch
|
||||
subcommand of
|
||||
.BR ctool (1);
|
||||
the two invocations are equivalent.
|
||||
.PP
|
||||
Two modes are distinguished by whether the second positional argument
|
||||
parses as an integer.
|
||||
.SS Leaf\-index mode
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
ctfetch <log\-url> <leaf\-index> [modifiers...]
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Fetches the data tile that contains
|
||||
.IR leaf\-index ,
|
||||
decompresses it, and decodes the single entry at that position.
|
||||
.SS Tile\-dump mode
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
ctfetch <tile\-url\-or\-file> [modifiers...]
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Fetches (or reads from disk) one tile and decodes every entry in it.
|
||||
Hash tiles (\fB/tile/N/\fR..., N \(>= 0) produce the list of 32\-byte
|
||||
SHA\-256 node hashes; output modifiers are an error in this case.
|
||||
Data tiles (\fB/tile/data/\fR...) produce the full decoded
|
||||
entry list.
|
||||
.SH OUTPUT MODIFIERS
|
||||
The modifiers are positional tokens beginning with
|
||||
.BR + .
|
||||
They control which optional fields are computed and included in the
|
||||
JSON output.
|
||||
.TP
|
||||
.B +sct
|
||||
Parse the embedded Signed Certificate Timestamp list from final
|
||||
(non\-precert) certificates and include it alongside the entry.
|
||||
.TP
|
||||
.B +issuer
|
||||
Fetch the issuer certificate from the log's
|
||||
.B /issuer/<fp>
|
||||
endpoint and include parsed issuer details.
|
||||
.TP
|
||||
.B +ctlog
|
||||
Look up each SCT's log ID in the CT log list (see
|
||||
.BR \-\-logs\-list\-url )
|
||||
and enrich it with operator and state information.
|
||||
.TP
|
||||
.B +all
|
||||
Shorthand for
|
||||
.BR +sct " " +issuer " " +ctlog .
|
||||
.SH FLAGS
|
||||
.TP
|
||||
.BI \-\-logs\-list\-url " url"
|
||||
CT log list JSON used for
|
||||
.B +ctlog
|
||||
enrichment.
|
||||
Default:
|
||||
.IR https://www.gstatic.com/ct/log_list/v3/all_logs_list.json .
|
||||
.TP
|
||||
.BI \-\-monitoring\-url " url"
|
||||
Log root URL used for
|
||||
.B +issuer
|
||||
lookups when the input is a local tile file. Ignored when the input
|
||||
is already an HTTP(S) URL; in that case the root is derived by
|
||||
stripping
|
||||
.I /tile/...
|
||||
from the path.
|
||||
.SH SUBCOMMANDS
|
||||
.TP
|
||||
.B version
|
||||
Print the binary's version, git commit hash, and build date, then
|
||||
exit.
|
||||
.SH EXAMPLES
|
||||
Fetch one entry with all enrichments:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Dump a data tile straight off the web:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctfetch https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +sct +ctlog
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Dump a tile from disk, pointing at a monitoring URL so
|
||||
.B +issuer
|
||||
lookups can find the issuer endpoint:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctfetch \-\-monitoring\-url https://halloumi2026h1.mon.ct.ipng.ch tile.bin +issuer
|
||||
.EE
|
||||
.RE
|
||||
.SH NOTES
|
||||
Partial tiles (the
|
||||
.I .p/N
|
||||
suffix) are tried first; on 404 the full tile is fetched
|
||||
automatically.
|
||||
The CT log list and any fetched issuer certificates are cached in
|
||||
memory for the lifetime of a single invocation.
|
||||
.SH SEE ALSO
|
||||
.BR ctool (1),
|
||||
.BR ctail (1)
|
||||
.SH AUTHOR
|
||||
Pim van Pelt <pim@ipng.ch>
|
||||
83
docs/ctool.1
Normal file
83
docs/ctool.1
Normal file
@@ -0,0 +1,83 @@
|
||||
.TH CTOOL 1 "April 2026" "ctool" "User Commands"
|
||||
.SH NAME
|
||||
ctool \- tools for working with Static CT log tiles
|
||||
.SH SYNOPSIS
|
||||
.B ctool
|
||||
.I command
|
||||
[\fIflags\fR] [\fIargs\fR...]
|
||||
.PP
|
||||
.B ctool
|
||||
.B version
|
||||
.SH DESCRIPTION
|
||||
.B ctool
|
||||
is a busybox\-style front\-end for a set of tools that read tiles from
|
||||
a Static CT API log (c2sp.org/static\-ct\-api).
|
||||
It dispatches on the first argument to one of the subcommands below.
|
||||
The same binary can also be invoked through the
|
||||
.B ctfetch
|
||||
or
|
||||
.B ctail
|
||||
symlinks, in which case the subcommand name is implied by the link
|
||||
name; flags and arguments are unchanged.
|
||||
.SH COMMANDS
|
||||
.TP
|
||||
.B fetch
|
||||
Fetch and decode one or more entries from a Static CT log and print
|
||||
them as structured JSON. Supports both leaf\-index lookups and
|
||||
full\-tile dumps, with optional SCT decoding, issuer\-certificate
|
||||
lookup, and CT log\-list enrichment. See
|
||||
.BR ctfetch (1)
|
||||
for the full reference.
|
||||
.TP
|
||||
.B tail
|
||||
Follow a Static CT log. Polls the log's checkpoint, walks new data
|
||||
tiles as they complete, and prints a one\-line summary per certificate
|
||||
or precertificate \(em leaf index, type, validity range, issuer, and
|
||||
subject name. See
|
||||
.BR ctail (1)
|
||||
for the full reference.
|
||||
.TP
|
||||
.B version
|
||||
Print the binary's version, git commit hash, and build date, then
|
||||
exit. Equivalent forms
|
||||
.B ctool fetch version
|
||||
and
|
||||
.B ctool tail version
|
||||
also work and print the same information.
|
||||
.SH EXAMPLES
|
||||
Fetch a single leaf entry with all optional enrichments:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctool fetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +all
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Tail a log from the current tree tip:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctool tail https://halloumi2026h2.mon.ct.ipng.ch
|
||||
.EE
|
||||
.RE
|
||||
.PP
|
||||
Print the build version:
|
||||
.PP
|
||||
.RS
|
||||
.EX
|
||||
$ ctool version
|
||||
ctool version 0.1.0 (commit abc1234, built 2026\-04\-21T12:00:00Z)
|
||||
.EE
|
||||
.RE
|
||||
.SH "FULL DOCUMENTATION"
|
||||
This manpage is an overview only. For the complete reference of each
|
||||
subcommand \(em every flag, every output modifier, and the format of
|
||||
the JSON and tail\-line output \(em see the per\-command manpages:
|
||||
.BR ctfetch (1)
|
||||
and
|
||||
.BR ctail (1).
|
||||
.SH SEE ALSO
|
||||
.BR ctfetch (1),
|
||||
.BR ctail (1)
|
||||
.SH AUTHOR
|
||||
Pim van Pelt <pim@ipng.ch>
|
||||
621
docs/design.md
Normal file
621
docs/design.md
Normal file
@@ -0,0 +1,621 @@
|
||||
# ctool Design Document
|
||||
|
||||
## Metadata
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| **Status** | Retrofit — describes shipped behavior as of `v0.1.0` |
|
||||
| **Author** | Pim van Pelt `<pim@ipng.ch>` |
|
||||
| **Last updated** | 2026-04-21 |
|
||||
| **Audience** | Operators and contributors who will read the source tree next |
|
||||
|
||||
The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and
|
||||
**MAY** are used as described in
|
||||
[RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119), and are
|
||||
reserved in this document for requirements that are actually enforced
|
||||
in code or by an external dependency. Plain-language descriptions of
|
||||
what the system or an operator can do are written in lowercase —
|
||||
"can", "will", "does" — and should not be read as normative.
|
||||
|
||||
## Summary
|
||||
|
||||
`ctool` is a small collection of command-line tools for working with
|
||||
[Static CT API](https://c2sp.org/static-ct-api) logs. It ships as a
|
||||
single Go binary that dispatches to one of several subcommands. Today
|
||||
there are two: `fetch`, which decodes one or more entries from a log
|
||||
tile as structured JSON, and `tail`, which follows a log's checkpoint
|
||||
and prints a one-liner per new certificate. The binary is also
|
||||
exported as two busybox-style symlinks, `ctfetch` and `ctail`, so that
|
||||
scripts can call the subcommand directly without the outer `ctool`
|
||||
prefix.
|
||||
|
||||
## Background
|
||||
|
||||
The Static CT API defines a stateless on-disk layout for Certificate
|
||||
Transparency logs: each leaf lives in a 256-entry gzip-compressed
|
||||
*data tile* fetched by path, and a Merkle tree over those leaves is
|
||||
published as a parallel set of *hash tiles*. Because the layout is
|
||||
static, a log is just a tree of files on a web server — anyone with
|
||||
an HTTP client and a decoder can read one. The CT ecosystem is
|
||||
shifting from the old RFC 6962 "logs are a database" model to this
|
||||
static-tile model; operators running or watching such logs need
|
||||
ad-hoc tools to poke at tiles by hand and to watch a log grow in real
|
||||
time.
|
||||
|
||||
`ctool` is those tools, gathered under one roof so that packaging,
|
||||
versioning, and future shared utilities live in a single place. The
|
||||
project deliberately stays small: each subcommand is a thin wrapper
|
||||
over [`filippo.io/sunlight`](https://pkg.go.dev/filippo.io/sunlight)
|
||||
and Go's standard `crypto/x509`, with a small shared helper package
|
||||
for the fiddly parts (issuer fetching, SCT decoding, CT log list
|
||||
enrichment).
|
||||
|
||||
## Goals and Non-Goals
|
||||
|
||||
### Product Goals
|
||||
|
||||
1. **Operator utility first.** The tools exist to make Static CT
|
||||
logs inspectable by a human with a shell. Scriptable output
|
||||
(JSON for `fetch`, fixed-column lines for `tail`) is a
|
||||
first-class concern.
|
||||
2. **No state.** Every invocation is independent; there are no
|
||||
databases, no lock files, no background processes.
|
||||
3. **Portable packaging.** A single statically-linked binary that
|
||||
works on any Linux amd64/arm64 host without touching system
|
||||
libraries.
|
||||
4. **Extensible dispatcher.** `ctool` will grow new subcommands
|
||||
over time. Adding one MUST NOT require a new binary, a new
|
||||
Debian package, or any changes outside the `cmd/ctool` tree.
|
||||
|
||||
### Non-Goals
|
||||
|
||||
- `ctool` is not a CT log itself. It does not host, sign, or
|
||||
distribute tiles.
|
||||
- It is not a CT monitor in the Google-Chrome-policy sense.
|
||||
Detecting misissuance is left to whatever the operator feeds
|
||||
`ctool tail`'s output into.
|
||||
- It is not a general x509 pretty-printer. The decoded fields
|
||||
are the ones CT operators typically care about (SCTs, issuer
|
||||
chain, validity range); a general "dump every extension"
|
||||
mode is out of scope.
|
||||
- It does not secure its own outbound traffic. Plain HTTP is
|
||||
accepted when the operator passes an `http://` URL; it is the
|
||||
operator's responsibility to prefer HTTPS for production logs.
|
||||
- It is not a long-running service. `tail` runs in a loop, but it
|
||||
is a foreground process — there is no daemon, no systemd unit,
|
||||
no PID file.
|
||||
|
||||
## Requirements
|
||||
|
||||
Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`)
|
||||
so that later sections can cite it.
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
**FR-1 Dispatcher (`ctool`)**
|
||||
|
||||
- **FR-1.1** A single binary MUST dispatch its first positional
|
||||
argument to a named subcommand. Unknown subcommands MUST exit
|
||||
non-zero with a usage banner on stderr.
|
||||
- **FR-1.2** The binary MUST recognize busybox-style invocation:
|
||||
when `os.Args[0]`'s basename is `ctfetch` or `ctail` (with or
|
||||
without a platform-native extension such as `.exe`), the
|
||||
corresponding subcommand is invoked directly and the
|
||||
remaining argv is passed through unchanged.
|
||||
- **FR-1.3** The dispatcher MUST expose a `version` subcommand
|
||||
that prints the build version, git commit hash, and build
|
||||
date on stdout and exits zero. The same information MUST also
|
||||
be reachable as `ctool -version`, `ctool --version`,
|
||||
`ctool fetch version`, and `ctool tail version`, so that
|
||||
"find out what you have" works regardless of which form the
|
||||
operator guesses first.
|
||||
- **FR-1.4** Usage banners emitted by a subcommand MUST refer to
|
||||
the tool by its effective invocation name — `ctfetch ...` when
|
||||
invoked through the symlink, `ctool fetch ...` when invoked
|
||||
through the outer dispatcher — so that copy/paste from an
|
||||
error message yields a command that works.
|
||||
- **FR-1.5** Adding a new subcommand MUST require only: a new
|
||||
`cmd_<name>.go` file in `cmd/ctool/`, a new `case` in the
|
||||
dispatcher's switch, and (optionally) a new busybox symlink
|
||||
in the package layout. No changes to the Makefile's build
|
||||
graph and no new binaries are required.
|
||||
|
||||
**FR-2 `fetch` subcommand**
|
||||
|
||||
- **FR-2.1** `ctool fetch` MUST operate in two modes,
|
||||
distinguished by whether the second positional argument parses
|
||||
as a signed decimal integer:
|
||||
- **Leaf-index mode**: `<log-url> <leaf-index> [modifiers]`
|
||||
fetches the data tile containing `leaf-index` and decodes
|
||||
the single entry at that position.
|
||||
- **Tile-dump mode**: `<tile-url-or-file> [modifiers]`
|
||||
fetches or reads one tile and decodes every entry in it.
|
||||
- **FR-2.2** Tile-dump mode MUST auto-detect hash tiles versus
|
||||
data tiles from the tile contents. For hash tiles, the output
|
||||
is the list of 32-byte SHA-256 node hashes and the `+sct`,
|
||||
`+issuer`, `+ctlog`, and `+all` modifiers MUST be rejected as
|
||||
an error.
|
||||
- **FR-2.3** The output format MUST be pretty-printed JSON on
|
||||
stdout with a trailing newline.
|
||||
- **FR-2.4** The following positional modifiers MUST be
|
||||
accepted, and any other `+`-prefixed token MUST be rejected as
|
||||
an unknown argument:
|
||||
- `+sct` — decode embedded SCTs on final certificates.
|
||||
- `+issuer` — fetch and parse the issuer certificate from
|
||||
the log's `/issuer/<fingerprint>` endpoint.
|
||||
- `+ctlog` — enrich each SCT with operator and state data
|
||||
from the CT log list.
|
||||
- `+all` — shorthand for all three of the above.
|
||||
- **FR-2.5** When `+issuer` is used with tile-dump mode on a
|
||||
local file, the operator MUST provide `--monitoring-url`; the
|
||||
subcommand MUST otherwise derive the log root from the tile
|
||||
URL by stripping the `/tile/...` path component.
|
||||
- **FR-2.6** Partial tile suffixes (`.p/N`) MUST be tried first
|
||||
when the path advertises one; on HTTP 404 the full tile MUST
|
||||
be fetched by stripping the suffix.
|
||||
|
||||
**FR-3 `tail` subcommand**
|
||||
|
||||
- **FR-3.1** `ctool tail` MUST poll the log's `/checkpoint`
|
||||
endpoint at a configurable interval (default 15s, minimum 1s),
|
||||
and for each completed data tile that appears after the
|
||||
current cursor, MUST decode every entry and print a one-line
|
||||
summary to stdout.
|
||||
- **FR-3.2** By default, the cursor starts at the current tree
|
||||
tip so that only *new* entries are printed. `--from-leaf N`
|
||||
MUST override this so that operators can replay from a known
|
||||
index (including `--from-leaf 0` for a full backfill).
|
||||
- **FR-3.3** The one-liner format MUST be fixed-column and
|
||||
space-separated: leaf index (right-aligned, nine columns),
|
||||
type (`cert` or `pre `), validity range
|
||||
(`YYYY-MM-DD..YYYY-MM-DD`), issuer label (up to 40 chars,
|
||||
truncated with `...`), subject name.
|
||||
- **FR-3.4** The subject name MUST be the first DNS SAN;
|
||||
falling back to the certificate's subject common name; falling
|
||||
back to the literal `(unknown)` if neither is present.
|
||||
- **FR-3.5** The issuer label MUST prepend the issuer
|
||||
organisation to the issuer CN when the CN's first word is not
|
||||
already contained in the organisation name (e.g. `R13` is
|
||||
shown as `Let's Encrypt R13`, but `Let's Encrypt Authority X3`
|
||||
is shown verbatim).
|
||||
- **FR-3.6** Status and error messages MUST go to stderr;
|
||||
one-liners MUST go to stdout, so that the output stream is
|
||||
safe to pipe into `grep`, `awk`, or a log shipper without
|
||||
interleaving diagnostic noise.
|
||||
- **FR-3.7** A tile MUST NOT be fetched until the checkpoint
|
||||
confirms it is complete (256 entries), to avoid unnecessary
|
||||
404s at the tree tip.
|
||||
- **FR-3.8** A configurable minimum delay between outgoing HTTP
|
||||
requests MUST be enforced (default 2s, minimum 100ms), to stay
|
||||
a polite distance below any per-IP rate limit a log operator
|
||||
might enforce.
|
||||
|
||||
**FR-4 Packaging and versioning**
|
||||
|
||||
- **FR-4.1** The build MUST produce statically linked binaries
|
||||
for `linux/amd64` and `linux/arm64`. `CGO_ENABLED=0` MUST be
|
||||
the default so that the resulting binary has no libc
|
||||
dependency and runs on any Linux host of the matching
|
||||
architecture.
|
||||
- **FR-4.2** The version, commit hash, and build date MUST be
|
||||
injected at link time via `-ldflags -X`, so that `ctool
|
||||
version` on a shipped binary identifies the exact build
|
||||
without requiring access to the source tree. The defaults
|
||||
baked into the source MUST keep `go run` / `go build` without
|
||||
ldflags usable.
|
||||
- **FR-4.3** The Debian package MUST install `ctool` under
|
||||
`/usr/bin`, MUST provide `/usr/bin/ctfetch` and
|
||||
`/usr/bin/ctail` as symlinks to it, and MUST install the
|
||||
three section-1 manpages (`ctool.1`, `ctfetch.1`, `ctail.1`)
|
||||
gzipped under `/usr/share/man/man1`.
|
||||
|
||||
### Non-Functional Requirements
|
||||
|
||||
**NFR-1 Availability and reliability**
|
||||
|
||||
- **NFR-1.1** A failing HTTP request MUST NOT crash `tail`. The
|
||||
error MUST be logged to stderr and the next poll MUST happen
|
||||
on schedule.
|
||||
- **NFR-1.2** A malformed tile MUST NOT be silently skipped. If
|
||||
a tile cannot be read, the subcommand MUST return an error
|
||||
that names the offending leaf index.
|
||||
|
||||
**NFR-2 Determinism and correctness**
|
||||
|
||||
- **NFR-2.1** Given the same tile input, `fetch` MUST produce
|
||||
byte-identical JSON output. No timestamps, random IDs, or
|
||||
map iteration order MAY leak into the output.
|
||||
- **NFR-2.2** `tail`'s polling interval timer MUST start when
|
||||
the checkpoint is fetched, not when the previous loop
|
||||
finished, so that time spent fetching data tiles counts
|
||||
against the interval and the next poll stays on schedule.
|
||||
|
||||
**NFR-3 Performance and scalability**
|
||||
|
||||
- **NFR-3.1** `fetch` MUST fetch at most one tile per
|
||||
invocation. Leaf-index mode derives the one tile that
|
||||
contains the requested index; tile-dump mode operates on
|
||||
exactly the tile the operator named.
|
||||
- **NFR-3.2** Caches (CT log list, issuer certificates) MUST
|
||||
be scoped to a single invocation. No on-disk cache, no
|
||||
cross-invocation state.
|
||||
|
||||
**NFR-4 Security**
|
||||
|
||||
- **NFR-4.1** Tiles MUST be decompressed with a bounded
|
||||
expansion ratio (currently 100×) to prevent a malicious log
|
||||
from driving the client out of memory via a zip bomb.
|
||||
- **NFR-4.2** The tools MUST NOT require any Linux
|
||||
capabilities or root privileges. They are plain HTTP clients
|
||||
and run unprivileged.
|
||||
- **NFR-4.3** `ctool` MUST NOT write anywhere on the
|
||||
filesystem except when the operator explicitly asks it to
|
||||
(e.g. by redirecting stdout). There is no cache directory,
|
||||
no log file, no state directory.
|
||||
|
||||
**NFR-5 Operability**
|
||||
|
||||
- **NFR-5.1** Every CLI flag SHOULD have a sensible default so
|
||||
that the most common invocation is a single URL plus maybe a
|
||||
leaf index.
|
||||
- **NFR-5.2** The JSON output of `fetch` SHOULD be stable
|
||||
across patch releases within a minor version. Field additions
|
||||
are allowed; renames and removals are not.
|
||||
- **NFR-5.3** Subcommand usage banners SHOULD include at least
|
||||
one concrete example against `mon.ct.ipng.ch`, which is the
|
||||
reference log we test against.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
### Process Model
|
||||
|
||||
`ctool` is a short-lived foreground process. There are no daemons,
|
||||
no sockets, no persistent state. Each invocation:
|
||||
|
||||
1. Parses argv and dispatches to a subcommand.
|
||||
2. Makes some number of HTTP requests (or reads a local file).
|
||||
3. Writes JSON or one-liners to stdout.
|
||||
4. Exits.
|
||||
|
||||
`tail` is the one exception to step 3 being a single write: it
|
||||
loops, making one checkpoint request per interval and zero or more
|
||||
tile requests per loop iteration. It still exits on Ctrl-C or on an
|
||||
unrecoverable error; there is no restart policy because there is no
|
||||
supervisor.
|
||||
|
||||
### Data Flow
|
||||
|
||||
Configuration flows **in** as command-line flags and positional
|
||||
arguments (there is no config file, no environment variables, no
|
||||
`/etc/` anything). Data flows **in** from the Static CT log's HTTP
|
||||
surface: the `/checkpoint` endpoint, the `/tile/...` tree, and the
|
||||
`/issuer/<fp>` endpoint. The CT log list JSON (default:
|
||||
`gstatic.com/ct/log_list/v3/all_logs_list.json`) is consumed
|
||||
optionally when `+ctlog` is requested. Data flows **out** as JSON or
|
||||
fixed-column lines on stdout, and as status/error messages on
|
||||
stderr.
|
||||
|
||||
## Components
|
||||
|
||||
### `ctool` (dispatcher)
|
||||
|
||||
`ctool` is the outer binary: a Go `main` package under `cmd/ctool/`
|
||||
whose job is to decide which subcommand to run and hand argv off to
|
||||
it. It is deliberately tiny — today about forty lines of code —
|
||||
because every new subcommand adds a `case` to its `switch` and
|
||||
nothing else.
|
||||
|
||||
#### Responsibilities
|
||||
|
||||
- Examine `os.Args[0]` and, if it matches the basename of a
|
||||
known busybox symlink (`ctfetch`, `ctail`), route directly
|
||||
to that subcommand (FR-1.2).
|
||||
- Otherwise, examine `os.Args[1]` and route to the named
|
||||
subcommand, print usage and exit non-zero on unknown names
|
||||
(FR-1.1).
|
||||
- Handle the `version` / `-version` / `--version` top-level
|
||||
alias by printing the build version and exiting zero
|
||||
(FR-1.3).
|
||||
- Provide a single `cmdName()` helper that subcommands use
|
||||
when building usage banners, so that every banner refers to
|
||||
the effective invocation name (FR-1.4).
|
||||
|
||||
#### Extension Model
|
||||
|
||||
Adding a new subcommand `foo` is a three-line change:
|
||||
|
||||
1. Add `cmd/ctool/cmd_foo.go` with an exported `runFoo(args
|
||||
[]string)` function.
|
||||
2. Add `case "foo": runFoo(os.Args[2:])` to `main()`'s switch
|
||||
and a matching line to `usage()`.
|
||||
3. (Optional, if busybox-style invocation is wanted) Add
|
||||
`case "ctfoo": runFoo(os.Args[1:])` to the symlink switch,
|
||||
add the symlink to `debian/build-deb.sh`, and add a
|
||||
`ctfoo(1)` manpage.
|
||||
|
||||
No Makefile change is required (FR-1.5) because the build target
|
||||
globs over `./cmd/ctool/`. No new Debian package is required; the
|
||||
existing `ctool` package gets the new subcommand for free, and
|
||||
optionally a new symlink.
|
||||
|
||||
`version` is specifically *not* a subcommand file — it lives in the
|
||||
dispatcher because every subcommand reuses the same `printVersion()`
|
||||
helper for its own `version` alias (`ctool fetch version`, `ctool
|
||||
tail version`).
|
||||
|
||||
#### Interfaces
|
||||
|
||||
**Presents.**
|
||||
|
||||
- **A command-line interface** driven by `os.Args`. The
|
||||
exit-code contract is 0 on success, 1 on unknown subcommand
|
||||
or subcommand-internal fatal error, and whatever the
|
||||
subcommand chooses otherwise.
|
||||
- **`stdout` and `stderr` streams**. Structured output (JSON,
|
||||
one-liners) goes to stdout; banners, progress messages, and
|
||||
errors go to stderr (FR-3.6, NFR-5.3).
|
||||
|
||||
**Consumes.**
|
||||
|
||||
- **`os.Args`** — the sole input to the dispatcher.
|
||||
|
||||
### `fetch`
|
||||
|
||||
`fetch` (reachable as `ctool fetch` or `ctfetch`) decodes one or
|
||||
more entries from a Static CT log tile.
|
||||
|
||||
#### Responsibilities
|
||||
|
||||
- Distinguish leaf-index mode from tile-dump mode by whether
|
||||
`args[1]` parses as a decimal integer (FR-2.1).
|
||||
- Fetch the relevant tile (or read it from disk), decompress it
|
||||
with a bounded ratio (NFR-4.1), and decode each leaf.
|
||||
- Optionally enrich each entry with decoded SCTs, the issuer
|
||||
certificate, and CT-log-list metadata per the `+sct`,
|
||||
`+issuer`, `+ctlog`, and `+all` modifiers (FR-2.4).
|
||||
- In tile-dump mode, auto-detect hash tiles versus data tiles
|
||||
and refuse the cert-oriented modifiers on hash tiles
|
||||
(FR-2.2).
|
||||
- Emit pretty-printed JSON on stdout (FR-2.3).
|
||||
|
||||
#### Tile Fetching
|
||||
|
||||
The tile URL is derived by [`filippo.io/sunlight`]' `TilePath`
|
||||
function from the requested leaf index; `fetch` tries the partial
|
||||
tile suffix first (`.p/N`) and falls back to the full tile on a 404
|
||||
(FR-2.6). Both paths return gzipped bytes, which are then
|
||||
decompressed with `io.LimitReader` capping expansion at 100× the
|
||||
input size (NFR-4.1). Tile-dump mode on a local file skips the URL
|
||||
derivation and simply reads the file.
|
||||
|
||||
#### Enrichment
|
||||
|
||||
The optional modifiers trigger network calls outside the main tile
|
||||
fetch:
|
||||
|
||||
- `+sct` parses the SCT list extension from the DER cert
|
||||
in-memory; no extra network calls.
|
||||
- `+issuer` fetches `/issuer/<fp>` from the log root for each
|
||||
referenced issuer fingerprint. Results are cached for the
|
||||
lifetime of the invocation (NFR-3.2).
|
||||
- `+ctlog` fetches the CT log list JSON once per invocation
|
||||
and looks up each SCT's log ID against it.
|
||||
|
||||
`+all` is syntactic sugar for all three.
|
||||
|
||||
#### Interfaces
|
||||
|
||||
**Presents.**
|
||||
|
||||
- **Pretty-printed JSON** on stdout, with a trailing newline.
|
||||
The top-level object is either a single `Entry` (leaf-index
|
||||
mode), a `DumpResult` with an `entries` array (data tile),
|
||||
or a `DumpResult` with a `hash_tile` field (hash tile).
|
||||
|
||||
**Consumes.**
|
||||
|
||||
- **Positional arguments** (log URL, leaf index, modifiers) and
|
||||
two flags (`--logs-list-url`, `--monitoring-url`) from argv.
|
||||
- **The Static CT log's HTTP surface** — checkpoint is not
|
||||
read by `fetch`, but the tile tree and the issuer endpoint
|
||||
are.
|
||||
- **The outbound network** directly. No proxy handling, no
|
||||
capability requirements (NFR-4.2).
|
||||
|
||||
### `tail`
|
||||
|
||||
`tail` (reachable as `ctool tail` or `ctail`) follows a Static CT
|
||||
log's checkpoint and prints a one-line summary per new entry.
|
||||
|
||||
#### Responsibilities
|
||||
|
||||
- Poll the log's `/checkpoint` on a configurable interval and
|
||||
track the current tree size (FR-3.1).
|
||||
- For each completed data tile whose entries have not yet been
|
||||
printed, fetch it, decompress it, decode each leaf, and print
|
||||
a fixed-column one-liner to stdout (FR-3.1, FR-3.3).
|
||||
- Throttle outbound HTTP requests with a configurable minimum
|
||||
inter-request delay (FR-3.8).
|
||||
- Respect a configurable starting leaf index so that operators
|
||||
can replay from any point in the log (FR-3.2).
|
||||
|
||||
#### Poll Loop
|
||||
|
||||
Each iteration fetches the checkpoint first and records the
|
||||
timestamp; then walks forward from the cursor tile by tile, stopping
|
||||
at the first tile that the checkpoint does not yet confirm complete
|
||||
(FR-3.7). The next poll sleeps until `checkpoint_time + interval`,
|
||||
so that the time spent fetching data tiles counts against the
|
||||
interval and consecutive polls stay on a stable cadence (NFR-2.2).
|
||||
|
||||
A tile fetch failure is logged to stderr and the iteration is
|
||||
retried on the next poll (NFR-1.1); the cursor does not advance
|
||||
past a leaf that was never printed, so no entries are lost even
|
||||
across transient log outages.
|
||||
|
||||
#### One-Liner Format
|
||||
|
||||
Columns are separated by a single space, each of fixed width. The
|
||||
format string is documented in `docs/ctail.md`, reproduced here:
|
||||
|
||||
```
|
||||
%9d %-4s %-21s %-40s %s
|
||||
```
|
||||
|
||||
Fields: leaf index (decimal), type (`cert` or `pre `), validity
|
||||
range (`YYYY-MM-DD..YYYY-MM-DD`), issuer label (truncated to 40
|
||||
with `...`), subject name. The truncation rule and the
|
||||
issuer-with-org prepending rule are both tested (FR-3.3, FR-3.4,
|
||||
FR-3.5).
|
||||
|
||||
#### Interfaces
|
||||
|
||||
**Presents.**
|
||||
|
||||
- **Fixed-column one-liners** on stdout, one per log entry.
|
||||
- **Status and error messages** on stderr (FR-3.6).
|
||||
|
||||
**Consumes.**
|
||||
|
||||
- **Positional `<log-url>`** plus flags `--interval`,
|
||||
`--from-leaf`, `--rate-limit`, `--user-agent`.
|
||||
- **The log's HTTP surface** (`/checkpoint` and the tile tree).
|
||||
- **The outbound network** directly.
|
||||
|
||||
### Shared Helpers (`internal/utils`)
|
||||
|
||||
`internal/utils` holds the code that both `fetch` and `tail`
|
||||
call into but neither one should own: tile fetching with partial
|
||||
fallback, bounded gzip decompression, SCT decoding, issuer fetching
|
||||
with per-invocation caching, and CT-log-list enrichment. The
|
||||
package is deliberately import-only from `cmd/ctool/`; nothing
|
||||
outside the binary imports it.
|
||||
|
||||
The helpers are not tested in isolation today beyond what the
|
||||
binary-level tests exercise; if the package grows substantially,
|
||||
per-package unit tests SHOULD be added before it becomes difficult
|
||||
to reason about.
|
||||
|
||||
## Operational Concerns
|
||||
|
||||
### Packaging
|
||||
|
||||
The repository ships a single Debian package, `ctool`, built for
|
||||
`amd64` and `arm64`. The package contains `/usr/bin/ctool` plus two
|
||||
symlinks (`ctfetch`, `ctail`) and three manpages under
|
||||
`/usr/share/man/man1`. There are no conffiles, no maintainer scripts,
|
||||
and no systemd units; the package is a pure binary-plus-docs bundle
|
||||
(FR-4.3).
|
||||
|
||||
Binaries are produced with `CGO_ENABLED=0` (FR-4.1). The effect is
|
||||
that the binary has no libc dependency and runs on any Linux
|
||||
amd64/arm64 host regardless of glibc version, at the cost of Go's
|
||||
`net` package using the pure-Go resolver instead of `getaddrinfo`
|
||||
(i.e. `/etc/nsswitch.conf` and NSS modules are ignored). `ctool`
|
||||
only does DNS-over-UDP lookups for log hostnames, so this is fine.
|
||||
|
||||
### Versioning
|
||||
|
||||
Versioning follows [Semantic Versioning](https://semver.org/). The
|
||||
authoritative version string lives in `Makefile`'s `VERSION`
|
||||
variable; on each build, the Makefile stamps `main.version`,
|
||||
`main.commit`, and `main.date` with `-ldflags -X` (FR-4.2). The
|
||||
commit and date are derived from `git rev-parse --short HEAD` and
|
||||
an ISO-8601 UTC timestamp at build time. `ctool version` prints
|
||||
all three; the Debian package's `Version:` field carries only the
|
||||
release version.
|
||||
|
||||
### Failure Modes
|
||||
|
||||
- **Log returns 404 on a tile we expected.** Treat as a
|
||||
transient error; `tail` logs to stderr and retries on the
|
||||
next poll (NFR-1.1). `fetch` exits non-zero with the HTTP
|
||||
status in the error message.
|
||||
- **Log returns gzip bomb.** The decompress helper's
|
||||
`LimitReader` caps expansion at 100× the input size
|
||||
(NFR-4.1); an attempt to exceed it surfaces as a decode
|
||||
error, not an out-of-memory crash.
|
||||
- **Checkpoint is malformed.** `tail` logs a "checkpoint
|
||||
error" on stderr and sleeps for one interval before
|
||||
retrying.
|
||||
- **CT log list is unreachable.** `+ctlog` degrades: a warning
|
||||
is printed to stderr and entries go out without the
|
||||
`ctlog` enrichment rather than failing the whole
|
||||
invocation.
|
||||
- **Operator passes a hash tile URL with `+sct` / `+issuer` /
|
||||
`+ctlog`.** The subcommand detects the tile type and exits
|
||||
with an error explaining that cert-oriented modifiers do not
|
||||
apply to hash tiles (FR-2.2).
|
||||
|
||||
### Observability
|
||||
|
||||
`ctool` has no metrics and no structured logging surface of its
|
||||
own. Observability in practice is whatever the operator builds
|
||||
around the output streams:
|
||||
|
||||
- Pipe `ctool tail` into a log shipper or `awk` to watch for
|
||||
specific issuers or subjects.
|
||||
- Pipe `ctool fetch ... +all` into `jq` for ad-hoc inspection.
|
||||
- Exit codes (0 / non-zero) are the only machine-readable
|
||||
signal of success or failure.
|
||||
|
||||
If a future operational need calls for structured logs or metrics
|
||||
(e.g. a `ctool watch` subcommand that is meant to be run under
|
||||
systemd), it can be added without disturbing `fetch` or `tail`.
|
||||
|
||||
### Security
|
||||
|
||||
The tools run unprivileged (NFR-4.2) and write nothing to disk
|
||||
outside stdout (NFR-4.3). They do not handle credentials, do not
|
||||
parse user-supplied certificates for signature verification, and
|
||||
do not act as a TLS server.
|
||||
|
||||
TLS verification on outbound requests uses Go's default roots;
|
||||
there is no flag to disable it. Operators probing a log over plain
|
||||
HTTP accept the risk of in-flight modification.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- **Separate `ctfetch` and `ctail` binaries, no dispatcher.**
|
||||
Rejected in favor of one binary with symlinks. A single
|
||||
binary is simpler to package, ship, and version, and
|
||||
scales better as new subcommands land (FR-1.5). Operators
|
||||
who prefer the separate-binary UX still get it through the
|
||||
`ctfetch` / `ctail` symlinks.
|
||||
- **A full CLI framework (cobra, urfave/cli).** Rejected
|
||||
because the dispatcher is under fifty lines and the
|
||||
subcommand parsers use the standard `flag` package. A
|
||||
framework would add a dependency and a build surface
|
||||
without improving the operator experience at this scale.
|
||||
If the subcommand count grows past ~five, this decision
|
||||
should be revisited.
|
||||
- **Embedding a local cache for issuer certificates across
|
||||
invocations.** Rejected for now (NFR-3.2). Caching introduces
|
||||
a state directory, a cache-invalidation policy, and a new
|
||||
failure mode (stale issuer); since `fetch` is typically run
|
||||
ad-hoc rather than in a tight loop, the per-invocation cache
|
||||
is sufficient.
|
||||
- **Making `tail` a daemon with a systemd unit.** Rejected in
|
||||
favor of a plain foreground loop. Operators who want a daemon
|
||||
wrap it in `systemd-run`, `tmux`, or their orchestration tool
|
||||
of choice; the tool itself stays stateless and exit-code
|
||||
honest.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- **Output schema stability.** `fetch`'s JSON output is currently
|
||||
whatever `internal/utils` emits. NFR-5.2 commits to additive
|
||||
compatibility across patch releases, but a minor-version bump
|
||||
may still reshape fields. A more explicit schema document (or
|
||||
a `--schema-version` flag) would help downstream consumers.
|
||||
- **More subcommands.** The dispatcher is built for growth —
|
||||
candidate ideas include `ctool verify` (checkpoint signature
|
||||
+ consistency proof) and `ctool stats` (issuer/validity
|
||||
distribution over a time window). These are out of scope for
|
||||
v0.1 but the FR-1.x requirements exist precisely so the
|
||||
additions stay cheap.
|
||||
- **Proxy and retry policy.** There is no HTTP retry today. A
|
||||
log that 502s intermittently will surface those as visible
|
||||
errors on stderr (NFR-1.1) but the cursor still stalls until
|
||||
the log recovers. A configurable retry/backoff policy may be
|
||||
worth adding once we see a deployment hit this in practice.
|
||||
@@ -167,7 +167,7 @@ func FetchURL(url string) ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
@@ -506,9 +506,7 @@ func parseCertDetails(certDER []byte) *CertDetails {
|
||||
IsCA: cert.IsCA,
|
||||
}
|
||||
|
||||
for _, e := range cert.EmailAddresses {
|
||||
d.EmailSANs = append(d.EmailSANs, e)
|
||||
}
|
||||
d.EmailSANs = append(d.EmailSANs, cert.EmailAddresses...)
|
||||
for _, u := range cert.URIs {
|
||||
d.URISANs = append(d.URISANs, u.String())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user