Add ctail, refactor README
This commit is contained in:
241
cmd/ctail/main.go
Normal file
241
cmd/ctail/main.go
Normal file
@@ -0,0 +1,241 @@
|
||||
// Command ctail tails a Static CT log, printing a one-liner per new entry.
|
||||
//
|
||||
// ctail [flags] <log-url>
|
||||
//
|
||||
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/x509"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"filippo.io/sunlight"
|
||||
"git.ipng.ch/certificate-transparency/ctfetch/internal/utils"
|
||||
"golang.org/x/mod/sumdb/tlog"
|
||||
)
|
||||
|
||||
const version = "0.1.0"
|
||||
|
||||
var (
|
||||
userAgent string
|
||||
rateLimit time.Duration
|
||||
lastRequest time.Time
|
||||
)
|
||||
|
||||
func main() {
|
||||
interval := flag.Duration("interval", 15*time.Second, "polling interval")
|
||||
fromLeaf := flag.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)")
|
||||
rateLimitFlag := flag.Duration("rate-limit", 2*time.Second, "minimum time between HTTP requests")
|
||||
flag.StringVar(&userAgent, "user-agent", "ctail/"+version+" (https://git.ipng.ch/certificate-transparency/)", "User-Agent header for HTTP requests")
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "Usage: ctail [flags] <log-url>\n")
|
||||
fmt.Fprintf(os.Stderr, "\nPrints a one-liner per cert/pre-cert as new entries arrive in a Static CT log.\n")
|
||||
fmt.Fprintf(os.Stderr, "\nExamples:\n")
|
||||
fmt.Fprintf(os.Stderr, " ctail https://halloumi2026h1.mon.ct.ipng.ch\n")
|
||||
fmt.Fprintf(os.Stderr, " ctail --from-leaf 0 --interval 10s https://halloumi2026h1.mon.ct.ipng.ch\n")
|
||||
fmt.Fprintf(os.Stderr, "\nFlags:\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
if *interval < time.Second {
|
||||
fmt.Fprintf(os.Stderr, "Error: --interval must be at least 1s\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
if *rateLimitFlag < 100*time.Millisecond {
|
||||
fmt.Fprintf(os.Stderr, "Error: --rate-limit must be at least 100ms\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
rateLimit = *rateLimitFlag
|
||||
|
||||
if flag.NArg() != 1 {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
logURL := strings.TrimSuffix(flag.Arg(0), "/")
|
||||
var nextLeaf int64 = -1
|
||||
|
||||
for {
|
||||
checkpointAt := time.Now()
|
||||
treeSize, err := fetchCheckpointSize(logURL)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "checkpoint error: %v\n", err)
|
||||
time.Sleep(*interval)
|
||||
continue
|
||||
}
|
||||
|
||||
if nextLeaf < 0 {
|
||||
if *fromLeaf >= 0 {
|
||||
fmt.Fprintf(os.Stderr, "Starting from leaf %d, tree size %d, polling every %s\n", *fromLeaf, treeSize, *interval)
|
||||
nextLeaf = *fromLeaf
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Starting at tree size %d, polling every %s\n", treeSize, *interval)
|
||||
nextLeaf = treeSize
|
||||
}
|
||||
}
|
||||
|
||||
if err := processNewEntries(logURL, &nextLeaf, treeSize); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
}
|
||||
|
||||
if wait := time.Until(checkpointAt.Add(*interval)); wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processNewEntries(logURL string, nextLeaf *int64, treeSize int64) error {
|
||||
for *nextLeaf < treeSize {
|
||||
tileStart := (*nextLeaf / sunlight.TileWidth) * sunlight.TileWidth
|
||||
if treeSize < tileStart+sunlight.TileWidth {
|
||||
break // tile not yet complete; wait for next poll
|
||||
}
|
||||
|
||||
tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, *nextLeaf))
|
||||
tile.L = -1
|
||||
tilePath := sunlight.TilePath(tile)
|
||||
// Always fetch the full tile; strip any .p/N partial suffix.
|
||||
if idx := strings.Index(tilePath, ".p/"); idx != -1 {
|
||||
tilePath = tilePath[:idx]
|
||||
}
|
||||
|
||||
tileData, err := fetchURL(logURL + "/" + tilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch tile at leaf %d: %w", *nextLeaf, err)
|
||||
}
|
||||
tileData, err = utils.Decompress(tileData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decompress tile: %w", err)
|
||||
}
|
||||
|
||||
prevNext := *nextLeaf
|
||||
remaining := tileData
|
||||
for len(remaining) > 0 {
|
||||
e, rest, err := sunlight.ReadTileLeaf(remaining)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read tile leaf: %w", err)
|
||||
}
|
||||
remaining = rest
|
||||
if e.LeafIndex >= *nextLeaf {
|
||||
fmt.Println(formatEntry(e))
|
||||
*nextLeaf = e.LeafIndex + 1
|
||||
}
|
||||
}
|
||||
if *nextLeaf == prevNext {
|
||||
return fmt.Errorf("no progress reading tile at leaf %d", *nextLeaf)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchURL(url string) ([]byte, error) {
|
||||
if wait := time.Until(lastRequest.Add(rateLimit)); wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
lastRequest = time.Now()
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func fetchCheckpointSize(logURL string) (int64, error) {
|
||||
data, err := fetchURL(logURL + "/checkpoint")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Checkpoint format (tlog/sunlight):
|
||||
// line 1: origin
|
||||
// line 2: tree size (decimal)
|
||||
// line 3: root hash (base64)
|
||||
// (blank line + signature lines)
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
for i := 0; scanner.Scan(); i++ {
|
||||
if i == 1 {
|
||||
return strconv.ParseInt(scanner.Text(), 10, 64)
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("checkpoint too short")
|
||||
}
|
||||
|
||||
// formatEntry formats a log entry as a single ~100-char line.
|
||||
func formatEntry(e *sunlight.LogEntry) string {
|
||||
certType := "cert"
|
||||
if e.IsPrecert {
|
||||
certType = "pre "
|
||||
}
|
||||
|
||||
certDER := e.Certificate
|
||||
if e.IsPrecert && len(e.PreCertificate) > 0 {
|
||||
certDER = e.PreCertificate
|
||||
}
|
||||
|
||||
name := "(unknown)"
|
||||
issuer := ""
|
||||
validRange := ""
|
||||
|
||||
// x509.ParseCertificate may return a non-nil cert even on error (e.g. precerts
|
||||
// with the CT poison critical extension), so we use the cert regardless.
|
||||
cert, _ := x509.ParseCertificate(certDER)
|
||||
if cert != nil {
|
||||
if len(cert.DNSNames) > 0 {
|
||||
name = cert.DNSNames[0]
|
||||
} else if cert.Subject.CommonName != "" {
|
||||
name = cert.Subject.CommonName
|
||||
}
|
||||
issuer = issuerLabel(cert)
|
||||
notBefore := cert.NotBefore.UTC().Format("2006-01-02")
|
||||
notAfter := cert.NotAfter.UTC().Format("2006-01-02")
|
||||
validRange = notBefore + ".." + notAfter
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%9d %-4s %-21s %-40s %s",
|
||||
e.LeafIndex, certType,
|
||||
validRange, trunc(issuer, 40), name)
|
||||
}
|
||||
|
||||
// issuerLabel builds a human-readable issuer string. When the CN is terse
|
||||
// (e.g. "R13") and the org name isn't already embedded in it, we prepend the
|
||||
// org so the result reads as "Let's Encrypt R13" instead of just "R13".
|
||||
func issuerLabel(cert *x509.Certificate) string {
|
||||
cn := cert.Issuer.CommonName
|
||||
if len(cert.Issuer.Organization) == 0 {
|
||||
return cn
|
||||
}
|
||||
org := cert.Issuer.Organization[0]
|
||||
// Only prepend if the CN doesn't already contain the org's first word.
|
||||
firstWord := strings.Fields(org)[0]
|
||||
if strings.Contains(cn, firstWord) {
|
||||
return cn
|
||||
}
|
||||
return org + " " + cn
|
||||
}
|
||||
|
||||
// trunc truncates s to at most n bytes, appending "..." if truncated.
|
||||
func trunc(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n-3] + "..."
|
||||
}
|
||||
Reference in New Issue
Block a user