243 lines
6.5 KiB
Go
243 lines
6.5 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"crypto/x509"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"filippo.io/sunlight"
|
|
"git.ipng.ch/certificate-transparency/ctfetch/internal/utils"
|
|
"golang.org/x/mod/sumdb/tlog"
|
|
)
|
|
|
|
var (
|
|
userAgent string
|
|
rateLimit time.Duration
|
|
lastRequest time.Time
|
|
)
|
|
|
|
func runTail(args []string) {
|
|
fs := flag.NewFlagSet("tail", flag.ContinueOnError)
|
|
interval := fs.Duration("interval", 15*time.Second, "polling interval (minimum 1s)")
|
|
fromLeaf := fs.Int64("from-leaf", -1, "start from this leaf index (-1 = current tree tip)")
|
|
rateLimitFlag := fs.Duration("rate-limit", 2*time.Second, "minimum time between HTTP requests (minimum 100ms)")
|
|
fs.StringVar(&userAgent, "user-agent", "ctool/"+version+" (https://git.ipng.ch/certificate-transparency/)", "User-Agent header for HTTP requests")
|
|
fs.Usage = func() {
|
|
n := cmdName("tail")
|
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <log-url>\n", n)
|
|
fmt.Fprintf(os.Stderr, "\nPrints a one-liner per cert/pre-cert as new entries arrive in a Static CT log.\n")
|
|
fmt.Fprintf(os.Stderr, "\nExamples:\n")
|
|
fmt.Fprintf(os.Stderr, " %s https://halloumi2026h2.mon.ct.ipng.ch\n", n)
|
|
fmt.Fprintf(os.Stderr, " %s --from-leaf 0 --interval 10s https://halloumi2026h2.mon.ct.ipng.ch\n", n)
|
|
fmt.Fprintf(os.Stderr, "\nFlags:\n")
|
|
fs.PrintDefaults()
|
|
}
|
|
|
|
if err := fs.Parse(args); err != nil {
|
|
if err == flag.ErrHelp {
|
|
os.Exit(0)
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
|
|
if *interval < time.Second {
|
|
fmt.Fprintf(os.Stderr, "Error: --interval must be at least 1s\n")
|
|
os.Exit(1)
|
|
}
|
|
if *rateLimitFlag < 100*time.Millisecond {
|
|
fmt.Fprintf(os.Stderr, "Error: --rate-limit must be at least 100ms\n")
|
|
os.Exit(1)
|
|
}
|
|
rateLimit = *rateLimitFlag
|
|
|
|
if fs.NArg() != 1 {
|
|
fs.Usage()
|
|
os.Exit(1)
|
|
}
|
|
|
|
logURL := strings.TrimSuffix(fs.Arg(0), "/")
|
|
var nextLeaf int64 = -1
|
|
|
|
for {
|
|
checkpointAt := time.Now()
|
|
treeSize, err := fetchCheckpointSize(logURL)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "checkpoint error: %v\n", err)
|
|
time.Sleep(*interval)
|
|
continue
|
|
}
|
|
|
|
if nextLeaf < 0 {
|
|
if *fromLeaf >= 0 {
|
|
fmt.Fprintf(os.Stderr, "Starting from leaf %d, tree size %d, polling every %s\n", *fromLeaf, treeSize, *interval)
|
|
nextLeaf = *fromLeaf
|
|
} else {
|
|
fmt.Fprintf(os.Stderr, "Starting at tree size %d, polling every %s\n", treeSize, *interval)
|
|
nextLeaf = treeSize
|
|
}
|
|
}
|
|
|
|
if err := processNewEntries(logURL, &nextLeaf, treeSize); err != nil {
|
|
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
|
}
|
|
|
|
if wait := time.Until(checkpointAt.Add(*interval)); wait > 0 {
|
|
time.Sleep(wait)
|
|
}
|
|
}
|
|
}
|
|
|
|
func processNewEntries(logURL string, nextLeaf *int64, treeSize int64) error {
|
|
for *nextLeaf < treeSize {
|
|
tileStart := (*nextLeaf / sunlight.TileWidth) * sunlight.TileWidth
|
|
if treeSize < tileStart+sunlight.TileWidth {
|
|
break // tile not yet complete; wait for next poll
|
|
}
|
|
|
|
tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, *nextLeaf))
|
|
tile.L = -1
|
|
tilePath := sunlight.TilePath(tile)
|
|
// Always fetch the full tile; strip any .p/N partial suffix.
|
|
if idx := strings.Index(tilePath, ".p/"); idx != -1 {
|
|
tilePath = tilePath[:idx]
|
|
}
|
|
|
|
tileData, err := fetchURL(logURL + "/" + tilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("fetch tile at leaf %d: %w", *nextLeaf, err)
|
|
}
|
|
tileData, err = utils.Decompress(tileData)
|
|
if err != nil {
|
|
return fmt.Errorf("decompress tile: %w", err)
|
|
}
|
|
|
|
prevNext := *nextLeaf
|
|
remaining := tileData
|
|
for len(remaining) > 0 {
|
|
e, rest, err := sunlight.ReadTileLeaf(remaining)
|
|
if err != nil {
|
|
return fmt.Errorf("read tile leaf: %w", err)
|
|
}
|
|
remaining = rest
|
|
if e.LeafIndex >= *nextLeaf {
|
|
fmt.Println(formatEntry(e))
|
|
*nextLeaf = e.LeafIndex + 1
|
|
}
|
|
}
|
|
if *nextLeaf == prevNext {
|
|
return fmt.Errorf("no progress reading tile at leaf %d", *nextLeaf)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fetchURL(url string) ([]byte, error) {
|
|
if wait := time.Until(lastRequest.Add(rateLimit)); wait > 0 {
|
|
time.Sleep(wait)
|
|
}
|
|
lastRequest = time.Now()
|
|
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
|
}
|
|
return io.ReadAll(resp.Body)
|
|
}
|
|
|
|
func fetchCheckpointSize(logURL string) (int64, error) {
|
|
data, err := fetchURL(logURL + "/checkpoint")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// Checkpoint format (tlog/sunlight):
|
|
// line 1: origin
|
|
// line 2: tree size (decimal)
|
|
// line 3: root hash (base64)
|
|
// (blank line + signature lines)
|
|
scanner := bufio.NewScanner(bytes.NewReader(data))
|
|
for i := 0; scanner.Scan(); i++ {
|
|
if i == 1 {
|
|
return strconv.ParseInt(scanner.Text(), 10, 64)
|
|
}
|
|
}
|
|
return 0, fmt.Errorf("checkpoint too short")
|
|
}
|
|
|
|
// formatEntry formats a log entry as a single ~100-char line.
|
|
func formatEntry(e *sunlight.LogEntry) string {
|
|
certType := "cert"
|
|
if e.IsPrecert {
|
|
certType = "pre "
|
|
}
|
|
|
|
certDER := e.Certificate
|
|
if e.IsPrecert && len(e.PreCertificate) > 0 {
|
|
certDER = e.PreCertificate
|
|
}
|
|
|
|
name := "(unknown)"
|
|
issuer := ""
|
|
validRange := ""
|
|
|
|
// x509.ParseCertificate may return a non-nil cert even on error (e.g. precerts
|
|
// with the CT poison critical extension), so we use the cert regardless.
|
|
cert, _ := x509.ParseCertificate(certDER)
|
|
if cert != nil {
|
|
if len(cert.DNSNames) > 0 {
|
|
name = cert.DNSNames[0]
|
|
} else if cert.Subject.CommonName != "" {
|
|
name = cert.Subject.CommonName
|
|
}
|
|
issuer = issuerLabel(cert)
|
|
notBefore := cert.NotBefore.UTC().Format("2006-01-02")
|
|
notAfter := cert.NotAfter.UTC().Format("2006-01-02")
|
|
validRange = notBefore + ".." + notAfter
|
|
}
|
|
|
|
return fmt.Sprintf("%9d %-4s %-21s %-40s %s",
|
|
e.LeafIndex, certType,
|
|
validRange, trunc(issuer, 40), name)
|
|
}
|
|
|
|
// issuerLabel builds a human-readable issuer string. When the CN is terse
|
|
// (e.g. "R13") and the org name isn't already embedded in it, we prepend the
|
|
// org so the result reads as "Let's Encrypt R13" instead of just "R13".
|
|
func issuerLabel(cert *x509.Certificate) string {
|
|
cn := cert.Issuer.CommonName
|
|
if len(cert.Issuer.Organization) == 0 {
|
|
return cn
|
|
}
|
|
org := cert.Issuer.Organization[0]
|
|
// Only prepend if the CN doesn't already contain the org's first word.
|
|
firstWord := strings.Fields(org)[0]
|
|
if strings.Contains(cn, firstWord) {
|
|
return cn
|
|
}
|
|
return org + " " + cn
|
|
}
|
|
|
|
// trunc truncates s to at most n bytes, appending "..." if truncated.
|
|
func trunc(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n-3] + "..."
|
|
}
|