From 1633ad52c94737cc0de02cca3697de3c3a62ebbf Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Sun, 11 Jan 2026 07:12:48 +0100 Subject: [PATCH] Refactor some utils.go; add another tool 'tiledump' --- .gitignore | 1 + README.md | 34 ++++++++++-- cmd/ctfetch/main.go | 113 ++++----------------------------------- cmd/tiledump/main.go | 62 ++++++++++++++++++++++ internal/utils/utils.go | 114 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 108 deletions(-) create mode 100644 cmd/tiledump/main.go create mode 100644 internal/utils/utils.go diff --git a/.gitignore b/.gitignore index 66a17d6..9f89c00 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /ctfetch +/tiledump diff --git a/README.md b/README.md index 712e992..e3d3b68 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,25 @@ # ctfetch -Fetch and dump leaf entries from Certificate Transparency logs. +Tools for working with Certificate Transparency log tiles. ## Install ```bash go install ./cmd/ctfetch +go install ./cmd/tiledump ``` -## Usage +## Commands + +### ctfetch + +Fetch and dump leaf entries from CT logs. ```bash ctfetch [--dumpall] ``` -### Examples +**Examples:** Dump a specific entry: ```bash @@ -26,6 +31,25 @@ Dump all entries in the tile: ctfetch --dumpall https://halloumi2026h1.mon.ct.ipng.ch 629794635 ``` -## Options - +**Options:** - `--dumpall`: Dump all entries in the tile instead of just the specified leaf + +### tiledump + +Read a CT log tile file or URL and dump all entries. + +```bash +tiledump +``` + +**Examples:** + +From a file: +```bash +tiledump tile.data +``` + +From a URL: +```bash +tiledump https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135 +``` diff --git a/cmd/ctfetch/main.go b/cmd/ctfetch/main.go index cfb7d92..4a9a624 100644 --- a/cmd/ctfetch/main.go +++ b/cmd/ctfetch/main.go @@ -4,17 +4,14 @@ package main import ( - "bytes" - "compress/gzip" - "encoding/json" "flag" "fmt" - "io" - "net/http" "os" "strconv" "strings" + "ctfetch/internal/utils" + "filippo.io/sunlight" "golang.org/x/mod/sumdb/tlog" ) @@ -66,7 +63,7 @@ func main() { // Try partial tile first partialURL := logURL + "/" + partialPath fmt.Fprintf(os.Stderr, "Trying: %s\n", partialURL) - tileData, err = fetchURL(partialURL) + tileData, err = utils.FetchURL(partialURL) if err == nil { fetchedPath = partialPath fmt.Fprintf(os.Stderr, "Successfully fetched partial tile\n") @@ -74,7 +71,7 @@ func main() { // Fall back to full tile fullURL := logURL + "/" + fullPath fmt.Fprintf(os.Stderr, "Partial tile failed, trying: %s\n", fullURL) - tileData, err = fetchURL(fullURL) + tileData, err = utils.FetchURL(fullURL) if err != nil { fatal("failed to fetch tile: %v", err) } @@ -83,7 +80,7 @@ func main() { } // Decompress if needed - tileData, err = decompress(tileData) + tileData, err = utils.Decompress(tileData) if err != nil { fatal("failed to decompress tile: %v", err) } @@ -93,107 +90,17 @@ func main() { if *dumpAll { // Dump all entries in the tile - dumpAllEntries(tileData) + if err := utils.DumpAllEntries(tileData); err != nil { + fatal("%v", err) + } } else { // Dump only the specific entry at the position - dumpEntryAtPosition(tileData, int(positionInTile), leafIndex) - } -} - -func fetchURL(url string) ([]byte, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP %d", resp.StatusCode) - } - - return io.ReadAll(resp.Body) -} - -func dumpAllEntries(tileData []byte) { - entryNum := 0 - for len(tileData) > 0 { - e, remaining, err := sunlight.ReadTileLeaf(tileData) - if err != nil { - fatal("failed to read entry %d: %v", entryNum, err) - } - tileData = remaining - - dumpEntry(e, entryNum) - fmt.Println() - entryNum++ - } - - fmt.Printf("Total entries: %d\n", entryNum) -} - -func dumpEntryAtPosition(tileData []byte, position int, expectedIndex int64) { - entryNum := 0 - for len(tileData) > 0 { - e, remaining, err := sunlight.ReadTileLeaf(tileData) - if err != nil { - fatal("failed to read entry %d: %v", entryNum, err) - } - tileData = remaining - - if entryNum == position { - if e.LeafIndex != expectedIndex { - fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n", - expectedIndex, e.LeafIndex, position) - } - dumpEntry(e, entryNum) - return - } - entryNum++ - } - - fatal("position %d not found in tile (only %d entries)", position, entryNum) -} - -func dumpEntry(e *sunlight.LogEntry, entryNum int) { - fmt.Printf("=== Entry %d ===\n", entryNum) - fmt.Printf("Leaf Index: %d\n", e.LeafIndex) - fmt.Printf("Timestamp: %d\n", e.Timestamp) - fmt.Printf("Is Precert: %v\n", e.IsPrecert) - - if e.IsPrecert { - fmt.Printf("Issuer Key Hash: %x\n", e.IssuerKeyHash) - } - - fmt.Printf("Certificate: %d bytes\n", len(e.Certificate)) - if e.PreCertificate != nil { - fmt.Printf("PreCertificate: %d bytes\n", len(e.PreCertificate)) - } - - fmt.Printf("Chain Fingerprints: %d entries\n", len(e.ChainFingerprints)) - for i, fp := range e.ChainFingerprints { - fmt.Printf(" [%d]: %x\n", i, fp) - } - - // Try to extract parsed certificate info - if trimmed, err := e.TrimmedEntry(); err == nil { - if data, err := json.MarshalIndent(trimmed, " ", " "); err == nil { - fmt.Printf("Parsed Certificate Info:\n %s\n", data) + if err := utils.DumpEntryAtPosition(tileData, int(positionInTile), leafIndex); err != nil { + fatal("%v", err) } } } -const maxCompressRatio = 100 - -func decompress(data []byte) ([]byte, error) { - r, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - // Not gzipped, return as-is - return data, nil - } - maxSize := int64(len(data)) * maxCompressRatio - return io.ReadAll(io.LimitReader(r, maxSize)) -} - func fatal(format string, args ...any) { fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...) os.Exit(1) diff --git a/cmd/tiledump/main.go b/cmd/tiledump/main.go new file mode 100644 index 0000000..419bd91 --- /dev/null +++ b/cmd/tiledump/main.go @@ -0,0 +1,62 @@ +// Command tiledump reads a CT log tile file and dumps all entries. +// (C) Copyright 2026 Pim van Pelt +package main + +import ( + "fmt" + "os" + "strings" + + "ctfetch/internal/utils" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Examples:\n") + fmt.Fprintf(os.Stderr, " %s tile.data\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135\n", os.Args[0]) + os.Exit(1) + } + + arg := os.Args[1] + + var tileData []byte + var err error + + // Check if argument is a URL + if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") { + // Fetch from URL + fmt.Fprintf(os.Stderr, "Fetching: %s\n", arg) + tileData, err = utils.FetchURL(arg) + if err != nil { + fatal("failed to fetch URL: %v", err) + } + fmt.Fprintf(os.Stderr, "Fetched %d bytes\n", len(tileData)) + } else { + // Read from file + tileData, err = os.ReadFile(arg) + if err != nil { + fatal("failed to read file: %v", err) + } + fmt.Fprintf(os.Stderr, "Read %d bytes from %s\n", len(tileData), arg) + } + + // Decompress if needed + tileData, err = utils.Decompress(tileData) + if err != nil { + fatal("failed to decompress tile: %v", err) + } + + fmt.Fprintf(os.Stderr, "Tile size: %d bytes\n\n", len(tileData)) + + // Dump all entries + if err := utils.DumpAllEntries(tileData); err != nil { + fatal("%v", err) + } +} + +func fatal(format string, args ...any) { + fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...) + os.Exit(1) +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..5263f05 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,114 @@ +// Package utils provides shared functionality for dumping CT log tile entries. +// (C) Copyright 2026 Pim van Pelt +package utils + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + + "filippo.io/sunlight" +) + +const maxCompressRatio = 100 + +// FetchURL fetches data from a URL. +func FetchURL(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +// Decompress decompresses gzip-compressed data, or returns the data as-is if not compressed. +func Decompress(data []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + // Not gzipped, return as-is + return data, nil + } + maxSize := int64(len(data)) * maxCompressRatio + return io.ReadAll(io.LimitReader(r, maxSize)) +} + +// DumpAllEntries reads and dumps all entries from tile data. +func DumpAllEntries(tileData []byte) error { + entryNum := 0 + for len(tileData) > 0 { + e, remaining, err := sunlight.ReadTileLeaf(tileData) + if err != nil { + return fmt.Errorf("failed to read entry %d: %w", entryNum, err) + } + tileData = remaining + + dumpEntry(e, entryNum) + fmt.Println() + entryNum++ + } + + fmt.Printf("Total entries: %d\n", entryNum) + return nil +} + +// DumpEntryAtPosition reads and dumps a specific entry at the given position. +func DumpEntryAtPosition(tileData []byte, position int, expectedIndex int64) error { + entryNum := 0 + for len(tileData) > 0 { + e, remaining, err := sunlight.ReadTileLeaf(tileData) + if err != nil { + return fmt.Errorf("failed to read entry %d: %w", entryNum, err) + } + tileData = remaining + + if entryNum == position { + if e.LeafIndex != expectedIndex { + fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n", + expectedIndex, e.LeafIndex, position) + } + dumpEntry(e, entryNum) + return nil + } + entryNum++ + } + + return fmt.Errorf("position %d not found in tile (only %d entries)", position, entryNum) +} + +func dumpEntry(e *sunlight.LogEntry, entryNum int) { + fmt.Printf("=== Entry %d ===\n", entryNum) + fmt.Printf("Leaf Index: %d\n", e.LeafIndex) + fmt.Printf("Timestamp: %d\n", e.Timestamp) + fmt.Printf("Is Precert: %v\n", e.IsPrecert) + + if e.IsPrecert { + fmt.Printf("Issuer Key Hash: %x\n", e.IssuerKeyHash) + } + + fmt.Printf("Certificate: %d bytes\n", len(e.Certificate)) + if e.PreCertificate != nil { + fmt.Printf("PreCertificate: %d bytes\n", len(e.PreCertificate)) + } + + fmt.Printf("Chain Fingerprints: %d entries\n", len(e.ChainFingerprints)) + for i, fp := range e.ChainFingerprints { + fmt.Printf(" [%d]: %x\n", i, fp) + } + + // Try to extract parsed certificate info + if trimmed, err := e.TrimmedEntry(); err == nil { + if data, err := json.MarshalIndent(trimmed, " ", " "); err == nil { + fmt.Printf("Parsed Certificate Info:\n %s\n", data) + } + } +}