certificate-transparency/ctfetch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor some utils.go; add another tool 'tiledump'

Browse Source
This commit is contained in:
Pim van Pelt
2026-01-11 07:12:48 +01:00
parent 25d07030d5
commit 1633ad52c9
5 changed files with 216 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1 +1,2 @@
/ctfetch
/tiledump

34
README.md
View File

@@ -1,20 +1,25 @@
# ctfetch
Fetch and dump leaf entries from Certificate Transparency logs.
Tools for working with Certificate Transparency log tiles.
## Install
```bash
go install ./cmd/ctfetch
go install ./cmd/tiledump
```
## Usage
## Commands
### ctfetch
Fetch and dump leaf entries from CT logs.
```bash
ctfetch [--dumpall] <log-url> <leaf-index>
```
### Examples
**Examples:**
Dump a specific entry:
```bash
@@ -26,6 +31,25 @@ Dump all entries in the tile:
ctfetch --dumpall https://halloumi2026h1.mon.ct.ipng.ch 629794635
```
## Options
**Options:**
- `--dumpall`: Dump all entries in the tile instead of just the specified leaf
### tiledump
Read a CT log tile file or URL and dump all entries.
```bash
tiledump <tile-file-or-url>
```
**Examples:**
From a file:
```bash
tiledump tile.data
```
From a URL:
```bash
tiledump https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135
```

113
cmd/ctfetch/main.go
View File

@@ -4,17 +4,14 @@
package main
import (
"bytes"
"compress/gzip"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"ctfetch/internal/utils"
"filippo.io/sunlight"
"golang.org/x/mod/sumdb/tlog"
)
@@ -66,7 +63,7 @@ func main() {
// Try partial tile first
partialURL := logURL + "/" + partialPath
fmt.Fprintf(os.Stderr, "Trying: %s\n", partialURL)
tileData, err = fetchURL(partialURL)
tileData, err = utils.FetchURL(partialURL)
if err == nil {
fetchedPath = partialPath
fmt.Fprintf(os.Stderr, "Successfully fetched partial tile\n")
@@ -74,7 +71,7 @@ func main() {
// Fall back to full tile
fullURL := logURL + "/" + fullPath
fmt.Fprintf(os.Stderr, "Partial tile failed, trying: %s\n", fullURL)
tileData, err = fetchURL(fullURL)
tileData, err = utils.FetchURL(fullURL)
if err != nil {
fatal("failed to fetch tile: %v", err)
}
@@ -83,7 +80,7 @@ func main() {
}
// Decompress if needed
tileData, err = decompress(tileData)
tileData, err = utils.Decompress(tileData)
if err != nil {
fatal("failed to decompress tile: %v", err)
}
@@ -93,107 +90,17 @@ func main() {
if *dumpAll {
// Dump all entries in the tile
dumpAllEntries(tileData)
if err := utils.DumpAllEntries(tileData); err != nil {
fatal("%v", err)
}
} else {
// Dump only the specific entry at the position
dumpEntryAtPosition(tileData, int(positionInTile), leafIndex)
}
}
func fetchURL(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
return io.ReadAll(resp.Body)
}
func dumpAllEntries(tileData []byte) {
entryNum := 0
for len(tileData) > 0 {
e, remaining, err := sunlight.ReadTileLeaf(tileData)
if err != nil {
fatal("failed to read entry %d: %v", entryNum, err)
}
tileData = remaining
dumpEntry(e, entryNum)
fmt.Println()
entryNum++
}
fmt.Printf("Total entries: %d\n", entryNum)
}
func dumpEntryAtPosition(tileData []byte, position int, expectedIndex int64) {
entryNum := 0
for len(tileData) > 0 {
e, remaining, err := sunlight.ReadTileLeaf(tileData)
if err != nil {
fatal("failed to read entry %d: %v", entryNum, err)
}
tileData = remaining
if entryNum == position {
if e.LeafIndex != expectedIndex {
fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n",
expectedIndex, e.LeafIndex, position)
}
dumpEntry(e, entryNum)
return
}
entryNum++
}
fatal("position %d not found in tile (only %d entries)", position, entryNum)
}
func dumpEntry(e *sunlight.LogEntry, entryNum int) {
fmt.Printf("=== Entry %d ===\n", entryNum)
fmt.Printf("Leaf Index: %d\n", e.LeafIndex)
fmt.Printf("Timestamp: %d\n", e.Timestamp)
fmt.Printf("Is Precert: %v\n", e.IsPrecert)
if e.IsPrecert {
fmt.Printf("Issuer Key Hash: %x\n", e.IssuerKeyHash)
}
fmt.Printf("Certificate: %d bytes\n", len(e.Certificate))
if e.PreCertificate != nil {
fmt.Printf("PreCertificate: %d bytes\n", len(e.PreCertificate))
}
fmt.Printf("Chain Fingerprints: %d entries\n", len(e.ChainFingerprints))
for i, fp := range e.ChainFingerprints {
fmt.Printf(" [%d]: %x\n", i, fp)
}
// Try to extract parsed certificate info
if trimmed, err := e.TrimmedEntry(); err == nil {
if data, err := json.MarshalIndent(trimmed, " ", " "); err == nil {
fmt.Printf("Parsed Certificate Info:\n %s\n", data)
if err := utils.DumpEntryAtPosition(tileData, int(positionInTile), leafIndex); err != nil {
fatal("%v", err)
}
}
}
const maxCompressRatio = 100
func decompress(data []byte) ([]byte, error) {
r, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
// Not gzipped, return as-is
return data, nil
}
maxSize := int64(len(data)) * maxCompressRatio
return io.ReadAll(io.LimitReader(r, maxSize))
}
func fatal(format string, args ...any) {
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...)
os.Exit(1)

62
cmd/tiledump/main.go Normal file
View File

@@ -0,0 +1,62 @@
// Command tiledump reads a CT log tile file and dumps all entries.
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
package main
import (
"fmt"
"os"
"strings"
"ctfetch/internal/utils"
)
func main() {
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s <tile-file-or-url>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Examples:\n")
fmt.Fprintf(os.Stderr, " %s tile.data\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s https://halloumi2026h1.mon.ct.ipng.ch/tile/data/x002/x460/135\n", os.Args[0])
os.Exit(1)
}
arg := os.Args[1]
var tileData []byte
var err error
// Check if argument is a URL
if strings.HasPrefix(arg, "http://") || strings.HasPrefix(arg, "https://") {
// Fetch from URL
fmt.Fprintf(os.Stderr, "Fetching: %s\n", arg)
tileData, err = utils.FetchURL(arg)
if err != nil {
fatal("failed to fetch URL: %v", err)
}
fmt.Fprintf(os.Stderr, "Fetched %d bytes\n", len(tileData))
} else {
// Read from file
tileData, err = os.ReadFile(arg)
if err != nil {
fatal("failed to read file: %v", err)
}
fmt.Fprintf(os.Stderr, "Read %d bytes from %s\n", len(tileData), arg)
}
// Decompress if needed
tileData, err = utils.Decompress(tileData)
if err != nil {
fatal("failed to decompress tile: %v", err)
}
fmt.Fprintf(os.Stderr, "Tile size: %d bytes\n\n", len(tileData))
// Dump all entries
if err := utils.DumpAllEntries(tileData); err != nil {
fatal("%v", err)
}
}
func fatal(format string, args ...any) {
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...)
os.Exit(1)
}

114
internal/utils/utils.go Normal file
View File

@@ -0,0 +1,114 @@
// Package utils provides shared functionality for dumping CT log tile entries.
// (C) Copyright 2026 Pim van Pelt <pim@ipng.ch>
package utils
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"filippo.io/sunlight"
)
const maxCompressRatio = 100
// FetchURL fetches data from a URL.
func FetchURL(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
return io.ReadAll(resp.Body)
}
// Decompress decompresses gzip-compressed data, or returns the data as-is if not compressed.
func Decompress(data []byte) ([]byte, error) {
r, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
// Not gzipped, return as-is
return data, nil
}
maxSize := int64(len(data)) * maxCompressRatio
return io.ReadAll(io.LimitReader(r, maxSize))
}
// DumpAllEntries reads and dumps all entries from tile data.
func DumpAllEntries(tileData []byte) error {
entryNum := 0
for len(tileData) > 0 {
e, remaining, err := sunlight.ReadTileLeaf(tileData)
if err != nil {
return fmt.Errorf("failed to read entry %d: %w", entryNum, err)
}
tileData = remaining
dumpEntry(e, entryNum)
fmt.Println()
entryNum++
}
fmt.Printf("Total entries: %d\n", entryNum)
return nil
}
// DumpEntryAtPosition reads and dumps a specific entry at the given position.
func DumpEntryAtPosition(tileData []byte, position int, expectedIndex int64) error {
entryNum := 0
for len(tileData) > 0 {
e, remaining, err := sunlight.ReadTileLeaf(tileData)
if err != nil {
return fmt.Errorf("failed to read entry %d: %w", entryNum, err)
}
tileData = remaining
if entryNum == position {
if e.LeafIndex != expectedIndex {
fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n",
expectedIndex, e.LeafIndex, position)
}
dumpEntry(e, entryNum)
return nil
}
entryNum++
}
return fmt.Errorf("position %d not found in tile (only %d entries)", position, entryNum)
}
func dumpEntry(e *sunlight.LogEntry, entryNum int) {
fmt.Printf("=== Entry %d ===\n", entryNum)
fmt.Printf("Leaf Index: %d\n", e.LeafIndex)
fmt.Printf("Timestamp: %d\n", e.Timestamp)
fmt.Printf("Is Precert: %v\n", e.IsPrecert)
if e.IsPrecert {
fmt.Printf("Issuer Key Hash: %x\n", e.IssuerKeyHash)
}
fmt.Printf("Certificate: %d bytes\n", len(e.Certificate))
if e.PreCertificate != nil {
fmt.Printf("PreCertificate: %d bytes\n", len(e.PreCertificate))
}
fmt.Printf("Chain Fingerprints: %d entries\n", len(e.ChainFingerprints))
for i, fp := range e.ChainFingerprints {
fmt.Printf(" [%d]: %x\n", i, fp)
}
// Try to extract parsed certificate info
if trimmed, err := e.TrimmedEntry(); err == nil {
if data, err := json.MarshalIndent(trimmed, " ", " "); err == nil {
fmt.Printf("Parsed Certificate Info:\n %s\n", data)
}
}
}