// Package utils provides shared functionality for dumping CT log tile entries. // (C) Copyright 2026 Pim van Pelt package utils import ( "bytes" "compress/gzip" "crypto/sha256" "crypto/x509" "encoding/asn1" "encoding/base64" "encoding/binary" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "os" "strings" "sync" "time" "filippo.io/sunlight" ) var ( ctLogCache = map[string]map[string]CTLogInfo{} ctLogCacheMu sync.Mutex issuerCache = map[string]*IssuerInfo{} issuerCacheMu sync.Mutex ) var ( oidSCTList = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 4, 2} oidCTPoison = asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 11129, 2, 4, 3} ) // CTLogInfo holds details about a CT log from the log list. type CTLogInfo struct { Description string `json:"description"` URL string `json:"url"` Operator string `json:"operator"` State string `json:"state"` } // SCT represents a Signed Certificate Timestamp. type SCT struct { Version int `json:"version"` LogID string `json:"log_id"` Timestamp int64 `json:"timestamp"` TimestampHuman string `json:"timestamp_human"` Extensions string `json:"extensions,omitempty"` HashAlgorithm int `json:"hash_algorithm"` SigAlgorithm int `json:"sig_algorithm"` Signature string `json:"signature"` CTLog *CTLogInfo `json:"ctlog,omitempty"` } const maxCompressRatio = 100 // Options controls which optional fields are fetched and included in output. type Options struct { LogURL string ShowSCT bool ShowIssuer bool ShowCTLog bool CTLogs map[string]CTLogInfo // keyed by hex log_id } // IssuerInfo holds parsed details of an issuer certificate fetched from the log. type IssuerInfo struct { Fingerprint string `json:"fingerprint"` Subject string `json:"subject"` Issuer string `json:"issuer"` NotBefore string `json:"not_before"` NotAfter string `json:"not_after"` SerialNumber string `json:"serial_number"` } // CertDetails holds fields parsed from the certificate beyond what TrimmedEntry provides. type CertDetails struct { NotBefore string `json:"not_before"` NotAfter string `json:"not_after"` SerialNumber string `json:"serial_number"` Issuer string `json:"issuer"` EmailSANs []string `json:"email_sans,omitempty"` URISANs []string `json:"uri_sans,omitempty"` SubjectKeyID string `json:"subject_key_id,omitempty"` AuthorityKeyID string `json:"authority_key_id,omitempty"` OCSPServers []string `json:"ocsp_servers,omitempty"` IssuingCertURLs []string `json:"issuing_cert_urls,omitempty"` CRLDistributionPoints []string `json:"crl_distribution_points,omitempty"` KeyUsage []string `json:"key_usage,omitempty"` ExtKeyUsage []string `json:"ext_key_usage,omitempty"` IsCA bool `json:"is_ca"` PoisonExtension bool `json:"poison_extension,omitempty"` } var keyUsageNames = []struct { usage x509.KeyUsage name string }{ {x509.KeyUsageDigitalSignature, "DigitalSignature"}, {x509.KeyUsageContentCommitment, "ContentCommitment"}, {x509.KeyUsageKeyEncipherment, "KeyEncipherment"}, {x509.KeyUsageDataEncipherment, "DataEncipherment"}, {x509.KeyUsageKeyAgreement, "KeyAgreement"}, {x509.KeyUsageCertSign, "CertSign"}, {x509.KeyUsageCRLSign, "CRLSign"}, {x509.KeyUsageEncipherOnly, "EncipherOnly"}, {x509.KeyUsageDecipherOnly, "DecipherOnly"}, } var extKeyUsageNames = map[x509.ExtKeyUsage]string{ x509.ExtKeyUsageAny: "Any", x509.ExtKeyUsageServerAuth: "ServerAuth", x509.ExtKeyUsageClientAuth: "ClientAuth", x509.ExtKeyUsageCodeSigning: "CodeSigning", x509.ExtKeyUsageEmailProtection: "EmailProtection", x509.ExtKeyUsageIPSECEndSystem: "IPSECEndSystem", x509.ExtKeyUsageIPSECTunnel: "IPSECTunnel", x509.ExtKeyUsageIPSECUser: "IPSECUser", x509.ExtKeyUsageTimeStamping: "TimeStamping", x509.ExtKeyUsageOCSPSigning: "OCSPSigning", x509.ExtKeyUsageMicrosoftServerGatedCrypto: "MicrosoftServerGatedCrypto", x509.ExtKeyUsageNetscapeServerGatedCrypto: "NetscapeServerGatedCrypto", x509.ExtKeyUsageMicrosoftCommercialCodeSigning: "MicrosoftCommercialCodeSigning", x509.ExtKeyUsageMicrosoftKernelCodeSigning: "MicrosoftKernelCodeSigning", } // Entry represents a CT log entry in JSON format. type Entry struct { EntryNumber int `json:"entry_number"` LeafIndex int64 `json:"leaf_index"` MerkleLeafHash string `json:"merkle_leaf_hash"` Timestamp int64 `json:"timestamp"` TimestampHuman string `json:"timestamp_human"` IsPrecert bool `json:"is_precert"` IssuerKeyHash string `json:"issuer_key_hash,omitempty"` CertificateSize int `json:"certificate_size"` PreCertificateSize *int `json:"precertificate_size,omitempty"` ChainFingerprints []string `json:"chain_fingerprints"` Issuers []IssuerInfo `json:"issuers,omitempty"` SCTs []SCT `json:"scts,omitempty"` CertDetails *CertDetails `json:"cert_details,omitempty"` ParsedCertInfo json.RawMessage `json:"parsed_cert_info,omitempty"` } // HashTileOutput represents a hash tile in JSON format. type HashTileOutput struct { NumHashes int `json:"num_hashes"` Hashes []string `json:"hashes"` } // DumpResult is the result of dumping entries or hashes from a tile. type DumpResult struct { Entries []Entry `json:"entries,omitempty"` HashTile *HashTileOutput `json:"hash_tile,omitempty"` TotalEntries int `json:"total_entries,omitempty"` } // FetchURL fetches data from a URL. func FetchURL(url string) ([]byte, error) { fmt.Fprintf(os.Stderr, "Fetching: %s\n", url) resp, err := http.Get(url) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP %d", resp.StatusCode) } return io.ReadAll(resp.Body) } // FetchTile fetches a tile from a URL, falling back from partial to full tile on 404. func FetchTile(url string) ([]byte, error) { data, err := FetchURL(url) if err == nil { return data, nil } // On 404, try stripping the partial-tile suffix (.p/NNN) if err.Error() == "HTTP 404" { if idx := strings.Index(url, ".p/"); idx != -1 { return FetchURL(url[:idx]) } } return nil, err } // Decompress decompresses gzip-compressed data, or returns the data as-is if not compressed. func Decompress(data []byte) ([]byte, error) { r, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { // Not gzipped, return as-is return data, nil } maxSize := int64(len(data)) * maxCompressRatio return io.ReadAll(io.LimitReader(r, maxSize)) } // DumpAllEntries reads and returns all entries from tile data as JSON-serializable structures. // Automatically detects if the tile is a data tile or hash tile. func DumpAllEntries(tileData []byte, opts Options) (*DumpResult, error) { // Try to read as data tile first result, err := dumpDataTile(tileData, opts) if err != nil { // If it fails, try as hash tile fmt.Fprintf(os.Stderr, "Not a data tile, trying as hash tile...\n") return dumpHashTile(tileData, opts) } return result, nil } func dumpDataTile(tileData []byte, opts Options) (*DumpResult, error) { entryNum := 0 var entries []Entry for len(tileData) > 0 { e, remaining, err := sunlight.ReadTileLeaf(tileData) if err != nil { return nil, fmt.Errorf("failed to read entry %d: %w", entryNum, err) } tileData = remaining entry := convertEntry(e, entryNum, opts) entries = append(entries, entry) entryNum++ } return &DumpResult{ Entries: entries, TotalEntries: entryNum, }, nil } func dumpHashTile(tileData []byte, opts Options) (*DumpResult, error) { if opts.ShowSCT || opts.ShowIssuer || opts.ShowCTLog { return nil, fmt.Errorf("+sct, +issuer, and +ctlog are not valid for hash tiles (only data tiles contain certificates)") } const hashSize = 32 // SHA-256 hash size if len(tileData)%hashSize != 0 { return nil, fmt.Errorf("invalid hash tile: size %d is not a multiple of %d", len(tileData), hashSize) } numHashes := len(tileData) / hashSize hashes := make([]string, numHashes) for i := 0; i < numHashes; i++ { hash := tileData[i*hashSize : (i+1)*hashSize] hashes[i] = hex.EncodeToString(hash) } return &DumpResult{ HashTile: &HashTileOutput{ NumHashes: numHashes, Hashes: hashes, }, }, nil } // DumpEntryAtPosition reads and returns a specific entry at the given position. func DumpEntryAtPosition(tileData []byte, position int, expectedIndex int64, opts Options) (*Entry, error) { entryNum := 0 for len(tileData) > 0 { e, remaining, err := sunlight.ReadTileLeaf(tileData) if err != nil { return nil, fmt.Errorf("failed to read entry %d: %w", entryNum, err) } tileData = remaining if entryNum == position { if e.LeafIndex != expectedIndex { fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n", expectedIndex, e.LeafIndex, position) } entry := convertEntry(e, entryNum, opts) return &entry, nil } entryNum++ } return nil, fmt.Errorf("position %d not found in tile (only %d entries)", position, entryNum) } // parseEmbeddedSCTs extracts SCTs from the SCT list extension of a DER-encoded certificate. func parseEmbeddedSCTs(certDER []byte) ([]SCT, error) { cert, err := x509.ParseCertificate(certDER) if err != nil { return nil, fmt.Errorf("parse certificate: %w", err) } for _, ext := range cert.Extensions { if !ext.Id.Equal(oidSCTList) { continue } // ext.Value is the DER encoding of the extension value, which is an OCTET STRING // wrapping the TLS-encoded SCTList. var inner []byte if rest, err := asn1.Unmarshal(ext.Value, &inner); err != nil || len(rest) != 0 { return nil, fmt.Errorf("unmarshal SCT extension: %w", err) } return parseSCTList(inner) } return nil, nil } // parseSCTList parses a TLS-encoded SignedCertificateTimestampList. func parseSCTList(data []byte) ([]SCT, error) { if len(data) < 2 { return nil, fmt.Errorf("SCT list too short") } listLen := int(binary.BigEndian.Uint16(data[:2])) data = data[2:] if len(data) < listLen { return nil, fmt.Errorf("SCT list truncated") } data = data[:listLen] var scts []SCT for len(data) > 0 { if len(data) < 2 { return nil, fmt.Errorf("SCT entry length truncated") } sctLen := int(binary.BigEndian.Uint16(data[:2])) data = data[2:] if len(data) < sctLen { return nil, fmt.Errorf("SCT entry truncated") } sct, err := parseSCT(data[:sctLen]) if err != nil { return nil, err } scts = append(scts, sct) data = data[sctLen:] } return scts, nil } // parseSCT parses a single v1 SCT from raw bytes. func parseSCT(data []byte) (SCT, error) { // version(1) + log_id(32) + timestamp(8) + ext_len(2) = 43 bytes minimum if len(data) < 43 { return SCT{}, fmt.Errorf("SCT too short: %d bytes", len(data)) } version := int(data[0]) logID := hex.EncodeToString(data[1:33]) ts := int64(binary.BigEndian.Uint64(data[33:41])) extLen := int(binary.BigEndian.Uint16(data[41:43])) pos := 43 if len(data) < pos+extLen+4 { return SCT{}, fmt.Errorf("SCT extensions/signature truncated") } extensions := "" if extLen > 0 { extensions = hex.EncodeToString(data[pos : pos+extLen]) } pos += extLen hashAlg := int(data[pos]) sigAlg := int(data[pos+1]) sigLen := int(binary.BigEndian.Uint16(data[pos+2 : pos+4])) pos += 4 if len(data) < pos+sigLen { return SCT{}, fmt.Errorf("SCT signature truncated") } sig := hex.EncodeToString(data[pos : pos+sigLen]) return SCT{ Version: version, LogID: logID, Timestamp: ts, TimestampHuman: time.UnixMilli(ts).UTC().Format(time.RFC3339), Extensions: extensions, HashAlgorithm: hashAlg, SigAlgorithm: sigAlg, Signature: sig, }, nil } // FetchCTLogList fetches the CT log list JSON and returns a map keyed by hex log_id. // Results are cached by URL so the network is only hit once per process. func FetchCTLogList(url string) (map[string]CTLogInfo, error) { ctLogCacheMu.Lock() if cached, ok := ctLogCache[url]; ok { ctLogCacheMu.Unlock() return cached, nil } ctLogCacheMu.Unlock() data, err := FetchURL(url) if err != nil { return nil, err } var list struct { Operators []struct { Name string `json:"name"` Logs []struct { Description string `json:"description"` LogID string `json:"log_id"` URL string `json:"url"` State map[string]json.RawMessage `json:"state"` } `json:"logs"` } `json:"operators"` } if err := json.Unmarshal(data, &list); err != nil { return nil, fmt.Errorf("parse log list: %w", err) } result := make(map[string]CTLogInfo) for _, op := range list.Operators { for _, log := range op.Logs { raw, err := base64.StdEncoding.DecodeString(log.LogID) if err != nil { continue } hexID := hex.EncodeToString(raw) state := "" for k := range log.State { state = k break } result[hexID] = CTLogInfo{ Description: log.Description, URL: log.URL, Operator: op.Name, State: state, } } } ctLogCacheMu.Lock() ctLogCache[url] = result ctLogCacheMu.Unlock() return result, nil } // fetchIssuer fetches the issuer certificate at /issuer/ and returns parsed info. // Results are cached by URL so the same issuer is only fetched once per process. func fetchIssuer(logURL, fingerprint string) (*IssuerInfo, error) { url := logURL + "/issuer/" + fingerprint issuerCacheMu.Lock() if cached, ok := issuerCache[url]; ok { issuerCacheMu.Unlock() return cached, nil } issuerCacheMu.Unlock() data, err := FetchURL(url) if err != nil { return nil, err } cert, err := x509.ParseCertificate(data) if err != nil { return nil, fmt.Errorf("parse issuer cert: %w", err) } info := &IssuerInfo{ Fingerprint: fingerprint, Subject: cert.Subject.String(), Issuer: cert.Issuer.String(), NotBefore: cert.NotBefore.UTC().Format(time.RFC3339), NotAfter: cert.NotAfter.UTC().Format(time.RFC3339), SerialNumber: cert.SerialNumber.String(), } issuerCacheMu.Lock() issuerCache[url] = info issuerCacheMu.Unlock() return info, nil } // parseCertDetails extracts certificate fields not covered by TrimmedEntry. func parseCertDetails(certDER []byte) *CertDetails { // Proceed as long as a cert was returned, even if there are unhandled // critical extensions (e.g. the CT poison extension in precertificates). cert, _ := x509.ParseCertificate(certDER) if cert == nil { return nil } d := &CertDetails{ NotBefore: cert.NotBefore.UTC().Format(time.RFC3339), NotAfter: cert.NotAfter.UTC().Format(time.RFC3339), SerialNumber: cert.SerialNumber.String(), Issuer: cert.Issuer.String(), OCSPServers: cert.OCSPServer, IssuingCertURLs: cert.IssuingCertificateURL, CRLDistributionPoints: cert.CRLDistributionPoints, IsCA: cert.IsCA, } for _, e := range cert.EmailAddresses { d.EmailSANs = append(d.EmailSANs, e) } for _, u := range cert.URIs { d.URISANs = append(d.URISANs, u.String()) } if len(cert.SubjectKeyId) > 0 { d.SubjectKeyID = hex.EncodeToString(cert.SubjectKeyId) } if len(cert.AuthorityKeyId) > 0 { d.AuthorityKeyID = hex.EncodeToString(cert.AuthorityKeyId) } for _, ku := range keyUsageNames { if cert.KeyUsage&ku.usage != 0 { d.KeyUsage = append(d.KeyUsage, ku.name) } } for _, eku := range cert.ExtKeyUsage { if name, ok := extKeyUsageNames[eku]; ok { d.ExtKeyUsage = append(d.ExtKeyUsage, name) } } for _, ext := range cert.Extensions { if ext.Id.Equal(oidCTPoison) { d.PoisonExtension = true break } } return d } // merkleLeafHash computes the RFC 6962 Merkle leaf hash: SHA-256(0x00 || leaf). func merkleLeafHash(leaf []byte) string { h := sha256.New() h.Write([]byte{0x00}) h.Write(leaf) return hex.EncodeToString(h.Sum(nil)) } func convertEntry(e *sunlight.LogEntry, entryNum int, opts Options) Entry { entry := Entry{ EntryNumber: entryNum, LeafIndex: e.LeafIndex, MerkleLeafHash: merkleLeafHash(e.MerkleTreeLeaf()), Timestamp: e.Timestamp, TimestampHuman: time.UnixMilli(e.Timestamp).UTC().Format(time.RFC3339), IsPrecert: e.IsPrecert, CertificateSize: len(e.Certificate), } if e.IsPrecert { entry.IssuerKeyHash = hex.EncodeToString(e.IssuerKeyHash[:]) } if e.PreCertificate != nil { size := len(e.PreCertificate) entry.PreCertificateSize = &size } // Convert chain fingerprints to hex strings and optionally fetch issuer details. entry.ChainFingerprints = make([]string, len(e.ChainFingerprints)) for i, fp := range e.ChainFingerprints { entry.ChainFingerprints[i] = hex.EncodeToString(fp[:]) } if opts.ShowIssuer && opts.LogURL != "" { for _, fp := range entry.ChainFingerprints { info, err := fetchIssuer(opts.LogURL, fp) if err != nil { fmt.Fprintf(os.Stderr, "WARNING: could not fetch issuer %s: %v\n", fp, err) continue } entry.Issuers = append(entry.Issuers, *info) } } // Optionally extract embedded SCTs from final (non-precert) certificates. if opts.ShowSCT && !e.IsPrecert && len(e.Certificate) > 0 { if scts, err := parseEmbeddedSCTs(e.Certificate); err == nil { if opts.ShowCTLog { for i := range scts { if info, ok := opts.CTLogs[scts[i].LogID]; ok { scts[i].CTLog = &info } } } entry.SCTs = scts } } // Parse extended certificate details. certDER := e.Certificate if e.IsPrecert { certDER = e.PreCertificate } if len(certDER) > 0 { entry.CertDetails = parseCertDetails(certDER) } // Try to extract parsed certificate info if trimmed, err := e.TrimmedEntry(); err == nil { if data, err := json.Marshal(trimmed); err == nil { entry.ParsedCertInfo = data } } return entry }