From f2c55164516090a1ab145c77edd5c20b8c88f65a Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Sun, 11 Jan 2026 06:53:39 +0100 Subject: [PATCH] Initial checkin --- .gitignore | 1 + LICENSE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 31 ++++++++ go.mod | 15 ++++ go.sum | 12 ++++ main.go | 200 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6c85d8c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +ctfetch diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ffdc147 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026, IPng Networks GmbH, Pim van Pelt + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..38bf853 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# ctfetch + +Fetch and dump leaf entries from Certificate Transparency logs. + +## Install + +```bash +go install +``` + +## Usage + +```bash +ctfetch [--dumpall] +``` + +### Examples + +Dump a specific entry: +```bash +ctfetch https://halloumi2026h1.mon.ct.ipng.ch 629794635 +``` + +Dump all entries in the tile: +```bash +ctfetch --dumpall https://halloumi2026h1.mon.ct.ipng.ch 629794635 +``` + +## Options + +- `--dumpall`: Dump all entries in the tile instead of just the specified leaf diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..db1356f --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module ctfetch + +go 1.24.6 + +require ( + filippo.io/sunlight v0.6.3 + golang.org/x/mod v0.32.0 +) + +require ( + filippo.io/torchwood v0.5.1-0.20250713221105-b067ac9d4cf6 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/sync v0.15.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1f83b4b --- /dev/null +++ b/go.sum @@ -0,0 +1,12 @@ +filippo.io/sunlight v0.6.3 h1:bxN0JLUoW2rV8jNluRWKpj+9ZpWutBfoiWgM6FGlTMw= +filippo.io/sunlight v0.6.3/go.mod h1:1wUWZmC0tYtzP0PC2rsegshLsLYZ6sgFSe4Utj33Tyg= +filippo.io/torchwood v0.5.1-0.20250713221105-b067ac9d4cf6 h1:feb1i6byodl8n5WEJJ1fafcP3eVBiiVloh3mYvnRmJY= +filippo.io/torchwood v0.5.1-0.20250713221105-b067ac9d4cf6/go.mod h1:Z+iz3Syg0RCaVkL9nBjG2STp/9HpuFl1+SbaNSZ/Ez8= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..cfb7d92 --- /dev/null +++ b/main.go @@ -0,0 +1,200 @@ +// Command ctfetch fetches and dumps a specific leaf entry from a given Static CT log. +// It can also dump the whole contents of the tile, if the -dumpall flag is specified. +// (C) Copyright 2026 Pim van Pelt +package main + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + + "filippo.io/sunlight" + "golang.org/x/mod/sumdb/tlog" +) + +func main() { + dumpAll := flag.Bool("dumpall", false, "dump all entries in the tile") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [--dumpall] \n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Example: %s https://halloumi2026h1.mon.ct.ipng.ch 457683896\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\nFlags:\n") + flag.PrintDefaults() + } + flag.Parse() + + if flag.NArg() != 2 { + flag.Usage() + os.Exit(1) + } + + logURL := strings.TrimSuffix(flag.Arg(0), "/") + leafIndex, err := strconv.ParseInt(flag.Arg(1), 10, 64) + if err != nil { + fatal("invalid leaf index: %v", err) + } + + // Convert leaf index to tile coordinates + tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, leafIndex)) + tile.L = -1 // Data tiles are at level -1 + + // Get the tile path (both partial and full versions) + partialPath := sunlight.TilePath(tile) + + // For full tile path, we need to remove the .p/W suffix if present + fullTile := tile + fullTile.W = sunlight.TileWidth + fullPath := sunlight.TilePath(fullTile) + + positionInTile := leafIndex % sunlight.TileWidth + + fmt.Fprintf(os.Stderr, "Leaf Index: %d\n", leafIndex) + fmt.Fprintf(os.Stderr, "Position in tile: %d\n", positionInTile) + fmt.Fprintf(os.Stderr, "Partial tile path: %s\n", partialPath) + fmt.Fprintf(os.Stderr, "Full tile path: %s\n", fullPath) + + // Try to fetch the tile (partial first, then full) + var tileData []byte + var fetchedPath string + + // Try partial tile first + partialURL := logURL + "/" + partialPath + fmt.Fprintf(os.Stderr, "Trying: %s\n", partialURL) + tileData, err = fetchURL(partialURL) + if err == nil { + fetchedPath = partialPath + fmt.Fprintf(os.Stderr, "Successfully fetched partial tile\n") + } else { + // Fall back to full tile + fullURL := logURL + "/" + fullPath + fmt.Fprintf(os.Stderr, "Partial tile failed, trying: %s\n", fullURL) + tileData, err = fetchURL(fullURL) + if err != nil { + fatal("failed to fetch tile: %v", err) + } + fetchedPath = fullPath + fmt.Fprintf(os.Stderr, "Successfully fetched full tile\n") + } + + // Decompress if needed + tileData, err = decompress(tileData) + if err != nil { + fatal("failed to decompress tile: %v", err) + } + + fmt.Fprintf(os.Stderr, "Tile size: %d bytes\n", len(tileData)) + fmt.Fprintf(os.Stderr, "Fetched path: %s\n\n", fetchedPath) + + if *dumpAll { + // Dump all entries in the tile + dumpAllEntries(tileData) + } else { + // Dump only the specific entry at the position + dumpEntryAtPosition(tileData, int(positionInTile), leafIndex) + } +} + +func fetchURL(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +func dumpAllEntries(tileData []byte) { + entryNum := 0 + for len(tileData) > 0 { + e, remaining, err := sunlight.ReadTileLeaf(tileData) + if err != nil { + fatal("failed to read entry %d: %v", entryNum, err) + } + tileData = remaining + + dumpEntry(e, entryNum) + fmt.Println() + entryNum++ + } + + fmt.Printf("Total entries: %d\n", entryNum) +} + +func dumpEntryAtPosition(tileData []byte, position int, expectedIndex int64) { + entryNum := 0 + for len(tileData) > 0 { + e, remaining, err := sunlight.ReadTileLeaf(tileData) + if err != nil { + fatal("failed to read entry %d: %v", entryNum, err) + } + tileData = remaining + + if entryNum == position { + if e.LeafIndex != expectedIndex { + fmt.Fprintf(os.Stderr, "WARNING: Expected leaf index %d but found %d at position %d\n", + expectedIndex, e.LeafIndex, position) + } + dumpEntry(e, entryNum) + return + } + entryNum++ + } + + fatal("position %d not found in tile (only %d entries)", position, entryNum) +} + +func dumpEntry(e *sunlight.LogEntry, entryNum int) { + fmt.Printf("=== Entry %d ===\n", entryNum) + fmt.Printf("Leaf Index: %d\n", e.LeafIndex) + fmt.Printf("Timestamp: %d\n", e.Timestamp) + fmt.Printf("Is Precert: %v\n", e.IsPrecert) + + if e.IsPrecert { + fmt.Printf("Issuer Key Hash: %x\n", e.IssuerKeyHash) + } + + fmt.Printf("Certificate: %d bytes\n", len(e.Certificate)) + if e.PreCertificate != nil { + fmt.Printf("PreCertificate: %d bytes\n", len(e.PreCertificate)) + } + + fmt.Printf("Chain Fingerprints: %d entries\n", len(e.ChainFingerprints)) + for i, fp := range e.ChainFingerprints { + fmt.Printf(" [%d]: %x\n", i, fp) + } + + // Try to extract parsed certificate info + if trimmed, err := e.TrimmedEntry(); err == nil { + if data, err := json.MarshalIndent(trimmed, " ", " "); err == nil { + fmt.Printf("Parsed Certificate Info:\n %s\n", data) + } + } +} + +const maxCompressRatio = 100 + +func decompress(data []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + // Not gzipped, return as-is + return data, nil + } + maxSize := int64(len(data)) * maxCompressRatio + return io.ReadAll(io.LimitReader(r, maxSize)) +} + +func fatal(format string, args ...any) { + fmt.Fprintf(os.Stderr, "Error: "+format+"\n", args...) + os.Exit(1) +}