ipng/nginx-logtail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add is_tor plumbing from collector->aggregator->frontend/cli

Browse Source
This commit is contained in:
Pim van Pelt
2026-03-23 22:17:39 +01:00
parent b89caa594c
commit cd7f15afaf
20 changed files with 1815 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/aggregator/aggregator_test.go
View File

@@ -163,8 +163,8 @@ func TestCacheCoarseRing(t *testing.T) {
func TestCacheQueryTopN(t *testing.T) {
m := NewMerger()
m.Apply(makeSnap("c1", map[string]int64{
st.EncodeTuple(st.Tuple4{"busy.com", "1.0.0.0/24", "/", "200"}): 300,
st.EncodeTuple(st.Tuple4{"quiet.com", "2.0.0.0/24", "/", "200"}): 50,
st.EncodeTuple(st.Tuple5{Website: "busy.com", Prefix: "1.0.0.0/24", URI: "/", Status: "200"}): 300,
st.EncodeTuple(st.Tuple5{Website: "quiet.com", Prefix: "2.0.0.0/24", URI: "/", Status: "200"}): 50,
}))
cache := NewCache(m, "test")
@@ -181,8 +181,8 @@ func TestCacheQueryTopN(t *testing.T) {
func TestCacheQueryTopNWithFilter(t *testing.T) {
m := NewMerger()
status429 := st.EncodeTuple(st.Tuple4{"example.com", "1.0.0.0/24", "/api", "429"})
status200 := st.EncodeTuple(st.Tuple4{"example.com", "2.0.0.0/24", "/api", "200"})
status429 := st.EncodeTuple(st.Tuple5{Website: "example.com", Prefix: "1.0.0.0/24", URI: "/api", Status: "429"})
status200 := st.EncodeTuple(st.Tuple5{Website: "example.com", Prefix: "2.0.0.0/24", URI: "/api", Status: "200"})
m.Apply(makeSnap("c1", map[string]int64{status429: 200, status200: 500}))
cache := NewCache(m, "test")
@@ -202,7 +202,7 @@ func TestCacheQueryTrend(t *testing.T) {
for i, count := range []int64{10, 20, 30} {
m.Apply(makeSnap("c1", map[string]int64{
st.EncodeTuple(st.Tuple4{"x.com", "1.0.0.0/24", "/", "200"}): count,
st.EncodeTuple(st.Tuple5{Website: "x.com", Prefix: "1.0.0.0/24", URI: "/", Status: "200"}): count,
}))
cache.rotate(now.Add(time.Duration(i) * time.Minute))
}
@@ -270,12 +270,12 @@ func startFakeCollector(t *testing.T, snaps []*pb.Snapshot) string {
func TestGRPCEndToEnd(t *testing.T) {
// Two fake collectors with overlapping labels.
snap1 := makeSnap("col1", map[string]int64{
st.EncodeTuple(st.Tuple4{"busy.com", "1.0.0.0/24", "/", "200"}): 500,
st.EncodeTuple(st.Tuple4{"quiet.com", "2.0.0.0/24", "/", "429"}): 100,
st.EncodeTuple(st.Tuple5{Website: "busy.com", Prefix: "1.0.0.0/24", URI: "/", Status: "200"}): 500,
st.EncodeTuple(st.Tuple5{Website: "quiet.com", Prefix: "2.0.0.0/24", URI: "/", Status: "429"}): 100,
})
snap2 := makeSnap("col2", map[string]int64{
st.EncodeTuple(st.Tuple4{"busy.com", "3.0.0.0/24", "/", "200"}): 300,
st.EncodeTuple(st.Tuple4{"other.com", "4.0.0.0/24", "/", "200"}): 50,
st.EncodeTuple(st.Tuple5{Website: "busy.com", Prefix: "3.0.0.0/24", URI: "/", Status: "200"}): 300,
st.EncodeTuple(st.Tuple5{Website: "other.com", Prefix: "4.0.0.0/24", URI: "/", Status: "200"}): 50,
})
addr1 := startFakeCollector(t, []*pb.Snapshot{snap1})
addr2 := startFakeCollector(t, []*pb.Snapshot{snap2})
@@ -388,7 +388,7 @@ func TestGRPCEndToEnd(t *testing.T) {
func TestDegradedCollector(t *testing.T) {
// Start one real and one immediately-gone collector.
snap1 := makeSnap("col1", map[string]int64{
st.EncodeTuple(st.Tuple4{"good.com", "1.0.0.0/24", "/", "200"}): 100,
st.EncodeTuple(st.Tuple5{Website: "good.com", Prefix: "1.0.0.0/24", URI: "/", Status: "200"}): 100,
})
addr1 := startFakeCollector(t, []*pb.Snapshot{snap1})
// addr2 points at nothing — connections will fail immediately.

15
cmd/cli/flags.go
View File

@@ -20,6 +20,7 @@ type sharedFlags struct {
status string // expression: "200", "!=200", ">=400", etc.
websiteRe string // RE2 regex against website
uriRe string // RE2 regex against request URI
isTor string // "", "1" / "!=0" (TOR only), "0" / "!=1" (non-TOR only)
}
// bindShared registers the shared flags on fs and returns a pointer to the
@@ -34,6 +35,7 @@ func bindShared(fs *flag.FlagSet) (*sharedFlags, *string) {
fs.StringVar(&sf.status, "status", "", "filter: HTTP status expression (200, !=200, >=400, <500, …)")
fs.StringVar(&sf.websiteRe, "website-re", "", "filter: RE2 regex against website")
fs.StringVar(&sf.uriRe, "uri-re", "", "filter: RE2 regex against request URI")
fs.StringVar(&sf.isTor, "is-tor", "", "filter: TOR traffic (1 or !=0 = TOR only; 0 or !=1 = non-TOR only)")
return sf, target
}
@@ -56,7 +58,7 @@ func parseTargets(s string) []string {
}
func buildFilter(sf *sharedFlags) *pb.Filter {
if sf.website == "" && sf.prefix == "" && sf.uri == "" && sf.status == "" && sf.websiteRe == "" && sf.uriRe == "" {
if sf.website == "" && sf.prefix == "" && sf.uri == "" && sf.status == "" && sf.websiteRe == "" && sf.uriRe == "" && sf.isTor == "" {
return nil
}
f := &pb.Filter{}
@@ -84,6 +86,17 @@ func buildFilter(sf *sharedFlags) *pb.Filter {
if sf.uriRe != "" {
f.UriRegex = &sf.uriRe
}
switch sf.isTor {
case "1", "!=0":
f.Tor = pb.TorFilter_TOR_YES
case "0", "!=1":
f.Tor = pb.TorFilter_TOR_NO
case "":
// no filter
default:
fmt.Fprintf(os.Stderr, "--is-tor: invalid value %q; use 1, 0, !=0, or !=1\n", sf.isTor)
os.Exit(1)
}
return f
}

14
cmd/collector/parser.go
View File

@@ -6,22 +6,25 @@ import (
"strings"
)
// LogRecord holds the four dimensions extracted from a single nginx log line.
// LogRecord holds the dimensions extracted from a single nginx log line.
type LogRecord struct {
Website string
ClientPrefix string
URI string
Status string
IsTor bool
}
// ParseLine parses a tab-separated logtail log line:
//
// $host \t $remote_addr \t $msec \t $request_method \t $request_uri \t $status \t $body_bytes_sent \t $request_time
// $host \t $remote_addr \t $msec \t $request_method \t $request_uri \t $status \t $body_bytes_sent \t $request_time \t $is_tor
//
// The is_tor field (0 or 1) is optional for backward compatibility with
// older log files that omit it; it defaults to false when absent.
// Returns false for lines with fewer than 8 fields.
func ParseLine(line string, v4bits, v6bits int) (LogRecord, bool) {
// SplitN caps allocations; we need exactly 8 fields.
fields := strings.SplitN(line, "\t", 8)
// SplitN caps allocations; we need up to 9 fields.
fields := strings.SplitN(line, "\t", 9)
if len(fields) < 8 {
return LogRecord{}, false
}
@@ -36,11 +39,14 @@ func ParseLine(line string, v4bits, v6bits int) (LogRecord, bool) {
return LogRecord{}, false
}
isTor := len(fields) == 9 && fields[8] == "1"
return LogRecord{
Website: fields[0],
ClientPrefix: prefix,
URI: uri,
Status: fields[5],
IsTor: isTor,
}, true
}

36
cmd/collector/parser_test.go
View File

@@ -72,6 +72,42 @@ func TestParseLine(t *testing.T) {
Status: "429",
},
},
{
name: "is_tor=1 sets IsTor true",
line: "tor.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
wantOK: true,
want: LogRecord{
Website: "tor.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
},
},
{
name: "is_tor=0 sets IsTor false",
line: "normal.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0",
wantOK: true,
want: LogRecord{
Website: "normal.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
},
},
{
name: "missing is_tor field defaults to false (backward compat)",
line: "old.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "old.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
},
},
}
for _, tc := range tests {

6
cmd/collector/smoke_test.go
View File

@@ -104,10 +104,10 @@ func TestGRPCEndToEnd(t *testing.T) {
// Pre-populate with known data then rotate so it's queryable
for i := 0; i < 500; i++ {
store.ingest(LogRecord{"busy.com", "1.2.3.0/24", "/api", "200"})
store.ingest(LogRecord{Website: "busy.com", ClientPrefix: "1.2.3.0/24", URI: "/api", Status: "200"})
}
for i := 0; i < 200; i++ {
store.ingest(LogRecord{"quiet.com", "5.6.7.0/24", "/", "429"})
store.ingest(LogRecord{Website: "quiet.com", ClientPrefix: "5.6.7.0/24", URI: "/", Status: "429"})
}
store.rotate(time.Now())
@@ -192,7 +192,7 @@ func TestGRPCEndToEnd(t *testing.T) {
t.Fatalf("StreamSnapshots error: %v", err)
}
store.ingest(LogRecord{"new.com", "9.9.9.0/24", "/new", "200"})
store.ingest(LogRecord{Website: "new.com", ClientPrefix: "9.9.9.0/24", URI: "/new", Status: "200"})
store.rotate(time.Now())
snap, err := stream.Recv()

8
cmd/collector/store.go
View File

@@ -15,7 +15,7 @@ type Store struct {
source string
// live map — written only by the Run goroutine; no locking needed on writes
live map[st.Tuple4]int64
live map[st.Tuple5]int64
liveLen int
// ring buffers — protected by mu for reads
@@ -36,7 +36,7 @@ type Store struct {
func NewStore(source string) *Store {
return &Store{
source: source,
live: make(map[st.Tuple4]int64, liveMapCap),
live: make(map[st.Tuple5]int64, liveMapCap),
subs: make(map[chan st.Snapshot]struct{}),
}
}
@@ -44,7 +44,7 @@ func NewStore(source string) *Store {
// ingest records one log record into the live map.
// Must only be called from the Run goroutine.
func (s *Store) ingest(r LogRecord) {
key := st.Tuple4{Website: r.Website, Prefix: r.ClientPrefix, URI: r.URI, Status: r.Status}
key := st.Tuple5{Website: r.Website, Prefix: r.ClientPrefix, URI: r.URI, Status: r.Status, IsTor: r.IsTor}
if _, exists := s.live[key]; !exists {
if s.liveLen >= liveMapCap {
return
@@ -77,7 +77,7 @@ func (s *Store) rotate(now time.Time) {
}
s.mu.Unlock()
s.live = make(map[st.Tuple4]int64, liveMapCap)
s.live = make(map[st.Tuple5]int64, liveMapCap)
s.liveLen = 0
s.broadcast(fine)

2
cmd/collector/store_test.go
View File

@@ -15,7 +15,7 @@ func makeStore() *Store {
func ingestN(s *Store, website, prefix, uri, status string, n int) {
for i := 0; i < n; i++ {
s.ingest(LogRecord{website, prefix, uri, status})
s.ingest(LogRecord{Website: website, ClientPrefix: prefix, URI: uri, Status: status})
}
}

18
cmd/frontend/filter.go
View File

@@ -113,8 +113,21 @@ func applyTerm(term string, fs *filterState) error {
return fmt.Errorf("prefix only supports =, not %q", op)
}
fs.Prefix = value
case "is_tor":
if op != "=" && op != "!=" {
return fmt.Errorf("is_tor only supports = and !=, not %q", op)
}
if value != "0" && value != "1" {
return fmt.Errorf("is_tor value must be 0 or 1, not %q", value)
}
// Normalise: is_tor=1 and is_tor!=0 both mean "TOR only"
if (op == "=" && value == "1") || (op == "!=" && value == "0") {
fs.IsTor = "1"
} else {
fs.IsTor = "0"
}
default:
return fmt.Errorf("unknown field %q; valid: status, website, uri, prefix", field)
return fmt.Errorf("unknown field %q; valid: status, website, uri, prefix, is_tor", field)
}
return nil
}
@@ -151,6 +164,9 @@ func FilterExprString(f filterState) string {
if f.Status != "" {
parts = append(parts, statusTermStr(f.Status))
}
if f.IsTor != "" {
parts = append(parts, "is_tor="+f.IsTor)
}
return strings.Join(parts, " AND ")
}

45
cmd/frontend/handler.go
View File

@@ -53,6 +53,7 @@ type filterState struct {
Status string // expression: "200", "!=200", ">=400", etc.
WebsiteRe string // RE2 regex against website
URIRe string // RE2 regex against request URI
IsTor string // "", "1" (TOR only), "0" (non-TOR only)
}
// QueryParams holds all parsed URL parameters for one page request.
@@ -77,6 +78,7 @@ type PageData struct {
Windows []Tab
GroupBys []Tab
Targets []Tab // source/target picker; empty when only one target available
TorTabs []Tab // all / tor / no-tor toggle
RefreshSecs int
Error string
FilterExpr string // current filter serialised to mini-language for the input box
@@ -156,12 +158,13 @@ func (h *Handler) parseParams(r *http.Request) QueryParams {
Status: q.Get("f_status"),
WebsiteRe: q.Get("f_website_re"),
URIRe: q.Get("f_uri_re"),
IsTor: q.Get("f_is_tor"),
},
}
}
func buildFilter(f filterState) *pb.Filter {
if f.Website == "" && f.Prefix == "" && f.URI == "" && f.Status == "" && f.WebsiteRe == "" && f.URIRe == "" {
if f.Website == "" && f.Prefix == "" && f.URI == "" && f.Status == "" && f.WebsiteRe == "" && f.URIRe == "" && f.IsTor == "" {
return nil
}
out := &pb.Filter{}
@@ -186,6 +189,12 @@ func buildFilter(f filterState) *pb.Filter {
if f.URIRe != "" {
out.UriRegex = &f.URIRe
}
switch f.IsTor {
case "1":
out.Tor = pb.TorFilter_TOR_YES
case "0":
out.Tor = pb.TorFilter_TOR_NO
}
return out
}
@@ -214,6 +223,9 @@ func (p QueryParams) toValues() url.Values {
if p.Filter.URIRe != "" {
v.Set("f_uri_re", p.Filter.URIRe)
}
if p.Filter.IsTor != "" {
v.Set("f_is_tor", p.Filter.IsTor)
}
return v
}
@@ -314,6 +326,18 @@ func buildCrumbs(p QueryParams) []Crumb {
RemoveURL: p.buildURL(map[string]string{"f_uri_re": ""}),
})
}
switch p.Filter.IsTor {
case "1":
crumbs = append(crumbs, Crumb{
Text: "is_tor=1 (TOR only)",
RemoveURL: p.buildURL(map[string]string{"f_is_tor": ""}),
})
case "0":
crumbs = append(crumbs, Crumb{
Text: "is_tor=0 (no TOR)",
RemoveURL: p.buildURL(map[string]string{"f_is_tor": ""}),
})
}
return crumbs
}
@@ -341,6 +365,23 @@ func buildGroupByTabs(p QueryParams) []Tab {
return tabs
}
func buildTorTabs(p QueryParams) []Tab {
specs := []struct{ val, label string }{
{"", "all"},
{"1", "tor"},
{"0", "no tor"},
}
tabs := make([]Tab, len(specs))
for i, s := range specs {
tabs[i] = Tab{
Label: s.label,
URL: p.buildURL(map[string]string{"f_is_tor": s.val}),
Active: p.Filter.IsTor == s.val,
}
}
return tabs
}
// buildTargetTabs builds the source/target picker tabs from a ListTargets response.
// Returns nil (hide picker) when only one endpoint is reachable.
func (h *Handler) buildTargetTabs(p QueryParams, lt *pb.ListTargetsResponse) []Tab {
@@ -502,6 +543,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Breadcrumbs: buildCrumbs(params),
Windows: buildWindowTabs(params),
GroupBys: buildGroupByTabs(params),
TorTabs: buildTorTabs(params),
Targets: h.buildTargetTabs(params, lt),
RefreshSecs: h.refreshSecs,
FilterExpr: filterExprInput,
@@ -524,6 +566,7 @@ func (h *Handler) errorPage(params QueryParams, msg string) PageData {
Params: params,
Windows: buildWindowTabs(params),
GroupBys: buildGroupByTabs(params),
TorTabs: buildTorTabs(params),
Breadcrumbs: buildCrumbs(params),
RefreshSecs: h.refreshSecs,
Error: msg,

1
cmd/frontend/templates/base.html
View File

@@ -35,6 +35,7 @@ a:hover { text-decoration: underline; }
.nodata { color: #999; margin: 2em 0; font-style: italic; }
footer { margin-top: 2em; padding-top: 0.6em; border-top: 1px solid #e0e0e0; font-size: 0.8em; color: #999; }
.tabs-targets { margin-top: -0.4em; }
.tabs-tor { margin-top: -0.4em; }
.tabs-label { font-size: 0.85em; color: #888; margin-right: 0.2em; align-self: center; }
.filter-form { display: flex; gap: 0.4em; align-items: center; margin-bottom: 0.7em; }
.filter-input { flex: 1; font-family: monospace; font-size: 13px; padding: 0.25em 0.5em; border: 1px solid #aaa; }

9
cmd/frontend/templates/index.html
View File

@@ -20,12 +20,19 @@
{{- end}}
</div>{{end}}
<div class="tabs tabs-tor">
<span class="tabs-label">tor:</span>
{{- range .TorTabs}}
<a href="{{.URL}}"{{if .Active}} class="active"{{end}}>{{.Label}}</a>
{{- end}}
</div>
<form class="filter-form" method="get" action="/">
<input type="hidden" name="target" value="{{.Params.Target}}">
<input type="hidden" name="w" value="{{.Params.WindowS}}">
<input type="hidden" name="by" value="{{.Params.GroupByS}}">
<input type="hidden" name="n" value="{{.Params.N}}">
<input class="filter-input" type="text" name="q" value="{{.FilterExpr}}" placeholder="status>=400 AND website~=gouda.* AND uri~=^/api/">
<input class="filter-input" type="text" name="q" value="{{.FilterExpr}}" placeholder="status>=400 AND website~=gouda.* AND uri~=^/api/ AND is_tor=1">
<button type="submit">filter</button>
{{- if .FilterExpr}} <a class="clear" href="{{.ClearFilterURL}}">× clear</a>{{end}}
</form>