ipng/nginx-logtail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ASN to logtail, collector, aggregator, frontend and CLI

Browse Source
This commit is contained in:
Pim van Pelt
2026-03-24 02:28:29 +01:00
parent a798bb1d1d
commit 30c8c40157
17 changed files with 566 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/store/store.go
View File

@@ -6,6 +6,7 @@ import (
"container/heap"
"log"
"regexp"
"strconv"
"time"
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
@@ -20,13 +21,14 @@ const (
CoarseEvery = 5 // fine ticks between coarse writes
)
// Tuple5 is the aggregation key (website, prefix, URI, status, is_tor).
type Tuple5 struct {
// Tuple6 is the aggregation key (website, prefix, URI, status, is_tor, asn).
type Tuple6 struct {
Website string
Prefix string
URI string
Status string
IsTor bool
ASN int32
}
// Entry is a labelled count used in snapshots and query results.
@@ -74,28 +76,33 @@ func BucketsForWindow(window pb.Window, fine, coarse RingView, fineFilled, coars
}
}
// --- label encoding: "website\x00prefix\x00uri\x00status\x00is_tor" ---
// --- label encoding: "website\x00prefix\x00uri\x00status\x00is_tor\x00asn" ---
// EncodeTuple encodes a Tuple5 as a NUL-separated string suitable for use
// EncodeTuple encodes a Tuple6 as a NUL-separated string suitable for use
// as a map key in snapshots.
func EncodeTuple(t Tuple5) string {
func EncodeTuple(t Tuple6) string {
tor := "0"
if t.IsTor {
tor = "1"
}
return t.Website + "\x00" + t.Prefix + "\x00" + t.URI + "\x00" + t.Status + "\x00" + tor
return t.Website + "\x00" + t.Prefix + "\x00" + t.URI + "\x00" + t.Status + "\x00" + tor + "\x00" + strconv.Itoa(int(t.ASN))
}
// LabelTuple decodes a NUL-separated snapshot label back into a Tuple5.
func LabelTuple(label string) Tuple5 {
parts := splitN(label, '\x00', 5)
// LabelTuple decodes a NUL-separated snapshot label back into a Tuple6.
func LabelTuple(label string) Tuple6 {
parts := splitN(label, '\x00', 6)
if len(parts) < 4 {
return Tuple5{}
return Tuple6{}
}
t := Tuple5{Website: parts[0], Prefix: parts[1], URI: parts[2], Status: parts[3]}
if len(parts) == 5 {
t := Tuple6{Website: parts[0], Prefix: parts[1], URI: parts[2], Status: parts[3]}
if len(parts) >= 5 {
t.IsTor = parts[4] == "1"
}
if len(parts) == 6 {
if n, err := strconv.Atoi(parts[5]); err == nil {
t.ASN = int32(n)
}
}
return t
}
@@ -159,7 +166,7 @@ func CompileFilter(f *pb.Filter) *CompiledFilter {
// MatchesFilter returns true if t satisfies all constraints in f.
// A nil filter matches everything.
func MatchesFilter(t Tuple5, f *CompiledFilter) bool {
func MatchesFilter(t Tuple6, f *CompiledFilter) bool {
if f == nil || f.Proto == nil {
return true
}
@@ -199,9 +206,30 @@ func MatchesFilter(t Tuple5, f *CompiledFilter) bool {
return false
}
}
if p.AsnNumber != nil && !matchesAsnOp(t.ASN, p.GetAsnNumber(), p.AsnOp) {
return false
}
return true
}
// matchesAsnOp applies op(asn, want) directly on int32 values.
func matchesAsnOp(asn, want int32, op pb.StatusOp) bool {
switch op {
case pb.StatusOp_NE:
return asn != want
case pb.StatusOp_GT:
return asn > want
case pb.StatusOp_GE:
return asn >= want
case pb.StatusOp_LT:
return asn < want
case pb.StatusOp_LE:
return asn <= want
default: // EQ
return asn == want
}
}
// matchesStatusOp applies op(statusStr, want), parsing statusStr as an integer.
// Returns false if statusStr is not a valid integer.
func matchesStatusOp(statusStr string, want int32, op pb.StatusOp) bool {
@@ -229,7 +257,7 @@ func matchesStatusOp(statusStr string, want int32, op pb.StatusOp) bool {
}
// DimensionLabel returns the string value of t for the given group-by dimension.
func DimensionLabel(t Tuple5, g pb.GroupBy) string {
func DimensionLabel(t Tuple6, g pb.GroupBy) string {
switch g {
case pb.GroupBy_WEBSITE:
return t.Website
@@ -239,6 +267,8 @@ func DimensionLabel(t Tuple5, g pb.GroupBy) string {
return t.URI
case pb.GroupBy_HTTP_RESPONSE:
return t.Status
case pb.GroupBy_ASN_NUMBER:
return strconv.Itoa(int(t.ASN))
default:
return t.Website
}
@@ -318,9 +348,9 @@ func TopKFromMap(m map[string]int64, k int) []Entry {
return result
}
// TopKFromTupleMap encodes a Tuple5 map and returns the top-k as a Snapshot.
// TopKFromTupleMap encodes a Tuple6 map and returns the top-k as a Snapshot.
// Used by the collector to snapshot its live map.
func TopKFromTupleMap(m map[Tuple5]int64, k int, ts time.Time) Snapshot {
func TopKFromTupleMap(m map[Tuple6]int64, k int, ts time.Time) Snapshot {
flat := make(map[string]int64, len(m))
for t, c := range m {
flat[EncodeTuple(t)] = c

136
internal/store/store_test.go
View File

@@ -83,10 +83,10 @@ func compiledEQ(status int32) *CompiledFilter {
}
func TestMatchesFilterNil(t *testing.T) {
if !MatchesFilter(Tuple5{Website: "x"}, nil) {
if !MatchesFilter(Tuple6{Website: "x"}, nil) {
t.Fatal("nil filter should match everything")
}
if !MatchesFilter(Tuple5{Website: "x"}, &CompiledFilter{}) {
if !MatchesFilter(Tuple6{Website: "x"}, &CompiledFilter{}) {
t.Fatal("empty compiled filter should match everything")
}
}
@@ -94,10 +94,10 @@ func TestMatchesFilterNil(t *testing.T) {
func TestMatchesFilterExactWebsite(t *testing.T) {
w := "example.com"
cf := CompileFilter(&pb.Filter{Website: &w})
if !MatchesFilter(Tuple5{Website: "example.com"}, cf) {
if !MatchesFilter(Tuple6{Website: "example.com"}, cf) {
t.Fatal("expected match")
}
if MatchesFilter(Tuple5{Website: "other.com"}, cf) {
if MatchesFilter(Tuple6{Website: "other.com"}, cf) {
t.Fatal("expected no match")
}
}
@@ -105,10 +105,10 @@ func TestMatchesFilterExactWebsite(t *testing.T) {
func TestMatchesFilterWebsiteRegex(t *testing.T) {
re := "gouda.*"
cf := CompileFilter(&pb.Filter{WebsiteRegex: &re})
if !MatchesFilter(Tuple5{Website: "gouda.example.com"}, cf) {
if !MatchesFilter(Tuple6{Website: "gouda.example.com"}, cf) {
t.Fatal("expected match")
}
if MatchesFilter(Tuple5{Website: "edam.example.com"}, cf) {
if MatchesFilter(Tuple6{Website: "edam.example.com"}, cf) {
t.Fatal("expected no match")
}
}
@@ -116,10 +116,10 @@ func TestMatchesFilterWebsiteRegex(t *testing.T) {
func TestMatchesFilterURIRegex(t *testing.T) {
re := "^/api/.*"
cf := CompileFilter(&pb.Filter{UriRegex: &re})
if !MatchesFilter(Tuple5{URI: "/api/users"}, cf) {
if !MatchesFilter(Tuple6{URI: "/api/users"}, cf) {
t.Fatal("expected match")
}
if MatchesFilter(Tuple5{URI: "/health"}, cf) {
if MatchesFilter(Tuple6{URI: "/health"}, cf) {
t.Fatal("expected no match")
}
}
@@ -127,17 +127,17 @@ func TestMatchesFilterURIRegex(t *testing.T) {
func TestMatchesFilterInvalidRegexMatchesNothing(t *testing.T) {
re := "[invalid"
cf := CompileFilter(&pb.Filter{WebsiteRegex: &re})
if MatchesFilter(Tuple5{Website: "anything"}, cf) {
if MatchesFilter(Tuple6{Website: "anything"}, cf) {
t.Fatal("invalid regex should match nothing")
}
}
func TestMatchesFilterStatusEQ(t *testing.T) {
cf := compiledEQ(200)
if !MatchesFilter(Tuple5{Status: "200"}, cf) {
if !MatchesFilter(Tuple6{Status: "200"}, cf) {
t.Fatal("expected match")
}
if MatchesFilter(Tuple5{Status: "404"}, cf) {
if MatchesFilter(Tuple6{Status: "404"}, cf) {
t.Fatal("expected no match")
}
}
@@ -145,10 +145,10 @@ func TestMatchesFilterStatusEQ(t *testing.T) {
func TestMatchesFilterStatusNE(t *testing.T) {
v := int32(200)
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_NE})
if MatchesFilter(Tuple5{Status: "200"}, cf) {
if MatchesFilter(Tuple6{Status: "200"}, cf) {
t.Fatal("expected no match for 200 != 200")
}
if !MatchesFilter(Tuple5{Status: "404"}, cf) {
if !MatchesFilter(Tuple6{Status: "404"}, cf) {
t.Fatal("expected match for 404 != 200")
}
}
@@ -156,13 +156,13 @@ func TestMatchesFilterStatusNE(t *testing.T) {
func TestMatchesFilterStatusGE(t *testing.T) {
v := int32(400)
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_GE})
if !MatchesFilter(Tuple5{Status: "400"}, cf) {
if !MatchesFilter(Tuple6{Status: "400"}, cf) {
t.Fatal("expected match: 400 >= 400")
}
if !MatchesFilter(Tuple5{Status: "500"}, cf) {
if !MatchesFilter(Tuple6{Status: "500"}, cf) {
t.Fatal("expected match: 500 >= 400")
}
if MatchesFilter(Tuple5{Status: "200"}, cf) {
if MatchesFilter(Tuple6{Status: "200"}, cf) {
t.Fatal("expected no match: 200 >= 400")
}
}
@@ -170,17 +170,17 @@ func TestMatchesFilterStatusGE(t *testing.T) {
func TestMatchesFilterStatusLT(t *testing.T) {
v := int32(400)
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_LT})
if !MatchesFilter(Tuple5{Status: "200"}, cf) {
if !MatchesFilter(Tuple6{Status: "200"}, cf) {
t.Fatal("expected match: 200 < 400")
}
if MatchesFilter(Tuple5{Status: "400"}, cf) {
if MatchesFilter(Tuple6{Status: "400"}, cf) {
t.Fatal("expected no match: 400 < 400")
}
}
func TestMatchesFilterStatusNonNumeric(t *testing.T) {
cf := compiledEQ(200)
if MatchesFilter(Tuple5{Status: "ok"}, cf) {
if MatchesFilter(Tuple6{Status: "ok"}, cf) {
t.Fatal("non-numeric status should not match")
}
}
@@ -193,13 +193,13 @@ func TestMatchesFilterCombined(t *testing.T) {
HttpResponse: &v,
StatusOp: pb.StatusOp_EQ,
})
if !MatchesFilter(Tuple5{Website: "example.com", Status: "200"}, cf) {
if !MatchesFilter(Tuple6{Website: "example.com", Status: "200"}, cf) {
t.Fatal("expected match")
}
if MatchesFilter(Tuple5{Website: "other.com", Status: "200"}, cf) {
if MatchesFilter(Tuple6{Website: "other.com", Status: "200"}, cf) {
t.Fatal("expected no match: wrong website")
}
if MatchesFilter(Tuple5{Website: "example.com", Status: "404"}, cf) {
if MatchesFilter(Tuple6{Website: "example.com", Status: "404"}, cf) {
t.Fatal("expected no match: wrong status")
}
}
@@ -208,7 +208,7 @@ func TestMatchesFilterCombined(t *testing.T) {
func TestEncodeLabelTupleRoundtripWithTor(t *testing.T) {
for _, isTor := range []bool{false, true} {
orig := Tuple5{Website: "a.com", Prefix: "1.2.3.0/24", URI: "/x", Status: "200", IsTor: isTor}
orig := Tuple6{Website: "a.com", Prefix: "1.2.3.0/24", URI: "/x", Status: "200", IsTor: isTor}
got := LabelTuple(EncodeTuple(orig))
if got != orig {
t.Errorf("roundtrip mismatch: got %+v, want %+v", got, orig)
@@ -230,30 +230,108 @@ func TestLabelTupleBackwardCompat(t *testing.T) {
func TestMatchesFilterTorYes(t *testing.T) {
cf := CompileFilter(&pb.Filter{Tor: pb.TorFilter_TOR_YES})
if !MatchesFilter(Tuple5{IsTor: true}, cf) {
if !MatchesFilter(Tuple6{IsTor: true}, cf) {
t.Fatal("TOR_YES should match TOR tuple")
}
if MatchesFilter(Tuple5{IsTor: false}, cf) {
if MatchesFilter(Tuple6{IsTor: false}, cf) {
t.Fatal("TOR_YES should not match non-TOR tuple")
}
}
func TestMatchesFilterTorNo(t *testing.T) {
cf := CompileFilter(&pb.Filter{Tor: pb.TorFilter_TOR_NO})
if !MatchesFilter(Tuple5{IsTor: false}, cf) {
if !MatchesFilter(Tuple6{IsTor: false}, cf) {
t.Fatal("TOR_NO should match non-TOR tuple")
}
if MatchesFilter(Tuple5{IsTor: true}, cf) {
if MatchesFilter(Tuple6{IsTor: true}, cf) {
t.Fatal("TOR_NO should not match TOR tuple")
}
}
func TestMatchesFilterTorAny(t *testing.T) {
cf := CompileFilter(&pb.Filter{Tor: pb.TorFilter_TOR_ANY})
if !MatchesFilter(Tuple5{IsTor: true}, cf) {
if !MatchesFilter(Tuple6{IsTor: true}, cf) {
t.Fatal("TOR_ANY should match TOR tuple")
}
if !MatchesFilter(Tuple5{IsTor: false}, cf) {
if !MatchesFilter(Tuple6{IsTor: false}, cf) {
t.Fatal("TOR_ANY should match non-TOR tuple")
}
}
// --- ASN label encoding, filtering, and DimensionLabel ---
func TestEncodeLabelTupleRoundtripWithASN(t *testing.T) {
for _, asn := range []int32{0, 1, 12345, 65535} {
orig := Tuple6{Website: "a.com", Prefix: "1.2.3.0/24", URI: "/x", Status: "200", ASN: asn}
got := LabelTuple(EncodeTuple(orig))
if got != orig {
t.Errorf("roundtrip mismatch for ASN=%d: got %+v, want %+v", asn, got, orig)
}
}
}
func TestLabelTupleBackwardCompatNoASN(t *testing.T) {
// 5-field label (no asn field) should decode with ASN=0.
label := "a.com\x001.2.3.0/24\x00/x\x00200\x000"
got := LabelTuple(label)
if got.ASN != 0 {
t.Errorf("expected ASN=0 for 5-field label, got %d", got.ASN)
}
}
func TestMatchesFilterAsnEQ(t *testing.T) {
n := int32(12345)
cf := CompileFilter(&pb.Filter{AsnNumber: &n})
if !MatchesFilter(Tuple6{ASN: 12345}, cf) {
t.Fatal("EQ should match equal ASN")
}
if MatchesFilter(Tuple6{ASN: 99999}, cf) {
t.Fatal("EQ should not match different ASN")
}
}
func TestMatchesFilterAsnNE(t *testing.T) {
n := int32(12345)
cf := CompileFilter(&pb.Filter{AsnNumber: &n, AsnOp: pb.StatusOp_NE})
if MatchesFilter(Tuple6{ASN: 12345}, cf) {
t.Fatal("NE should not match equal ASN")
}
if !MatchesFilter(Tuple6{ASN: 99999}, cf) {
t.Fatal("NE should match different ASN")
}
}
func TestMatchesFilterAsnGE(t *testing.T) {
n := int32(1000)
cf := CompileFilter(&pb.Filter{AsnNumber: &n, AsnOp: pb.StatusOp_GE})
if !MatchesFilter(Tuple6{ASN: 1000}, cf) {
t.Fatal("GE should match equal ASN")
}
if !MatchesFilter(Tuple6{ASN: 2000}, cf) {
t.Fatal("GE should match larger ASN")
}
if MatchesFilter(Tuple6{ASN: 500}, cf) {
t.Fatal("GE should not match smaller ASN")
}
}
func TestMatchesFilterAsnLT(t *testing.T) {
n := int32(64512)
cf := CompileFilter(&pb.Filter{AsnNumber: &n, AsnOp: pb.StatusOp_LT})
if !MatchesFilter(Tuple6{ASN: 1000}, cf) {
t.Fatal("LT should match smaller ASN")
}
if MatchesFilter(Tuple6{ASN: 64512}, cf) {
t.Fatal("LT should not match equal ASN")
}
if MatchesFilter(Tuple6{ASN: 65535}, cf) {
t.Fatal("LT should not match larger ASN")
}
}
func TestDimensionLabelASN(t *testing.T) {
got := DimensionLabel(Tuple6{ASN: 12345}, pb.GroupBy_ASN_NUMBER)
if got != "12345" {
t.Errorf("DimensionLabel ASN: got %q, want %q", got, "12345")
}
}