Implement filter in status, website and uri in CLI and Frontend
This commit is contained in:
@@ -4,7 +4,8 @@ package store
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
|
||||
@@ -113,27 +114,101 @@ func indexOf(s string, b byte) int {
|
||||
|
||||
// --- filtering and grouping ---
|
||||
|
||||
// CompiledFilter wraps a pb.Filter with pre-compiled regular expressions.
|
||||
// Use CompileFilter to construct one before a query loop.
|
||||
type CompiledFilter struct {
|
||||
Proto *pb.Filter
|
||||
WebsiteRe *regexp.Regexp // nil if no website_regex or compilation failed
|
||||
URIRe *regexp.Regexp // nil if no uri_regex or compilation failed
|
||||
}
|
||||
|
||||
// CompileFilter compiles the regex fields in f once. Invalid regexes are
|
||||
// logged and treated as "match nothing" for that field.
|
||||
func CompileFilter(f *pb.Filter) *CompiledFilter {
|
||||
cf := &CompiledFilter{Proto: f}
|
||||
if f == nil {
|
||||
return cf
|
||||
}
|
||||
if f.WebsiteRegex != nil {
|
||||
re, err := regexp.Compile(f.GetWebsiteRegex())
|
||||
if err != nil {
|
||||
log.Printf("store: invalid website_regex %q: %v", f.GetWebsiteRegex(), err)
|
||||
} else {
|
||||
cf.WebsiteRe = re
|
||||
}
|
||||
}
|
||||
if f.UriRegex != nil {
|
||||
re, err := regexp.Compile(f.GetUriRegex())
|
||||
if err != nil {
|
||||
log.Printf("store: invalid uri_regex %q: %v", f.GetUriRegex(), err)
|
||||
} else {
|
||||
cf.URIRe = re
|
||||
}
|
||||
}
|
||||
return cf
|
||||
}
|
||||
|
||||
// MatchesFilter returns true if t satisfies all constraints in f.
|
||||
// A nil filter matches everything.
|
||||
func MatchesFilter(t Tuple4, f *pb.Filter) bool {
|
||||
if f == nil {
|
||||
func MatchesFilter(t Tuple4, f *CompiledFilter) bool {
|
||||
if f == nil || f.Proto == nil {
|
||||
return true
|
||||
}
|
||||
if f.Website != nil && t.Website != f.GetWebsite() {
|
||||
p := f.Proto
|
||||
if p.Website != nil && t.Website != p.GetWebsite() {
|
||||
return false
|
||||
}
|
||||
if f.ClientPrefix != nil && t.Prefix != f.GetClientPrefix() {
|
||||
if f.WebsiteRe != nil && !f.WebsiteRe.MatchString(t.Website) {
|
||||
return false
|
||||
}
|
||||
if f.HttpRequestUri != nil && t.URI != f.GetHttpRequestUri() {
|
||||
// website_regex set but failed to compile → match nothing
|
||||
if p.WebsiteRegex != nil && f.WebsiteRe == nil {
|
||||
return false
|
||||
}
|
||||
if f.HttpResponse != nil && t.Status != fmt.Sprint(f.GetHttpResponse()) {
|
||||
if p.ClientPrefix != nil && t.Prefix != p.GetClientPrefix() {
|
||||
return false
|
||||
}
|
||||
if p.HttpRequestUri != nil && t.URI != p.GetHttpRequestUri() {
|
||||
return false
|
||||
}
|
||||
if f.URIRe != nil && !f.URIRe.MatchString(t.URI) {
|
||||
return false
|
||||
}
|
||||
if p.UriRegex != nil && f.URIRe == nil {
|
||||
return false
|
||||
}
|
||||
if p.HttpResponse != nil && !matchesStatusOp(t.Status, p.GetHttpResponse(), p.StatusOp) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// matchesStatusOp applies op(statusStr, want), parsing statusStr as an integer.
|
||||
// Returns false if statusStr is not a valid integer.
|
||||
func matchesStatusOp(statusStr string, want int32, op pb.StatusOp) bool {
|
||||
var got int32
|
||||
for _, c := range []byte(statusStr) {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
got = got*10 + int32(c-'0')
|
||||
}
|
||||
switch op {
|
||||
case pb.StatusOp_NE:
|
||||
return got != want
|
||||
case pb.StatusOp_GT:
|
||||
return got > want
|
||||
case pb.StatusOp_GE:
|
||||
return got >= want
|
||||
case pb.StatusOp_LT:
|
||||
return got < want
|
||||
case pb.StatusOp_LE:
|
||||
return got <= want
|
||||
default: // EQ
|
||||
return got == want
|
||||
}
|
||||
}
|
||||
|
||||
// DimensionLabel returns the string value of t for the given group-by dimension.
|
||||
func DimensionLabel(t Tuple4, g pb.GroupBy) string {
|
||||
switch g {
|
||||
@@ -150,6 +225,45 @@ func DimensionLabel(t Tuple4, g pb.GroupBy) string {
|
||||
}
|
||||
}
|
||||
|
||||
// ParseStatusExpr parses a status filter expression into a value and operator.
|
||||
// Accepted syntax: 200, =200, ==200, !=200, >400, >=400, <500, <=500.
|
||||
// Returns ok=false if the expression is empty or unparseable.
|
||||
func ParseStatusExpr(s string) (value int32, op pb.StatusOp, ok bool) {
|
||||
if s == "" {
|
||||
return 0, pb.StatusOp_EQ, false
|
||||
}
|
||||
var digits string
|
||||
switch {
|
||||
case len(s) >= 2 && s[:2] == "!=":
|
||||
op, digits = pb.StatusOp_NE, s[2:]
|
||||
case len(s) >= 2 && s[:2] == ">=":
|
||||
op, digits = pb.StatusOp_GE, s[2:]
|
||||
case len(s) >= 2 && s[:2] == "<=":
|
||||
op, digits = pb.StatusOp_LE, s[2:]
|
||||
case len(s) >= 2 && s[:2] == "==":
|
||||
op, digits = pb.StatusOp_EQ, s[2:]
|
||||
case s[0] == '>':
|
||||
op, digits = pb.StatusOp_GT, s[1:]
|
||||
case s[0] == '<':
|
||||
op, digits = pb.StatusOp_LT, s[1:]
|
||||
case s[0] == '=':
|
||||
op, digits = pb.StatusOp_EQ, s[1:]
|
||||
default:
|
||||
op, digits = pb.StatusOp_EQ, s
|
||||
}
|
||||
var n int32
|
||||
if digits == "" {
|
||||
return 0, pb.StatusOp_EQ, false
|
||||
}
|
||||
for _, c := range []byte(digits) {
|
||||
if c < '0' || c > '9' {
|
||||
return 0, pb.StatusOp_EQ, false
|
||||
}
|
||||
n = n*10 + int32(c-'0')
|
||||
}
|
||||
return n, op, true
|
||||
}
|
||||
|
||||
// --- heap-based top-K selection ---
|
||||
|
||||
type entryHeap []Entry
|
||||
|
||||
205
internal/store/store_test.go
Normal file
205
internal/store/store_test.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
|
||||
)
|
||||
|
||||
// --- ParseStatusExpr ---
|
||||
|
||||
func TestParseStatusExprEQ(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr("200")
|
||||
if !ok || n != 200 || op != pb.StatusOp_EQ {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprExplicitEQ(t *testing.T) {
|
||||
for _, expr := range []string{"=200", "==200"} {
|
||||
n, op, ok := ParseStatusExpr(expr)
|
||||
if !ok || n != 200 || op != pb.StatusOp_EQ {
|
||||
t.Fatalf("expr %q: got (%d,%v,%v)", expr, n, op, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprNE(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr("!=200")
|
||||
if !ok || n != 200 || op != pb.StatusOp_NE {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprGE(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr(">=400")
|
||||
if !ok || n != 400 || op != pb.StatusOp_GE {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprGT(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr(">400")
|
||||
if !ok || n != 400 || op != pb.StatusOp_GT {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprLE(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr("<=500")
|
||||
if !ok || n != 500 || op != pb.StatusOp_LE {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprLT(t *testing.T) {
|
||||
n, op, ok := ParseStatusExpr("<500")
|
||||
if !ok || n != 500 || op != pb.StatusOp_LT {
|
||||
t.Fatalf("got (%d,%v,%v)", n, op, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprEmpty(t *testing.T) {
|
||||
_, _, ok := ParseStatusExpr("")
|
||||
if ok {
|
||||
t.Fatal("expected ok=false for empty string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStatusExprInvalid(t *testing.T) {
|
||||
for _, expr := range []string{"abc", "!=", ">=", "2xx"} {
|
||||
_, _, ok := ParseStatusExpr(expr)
|
||||
if ok {
|
||||
t.Fatalf("expr %q: expected ok=false", expr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- MatchesFilter ---
|
||||
|
||||
func compiledEQ(status int32) *CompiledFilter {
|
||||
v := status
|
||||
return CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_EQ})
|
||||
}
|
||||
|
||||
func TestMatchesFilterNil(t *testing.T) {
|
||||
if !MatchesFilter(Tuple4{Website: "x"}, nil) {
|
||||
t.Fatal("nil filter should match everything")
|
||||
}
|
||||
if !MatchesFilter(Tuple4{Website: "x"}, &CompiledFilter{}) {
|
||||
t.Fatal("empty compiled filter should match everything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterExactWebsite(t *testing.T) {
|
||||
w := "example.com"
|
||||
cf := CompileFilter(&pb.Filter{Website: &w})
|
||||
if !MatchesFilter(Tuple4{Website: "example.com"}, cf) {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Website: "other.com"}, cf) {
|
||||
t.Fatal("expected no match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterWebsiteRegex(t *testing.T) {
|
||||
re := "gouda.*"
|
||||
cf := CompileFilter(&pb.Filter{WebsiteRegex: &re})
|
||||
if !MatchesFilter(Tuple4{Website: "gouda.example.com"}, cf) {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Website: "edam.example.com"}, cf) {
|
||||
t.Fatal("expected no match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterURIRegex(t *testing.T) {
|
||||
re := "^/api/.*"
|
||||
cf := CompileFilter(&pb.Filter{UriRegex: &re})
|
||||
if !MatchesFilter(Tuple4{URI: "/api/users"}, cf) {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if MatchesFilter(Tuple4{URI: "/health"}, cf) {
|
||||
t.Fatal("expected no match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterInvalidRegexMatchesNothing(t *testing.T) {
|
||||
re := "[invalid"
|
||||
cf := CompileFilter(&pb.Filter{WebsiteRegex: &re})
|
||||
if MatchesFilter(Tuple4{Website: "anything"}, cf) {
|
||||
t.Fatal("invalid regex should match nothing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterStatusEQ(t *testing.T) {
|
||||
cf := compiledEQ(200)
|
||||
if !MatchesFilter(Tuple4{Status: "200"}, cf) {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Status: "404"}, cf) {
|
||||
t.Fatal("expected no match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterStatusNE(t *testing.T) {
|
||||
v := int32(200)
|
||||
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_NE})
|
||||
if MatchesFilter(Tuple4{Status: "200"}, cf) {
|
||||
t.Fatal("expected no match for 200 != 200")
|
||||
}
|
||||
if !MatchesFilter(Tuple4{Status: "404"}, cf) {
|
||||
t.Fatal("expected match for 404 != 200")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterStatusGE(t *testing.T) {
|
||||
v := int32(400)
|
||||
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_GE})
|
||||
if !MatchesFilter(Tuple4{Status: "400"}, cf) {
|
||||
t.Fatal("expected match: 400 >= 400")
|
||||
}
|
||||
if !MatchesFilter(Tuple4{Status: "500"}, cf) {
|
||||
t.Fatal("expected match: 500 >= 400")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Status: "200"}, cf) {
|
||||
t.Fatal("expected no match: 200 >= 400")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterStatusLT(t *testing.T) {
|
||||
v := int32(400)
|
||||
cf := CompileFilter(&pb.Filter{HttpResponse: &v, StatusOp: pb.StatusOp_LT})
|
||||
if !MatchesFilter(Tuple4{Status: "200"}, cf) {
|
||||
t.Fatal("expected match: 200 < 400")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Status: "400"}, cf) {
|
||||
t.Fatal("expected no match: 400 < 400")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterStatusNonNumeric(t *testing.T) {
|
||||
cf := compiledEQ(200)
|
||||
if MatchesFilter(Tuple4{Status: "ok"}, cf) {
|
||||
t.Fatal("non-numeric status should not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterCombined(t *testing.T) {
|
||||
w := "example.com"
|
||||
v := int32(200)
|
||||
cf := CompileFilter(&pb.Filter{
|
||||
Website: &w,
|
||||
HttpResponse: &v,
|
||||
StatusOp: pb.StatusOp_EQ,
|
||||
})
|
||||
if !MatchesFilter(Tuple4{Website: "example.com", Status: "200"}, cf) {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Website: "other.com", Status: "200"}, cf) {
|
||||
t.Fatal("expected no match: wrong website")
|
||||
}
|
||||
if MatchesFilter(Tuple4{Website: "example.com", Status: "404"}, cf) {
|
||||
t.Fatal("expected no match: wrong status")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user