// Package store provides the shared ring-buffer, label-encoding and query // helpers used by both the collector and the aggregator. package store import ( "container/heap" "fmt" "time" pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb" ) // Ring-buffer dimensions — shared between collector and aggregator. const ( FineRingSize = 60 // 60 × 1-min buckets → 1 hour CoarseRingSize = 288 // 288 × 5-min buckets → 24 hours FineTopK = 50_000 // entries kept per fine snapshot CoarseTopK = 5_000 // entries kept per coarse snapshot CoarseEvery = 5 // fine ticks between coarse writes ) // Tuple4 is the four-dimensional aggregation key. type Tuple4 struct { Website string Prefix string URI string Status string } // Entry is a labelled count used in snapshots and query results. type Entry struct { Label string Count int64 } // Snapshot is one sorted (desc) slice of top-K entries for a time bucket. type Snapshot struct { Timestamp time.Time Entries []Entry } // TrendPoint is a (timestamp, total-count) pair for sparkline queries. type TrendPoint struct { Timestamp time.Time Count int64 } // RingView is a read-only snapshot of a ring buffer for iteration. type RingView struct { Ring []Snapshot Head int // index of next write slot (one past the latest entry) Size int } // BucketsForWindow returns the RingView and number of buckets to sum for window. func BucketsForWindow(window pb.Window, fine, coarse RingView, fineFilled, coarseFilled int) (RingView, int) { switch window { case pb.Window_W1M: return fine, min(1, fineFilled) case pb.Window_W5M: return fine, min(5, fineFilled) case pb.Window_W15M: return fine, min(15, fineFilled) case pb.Window_W60M: return fine, min(60, fineFilled) case pb.Window_W6H: return coarse, min(72, coarseFilled) // 72 × 5-min = 6 h case pb.Window_W24H: return coarse, min(288, coarseFilled) default: return fine, min(5, fineFilled) } } // --- label encoding: "website\x00prefix\x00uri\x00status" --- // EncodeTuple encodes a Tuple4 as a NUL-separated string suitable for use // as a map key in snapshots. func EncodeTuple(t Tuple4) string { return t.Website + "\x00" + t.Prefix + "\x00" + t.URI + "\x00" + t.Status } // LabelTuple decodes a NUL-separated snapshot label back into a Tuple4. func LabelTuple(label string) Tuple4 { parts := splitN(label, '\x00', 4) if len(parts) != 4 { return Tuple4{} } return Tuple4{parts[0], parts[1], parts[2], parts[3]} } func splitN(s string, sep byte, n int) []string { result := make([]string, 0, n) for len(result) < n-1 { i := indexOf(s, sep) if i < 0 { break } result = append(result, s[:i]) s = s[i+1:] } return append(result, s) } func indexOf(s string, b byte) int { for i := 0; i < len(s); i++ { if s[i] == b { return i } } return -1 } // --- filtering and grouping --- // MatchesFilter returns true if t satisfies all constraints in f. // A nil filter matches everything. func MatchesFilter(t Tuple4, f *pb.Filter) bool { if f == nil { return true } if f.Website != nil && t.Website != f.GetWebsite() { return false } if f.ClientPrefix != nil && t.Prefix != f.GetClientPrefix() { return false } if f.HttpRequestUri != nil && t.URI != f.GetHttpRequestUri() { return false } if f.HttpResponse != nil && t.Status != fmt.Sprint(f.GetHttpResponse()) { return false } return true } // DimensionLabel returns the string value of t for the given group-by dimension. func DimensionLabel(t Tuple4, g pb.GroupBy) string { switch g { case pb.GroupBy_WEBSITE: return t.Website case pb.GroupBy_CLIENT_PREFIX: return t.Prefix case pb.GroupBy_REQUEST_URI: return t.URI case pb.GroupBy_HTTP_RESPONSE: return t.Status default: return t.Website } } // --- heap-based top-K selection --- type entryHeap []Entry func (h entryHeap) Len() int { return len(h) } func (h entryHeap) Less(i, j int) bool { return h[i].Count < h[j].Count } // min-heap func (h entryHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h *entryHeap) Push(x interface{}) { *h = append(*h, x.(Entry)) } func (h *entryHeap) Pop() interface{} { old := *h n := len(old) x := old[n-1] *h = old[:n-1] return x } // TopKFromMap selects the top-k entries from a label→count map, sorted desc. func TopKFromMap(m map[string]int64, k int) []Entry { if k <= 0 { return nil } h := make(entryHeap, 0, k+1) for label, count := range m { heap.Push(&h, Entry{Label: label, Count: count}) if h.Len() > k { heap.Pop(&h) } } result := make([]Entry, h.Len()) for i := len(result) - 1; i >= 0; i-- { result[i] = heap.Pop(&h).(Entry) } return result } // TopKFromTupleMap encodes a Tuple4 map and returns the top-k as a Snapshot. // Used by the collector to snapshot its live map. func TopKFromTupleMap(m map[Tuple4]int64, k int, ts time.Time) Snapshot { flat := make(map[string]int64, len(m)) for t, c := range m { flat[EncodeTuple(t)] = c } return Snapshot{Timestamp: ts, Entries: TopKFromMap(flat, k)} }