Execute PLAN_AGGREGATOR.md

internal/store/store.go

@@ -0,0 +1,196 @@
+// Package store provides the shared ring-buffer, label-encoding and query
+// helpers used by both the collector and the aggregator.
+package store
+
+import (
+	"container/heap"
+	"fmt"
+	"time"
+
+	pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
+)
+
+// Ring-buffer dimensions — shared between collector and aggregator.
+const (
+	FineRingSize   = 60      // 60 × 1-min buckets  → 1 hour
+	CoarseRingSize = 288     // 288 × 5-min buckets → 24 hours
+	FineTopK       = 50_000  // entries kept per fine snapshot
+	CoarseTopK     = 5_000   // entries kept per coarse snapshot
+	CoarseEvery    = 5       // fine ticks between coarse writes
+)
+
+// Tuple4 is the four-dimensional aggregation key.
+type Tuple4 struct {
+	Website string
+	Prefix  string
+	URI     string
+	Status  string
+}
+
+// Entry is a labelled count used in snapshots and query results.
+type Entry struct {
+	Label string
+	Count int64
+}
+
+// Snapshot is one sorted (desc) slice of top-K entries for a time bucket.
+type Snapshot struct {
+	Timestamp time.Time
+	Entries   []Entry
+}
+
+// TrendPoint is a (timestamp, total-count) pair for sparkline queries.
+type TrendPoint struct {
+	Timestamp time.Time
+	Count     int64
+}
+
+// RingView is a read-only snapshot of a ring buffer for iteration.
+type RingView struct {
+	Ring []Snapshot
+	Head int // index of next write slot (one past the latest entry)
+	Size int
+}
+
+// BucketsForWindow returns the RingView and number of buckets to sum for window.
+func BucketsForWindow(window pb.Window, fine, coarse RingView, fineFilled, coarseFilled int) (RingView, int) {
+	switch window {
+	case pb.Window_W1M:
+		return fine, min(1, fineFilled)
+	case pb.Window_W5M:
+		return fine, min(5, fineFilled)
+	case pb.Window_W15M:
+		return fine, min(15, fineFilled)
+	case pb.Window_W60M:
+		return fine, min(60, fineFilled)
+	case pb.Window_W6H:
+		return coarse, min(72, coarseFilled) // 72 × 5-min = 6 h
+	case pb.Window_W24H:
+		return coarse, min(288, coarseFilled)
+	default:
+		return fine, min(5, fineFilled)
+	}
+}
+
+// --- label encoding: "website\x00prefix\x00uri\x00status" ---
+
+// EncodeTuple encodes a Tuple4 as a NUL-separated string suitable for use
+// as a map key in snapshots.
+func EncodeTuple(t Tuple4) string {
+	return t.Website + "\x00" + t.Prefix + "\x00" + t.URI + "\x00" + t.Status
+}
+
+// LabelTuple decodes a NUL-separated snapshot label back into a Tuple4.
+func LabelTuple(label string) Tuple4 {
+	parts := splitN(label, '\x00', 4)
+	if len(parts) != 4 {
+		return Tuple4{}
+	}
+	return Tuple4{parts[0], parts[1], parts[2], parts[3]}
+}
+
+func splitN(s string, sep byte, n int) []string {
+	result := make([]string, 0, n)
+	for len(result) < n-1 {
+		i := indexOf(s, sep)
+		if i < 0 {
+			break
+		}
+		result = append(result, s[:i])
+		s = s[i+1:]
+	}
+	return append(result, s)
+}
+
+func indexOf(s string, b byte) int {
+	for i := 0; i < len(s); i++ {
+		if s[i] == b {
+			return i
+		}
+	}
+	return -1
+}
+
+// --- filtering and grouping ---
+
+// MatchesFilter returns true if t satisfies all constraints in f.
+// A nil filter matches everything.
+func MatchesFilter(t Tuple4, f *pb.Filter) bool {
+	if f == nil {
+		return true
+	}
+	if f.Website != nil && t.Website != f.GetWebsite() {
+		return false
+	}
+	if f.ClientPrefix != nil && t.Prefix != f.GetClientPrefix() {
+		return false
+	}
+	if f.HttpRequestUri != nil && t.URI != f.GetHttpRequestUri() {
+		return false
+	}
+	if f.HttpResponse != nil && t.Status != fmt.Sprint(f.GetHttpResponse()) {
+		return false
+	}
+	return true
+}
+
+// DimensionLabel returns the string value of t for the given group-by dimension.
+func DimensionLabel(t Tuple4, g pb.GroupBy) string {
+	switch g {
+	case pb.GroupBy_WEBSITE:
+		return t.Website
+	case pb.GroupBy_CLIENT_PREFIX:
+		return t.Prefix
+	case pb.GroupBy_REQUEST_URI:
+		return t.URI
+	case pb.GroupBy_HTTP_RESPONSE:
+		return t.Status
+	default:
+		return t.Website
+	}
+}
+
+// --- heap-based top-K selection ---
+
+type entryHeap []Entry
+
+func (h entryHeap) Len() int            { return len(h) }
+func (h entryHeap) Less(i, j int) bool  { return h[i].Count < h[j].Count } // min-heap
+func (h entryHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
+func (h *entryHeap) Push(x interface{}) { *h = append(*h, x.(Entry)) }
+func (h *entryHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[:n-1]
+	return x
+}
+
+// TopKFromMap selects the top-k entries from a label→count map, sorted desc.
+func TopKFromMap(m map[string]int64, k int) []Entry {
+	if k <= 0 {
+		return nil
+	}
+	h := make(entryHeap, 0, k+1)
+	for label, count := range m {
+		heap.Push(&h, Entry{Label: label, Count: count})
+		if h.Len() > k {
+			heap.Pop(&h)
+		}
+	}
+	result := make([]Entry, h.Len())
+	for i := len(result) - 1; i >= 0; i-- {
+		result[i] = heap.Pop(&h).(Entry)
+	}
+	return result
+}
+
+// TopKFromTupleMap encodes a Tuple4 map and returns the top-k as a Snapshot.
+// Used by the collector to snapshot its live map.
+func TopKFromTupleMap(m map[Tuple4]int64, k int, ts time.Time) Snapshot {
+	flat := make(map[string]int64, len(m))
+	for t, c := range m {
+		flat[EncodeTuple(t)] = c
+	}
+	return Snapshot{Timestamp: ts, Entries: TopKFromMap(flat, k)}
+}