Execute PLAN_AGGREGATOR.md

cmd/aggregator/cache.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	st "git.ipng.ch/ipng/nginx-logtail/internal/store"
+	pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
+)
+
+// Cache holds the tiered ring buffers for the aggregator and answers TopN,
+// Trend, and StreamSnapshots queries from them. It is populated by a
+// 1-minute ticker that snapshots the current merged view from the Merger.
+//
+// Tick-based (not snapshot-triggered) so the ring stays on the same 1-minute
+// cadence as the collectors, regardless of how many collectors are connected.
+type Cache struct {
+	source string
+	merger *Merger
+
+	mu           sync.RWMutex
+	fineRing     [st.FineRingSize]st.Snapshot
+	fineHead     int
+	fineFilled   int
+	coarseRing   [st.CoarseRingSize]st.Snapshot
+	coarseHead   int
+	coarseFilled int
+	fineTick     int
+
+	subMu sync.Mutex
+	subs  map[chan st.Snapshot]struct{}
+}
+
+func NewCache(merger *Merger, source string) *Cache {
+	return &Cache{
+		merger: merger,
+		source: source,
+		subs:   make(map[chan st.Snapshot]struct{}),
+	}
+}
+
+// Run drives the 1-minute rotation ticker. Blocks until ctx is cancelled.
+func (c *Cache) Run(ctx context.Context) {
+	ticker := time.NewTicker(time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case t := <-ticker.C:
+			c.rotate(t)
+		}
+	}
+}
+
+func (c *Cache) rotate(now time.Time) {
+	fine := st.Snapshot{Timestamp: now, Entries: c.merger.TopK(st.FineTopK)}
+
+	c.mu.Lock()
+	c.fineRing[c.fineHead] = fine
+	c.fineHead = (c.fineHead + 1) % st.FineRingSize
+	if c.fineFilled < st.FineRingSize {
+		c.fineFilled++
+	}
+	c.fineTick++
+	if c.fineTick >= st.CoarseEvery {
+		c.fineTick = 0
+		coarse := c.mergeFineBuckets(now)
+		c.coarseRing[c.coarseHead] = coarse
+		c.coarseHead = (c.coarseHead + 1) % st.CoarseRingSize
+		if c.coarseFilled < st.CoarseRingSize {
+			c.coarseFilled++
+		}
+	}
+	c.mu.Unlock()
+
+	c.broadcast(fine)
+}
+
+func (c *Cache) mergeFineBuckets(now time.Time) st.Snapshot {
+	merged := make(map[string]int64)
+	count := min(st.CoarseEvery, c.fineFilled)
+	for i := 0; i < count; i++ {
+		idx := (c.fineHead - 1 - i + st.FineRingSize) % st.FineRingSize
+		for _, e := range c.fineRing[idx].Entries {
+			merged[e.Label] += e.Count
+		}
+	}
+	return st.Snapshot{Timestamp: now, Entries: st.TopKFromMap(merged, st.CoarseTopK)}
+}
+
+// QueryTopN answers a TopN request from the ring buffers.
+func (c *Cache) QueryTopN(filter *pb.Filter, groupBy pb.GroupBy, n int, window pb.Window) []st.Entry {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	buckets, count := st.BucketsForWindow(window, c.fineView(), c.coarseView(), c.fineFilled, c.coarseFilled)
+	grouped := make(map[string]int64)
+	for i := 0; i < count; i++ {
+		idx := (buckets.Head - 1 - i + buckets.Size) % buckets.Size
+		for _, e := range buckets.Ring[idx].Entries {
+			t := st.LabelTuple(e.Label)
+			if !st.MatchesFilter(t, filter) {
+				continue
+			}
+			grouped[st.DimensionLabel(t, groupBy)] += e.Count
+		}
+	}
+	return st.TopKFromMap(grouped, n)
+}
+
+// QueryTrend answers a Trend request from the ring buffers.
+func (c *Cache) QueryTrend(filter *pb.Filter, window pb.Window) []st.TrendPoint {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	buckets, count := st.BucketsForWindow(window, c.fineView(), c.coarseView(), c.fineFilled, c.coarseFilled)
+	points := make([]st.TrendPoint, count)
+	for i := 0; i < count; i++ {
+		idx := (buckets.Head - count + i + buckets.Size) % buckets.Size
+		snap := buckets.Ring[idx]
+		var total int64
+		for _, e := range snap.Entries {
+			if st.MatchesFilter(st.LabelTuple(e.Label), filter) {
+				total += e.Count
+			}
+		}
+		points[i] = st.TrendPoint{Timestamp: snap.Timestamp, Count: total}
+	}
+	return points
+}
+
+func (c *Cache) fineView() st.RingView {
+	ring := make([]st.Snapshot, st.FineRingSize)
+	copy(ring, c.fineRing[:])
+	return st.RingView{Ring: ring, Head: c.fineHead, Size: st.FineRingSize}
+}
+
+func (c *Cache) coarseView() st.RingView {
+	ring := make([]st.Snapshot, st.CoarseRingSize)
+	copy(ring, c.coarseRing[:])
+	return st.RingView{Ring: ring, Head: c.coarseHead, Size: st.CoarseRingSize}
+}
+
+func (c *Cache) Subscribe() chan st.Snapshot {
+	ch := make(chan st.Snapshot, 4)
+	c.subMu.Lock()
+	c.subs[ch] = struct{}{}
+	c.subMu.Unlock()
+	return ch
+}
+
+func (c *Cache) Unsubscribe(ch chan st.Snapshot) {
+	c.subMu.Lock()
+	delete(c.subs, ch)
+	c.subMu.Unlock()
+	close(ch)
+}
+
+func (c *Cache) broadcast(snap st.Snapshot) {
+	c.subMu.Lock()
+	defer c.subMu.Unlock()
+	for ch := range c.subs {
+		select {
+		case ch <- snap:
+		default:
+		}
+	}
+}