nginx-logtail/cmd/aggregator/cache.go

package main

import (
	"context"
	"sync"
	"time"

	st "git.ipng.ch/ipng/nginx-logtail/internal/store"
	pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
)

// Cache holds the tiered ring buffers for the aggregator and answers TopN,
// Trend, and StreamSnapshots queries from them. It is populated by a
// 1-minute ticker that snapshots the current merged view from the Merger.
//
// Tick-based (not snapshot-triggered) so the ring stays on the same 1-minute
// cadence as the collectors, regardless of how many collectors are connected.
type Cache struct {
	source string
	merger *Merger

	mu           sync.RWMutex
	fineRing     [st.FineRingSize]st.Snapshot
	fineHead     int
	fineFilled   int
	coarseRing   [st.CoarseRingSize]st.Snapshot
	coarseHead   int
	coarseFilled int
	fineTick     int

	subMu sync.Mutex
	subs  map[chan st.Snapshot]struct{}
}

func NewCache(merger *Merger, source string) *Cache {
	return &Cache{
		merger: merger,
		source: source,
		subs:   make(map[chan st.Snapshot]struct{}),
	}
}

// Run drives the 1-minute rotation ticker. Blocks until ctx is cancelled.
func (c *Cache) Run(ctx context.Context) {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case t := <-ticker.C:
			c.rotate(t)
		}
	}
}

func (c *Cache) rotate(now time.Time) {
	fine := st.Snapshot{Timestamp: now, Entries: c.merger.TopK(st.FineTopK)}

	c.mu.Lock()
	c.fineRing[c.fineHead] = fine
	c.fineHead = (c.fineHead + 1) % st.FineRingSize
	if c.fineFilled < st.FineRingSize {
		c.fineFilled++
	}
	c.fineTick++
	if c.fineTick >= st.CoarseEvery {
		c.fineTick = 0
		coarse := c.mergeFineBuckets(now)
		c.coarseRing[c.coarseHead] = coarse
		c.coarseHead = (c.coarseHead + 1) % st.CoarseRingSize
		if c.coarseFilled < st.CoarseRingSize {
			c.coarseFilled++
		}
	}
	c.mu.Unlock()

	c.broadcast(fine)
}

func (c *Cache) mergeFineBuckets(now time.Time) st.Snapshot {
	merged := make(map[string]int64)
	count := min(st.CoarseEvery, c.fineFilled)
	for i := 0; i < count; i++ {
		idx := (c.fineHead - 1 - i + st.FineRingSize) % st.FineRingSize
		for _, e := range c.fineRing[idx].Entries {
			merged[e.Label] += e.Count
		}
	}
	return st.Snapshot{Timestamp: now, Entries: st.TopKFromMap(merged, st.CoarseTopK)}
}

// QueryTopN answers a TopN request from the ring buffers.
func (c *Cache) QueryTopN(filter *pb.Filter, groupBy pb.GroupBy, n int, window pb.Window) []st.Entry {
	cf := st.CompileFilter(filter)
	c.mu.RLock()
	defer c.mu.RUnlock()

	buckets, count := st.BucketsForWindow(window, c.fineView(), c.coarseView(), c.fineFilled, c.coarseFilled)
	grouped := make(map[string]int64)
	for i := 0; i < count; i++ {
		idx := (buckets.Head - 1 - i + buckets.Size) % buckets.Size
		for _, e := range buckets.Ring[idx].Entries {
			t := st.LabelTuple(e.Label)
			if !st.MatchesFilter(t, cf) {
				continue
			}
			grouped[st.DimensionLabel(t, groupBy)] += e.Count
		}
	}
	return st.TopKFromMap(grouped, n)
}

// QueryTrend answers a Trend request from the ring buffers.
func (c *Cache) QueryTrend(filter *pb.Filter, window pb.Window) []st.TrendPoint {
	cf := st.CompileFilter(filter)
	c.mu.RLock()
	defer c.mu.RUnlock()

	buckets, count := st.BucketsForWindow(window, c.fineView(), c.coarseView(), c.fineFilled, c.coarseFilled)
	points := make([]st.TrendPoint, count)
	for i := 0; i < count; i++ {
		idx := (buckets.Head - count + i + buckets.Size) % buckets.Size
		snap := buckets.Ring[idx]
		var total int64
		for _, e := range snap.Entries {
			if st.MatchesFilter(st.LabelTuple(e.Label), cf) {
				total += e.Count
			}
		}
		points[i] = st.TrendPoint{Timestamp: snap.Timestamp, Count: total}
	}
	return points
}

func (c *Cache) fineView() st.RingView {
	ring := make([]st.Snapshot, st.FineRingSize)
	copy(ring, c.fineRing[:])
	return st.RingView{Ring: ring, Head: c.fineHead, Size: st.FineRingSize}
}

func (c *Cache) coarseView() st.RingView {
	ring := make([]st.Snapshot, st.CoarseRingSize)
	copy(ring, c.coarseRing[:])
	return st.RingView{Ring: ring, Head: c.coarseHead, Size: st.CoarseRingSize}
}

func (c *Cache) Subscribe() chan st.Snapshot {
	ch := make(chan st.Snapshot, 4)
	c.subMu.Lock()
	c.subs[ch] = struct{}{}
	c.subMu.Unlock()
	return ch
}

func (c *Cache) Unsubscribe(ch chan st.Snapshot) {
	c.subMu.Lock()
	delete(c.subs, ch)
	c.subMu.Unlock()
	close(ch)
}

func (c *Cache) broadcast(snap st.Snapshot) {
	c.subMu.Lock()
	defer c.subMu.Unlock()
	for ch := range c.subs {
		select {
		case ch <- snap:
		default:
		}
	}
}