Add aggregator backfill, pulling fine+coarse buckets from collectors

cmd/collector/server.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"time"
 
+	st "git.ipng.ch/ipng/nginx-logtail/internal/store"
 	pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -70,6 +71,33 @@ func peerAddr(ctx context.Context) string {
 	return "unknown"
 }
 
+func (srv *Server) DumpSnapshots(_ *pb.DumpSnapshotsRequest, stream grpc.ServerStreamingServer[pb.Snapshot]) error {
+	fine, coarse := srv.store.DumpRings()
+	for _, snap := range fine {
+		if err := stream.Send(storeSnapshotToProto(snap, srv.source, false)); err != nil {
+			return err
+		}
+	}
+	for _, snap := range coarse {
+		if err := stream.Send(storeSnapshotToProto(snap, srv.source, true)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func storeSnapshotToProto(snap st.Snapshot, source string, isCoarse bool) *pb.Snapshot {
+	msg := &pb.Snapshot{
+		Source:    source,
+		Timestamp: snap.Timestamp.Unix(),
+		IsCoarse:  isCoarse,
+	}
+	for _, e := range snap.Entries {
+		msg.Entries = append(msg.Entries, &pb.TopNEntry{Label: e.Label, Count: e.Count})
+	}
+	return msg
+}
+
 func (srv *Server) StreamSnapshots(req *pb.SnapshotRequest, stream grpc.ServerStreamingServer[pb.Snapshot]) error {
 	ch := srv.store.Subscribe()
 	defer srv.store.Unsubscribe(ch)

cmd/collector/store.go

@@ -154,6 +154,32 @@ func (s *Store) coarseView() st.RingView {
 	return st.RingView{Ring: ring, Head: s.coarseHead, Size: st.CoarseRingSize}
 }
 
+// DumpRings returns copies of all non-empty fine and coarse ring snapshots in
+// chronological order. The lock is held only for the duration of the copy.
+func (s *Store) DumpRings() (fine, coarse []st.Snapshot) {
+	s.mu.RLock()
+	fineRing := s.fineRing
+	fineHead := s.fineHead
+	fineFilled := s.fineFilled
+	coarseRing := s.coarseRing
+	coarseHead := s.coarseHead
+	coarseFilled := s.coarseFilled
+	s.mu.RUnlock()
+
+	fine = make([]st.Snapshot, 0, fineFilled)
+	for i := 0; i < fineFilled; i++ {
+		idx := (fineHead - fineFilled + i + st.FineRingSize) % st.FineRingSize
+		fine = append(fine, fineRing[idx])
+	}
+
+	coarse = make([]st.Snapshot, 0, coarseFilled)
+	for i := 0; i < coarseFilled; i++ {
+		idx := (coarseHead - coarseFilled + i + st.CoarseRingSize) % st.CoarseRingSize
+		coarse = append(coarse, coarseRing[idx])
+	}
+	return fine, coarse
+}
+
 func (s *Store) Subscribe() chan st.Snapshot {
 	ch := make(chan st.Snapshot, 4)
 	s.subMu.Lock()