Implement target selection, autodiscovery via aggregator, implement listTargets

cmd/aggregator/aggregator_test.go

@@ -287,8 +287,8 @@ func TestGRPCEndToEnd(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	go NewCollectorSub(addr1, merger).Run(ctx)
-	go NewCollectorSub(addr2, merger).Run(ctx)
+	go NewCollectorSub(addr1, merger, NewTargetRegistry(nil)).Run(ctx)
+	go NewCollectorSub(addr2, merger, NewTargetRegistry(nil)).Run(ctx)
 
 	// Wait for both snapshots to be applied.
 	deadline := time.Now().Add(3 * time.Second)
@@ -309,7 +309,7 @@ func TestGRPCEndToEnd(t *testing.T) {
 		t.Fatal(err)
 	}
 	grpcSrv := grpc.NewServer()
-	pb.RegisterLogtailServiceServer(grpcSrv, NewServer(cache, "agg-test"))
+	pb.RegisterLogtailServiceServer(grpcSrv, NewServer(cache, "agg-test", NewTargetRegistry(nil)))
 	go grpcSrv.Serve(lis)
 	defer grpcSrv.Stop()
 
@@ -399,8 +399,8 @@ func TestDegradedCollector(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	go NewCollectorSub(addr1, merger).Run(ctx)
-	go NewCollectorSub(addr2, merger).Run(ctx)
+	go NewCollectorSub(addr1, merger, NewTargetRegistry(nil)).Run(ctx)
+	go NewCollectorSub(addr2, merger, NewTargetRegistry(nil)).Run(ctx)
 
 	// Wait for col1's data to appear.
 	deadline := time.Now().Add(3 * time.Second)

cmd/aggregator/main.go

@@ -27,16 +27,21 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	merger := NewMerger()
-	cache := NewCache(merger, *source)
-	go cache.Run(ctx)
-
+	var collectorAddrs []string
 	for _, addr := range strings.Split(*collectors, ",") {
 		addr = strings.TrimSpace(addr)
-		if addr == "" {
-			continue
+		if addr != "" {
+			collectorAddrs = append(collectorAddrs, addr)
 		}
-		sub := NewCollectorSub(addr, merger)
+	}
+
+	merger := NewMerger()
+	cache := NewCache(merger, *source)
+	registry := NewTargetRegistry(collectorAddrs)
+	go cache.Run(ctx)
+
+	for _, addr := range collectorAddrs {
+		sub := NewCollectorSub(addr, merger, registry)
 		go sub.Run(ctx)
 		log.Printf("aggregator: subscribing to collector %s", addr)
 	}
@@ -46,7 +51,7 @@ func main() {
 		log.Fatalf("aggregator: failed to listen on %s: %v", *listen, err)
 	}
 	grpcServer := grpc.NewServer()
-	pb.RegisterLogtailServiceServer(grpcServer, NewServer(cache, *source))
+	pb.RegisterLogtailServiceServer(grpcServer, NewServer(cache, *source, registry))
 
 	go func() {
 		log.Printf("aggregator: gRPC listening on %s (source=%s)", *listen, *source)

cmd/aggregator/registry.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"sort"
+	"sync"
+)
+
+// TargetInfo holds the display name and gRPC address of one collector target.
+type TargetInfo struct {
+	Name string // collector --source value, falls back to addr until first snapshot
+	Addr string // configured gRPC address
+}
+
+// TargetRegistry tracks addr → display name for all configured collectors.
+// Names default to the addr and are updated to the collector's --source value
+// when the first snapshot arrives.
+type TargetRegistry struct {
+	mu    sync.RWMutex
+	names map[string]string // addr → current name
+}
+
+func NewTargetRegistry(addrs []string) *TargetRegistry {
+	names := make(map[string]string, len(addrs))
+	for _, a := range addrs {
+		names[a] = a // default until first snapshot
+	}
+	return &TargetRegistry{names: names}
+}
+
+// SetName updates the display name for addr (called when a snapshot arrives).
+func (r *TargetRegistry) SetName(addr, name string) {
+	r.mu.Lock()
+	r.names[addr] = name
+	r.mu.Unlock()
+}
+
+// Targets returns all registered targets sorted by addr.
+func (r *TargetRegistry) Targets() []TargetInfo {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	out := make([]TargetInfo, 0, len(r.names))
+	for addr, name := range r.names {
+		out = append(out, TargetInfo{Name: name, Addr: addr})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Addr < out[j].Addr })
+	return out
+}

cmd/aggregator/server.go

@@ -14,12 +14,13 @@ import (
 // Server implements pb.LogtailServiceServer backed by the aggregator Cache.
 type Server struct {
 	pb.UnimplementedLogtailServiceServer
-	cache  *Cache
-	source string
+	cache    *Cache
+	source   string
+	registry *TargetRegistry
 }
 
-func NewServer(cache *Cache, source string) *Server {
-	return &Server{cache: cache, source: source}
+func NewServer(cache *Cache, source string, registry *TargetRegistry) *Server {
+	return &Server{cache: cache, source: source, registry: registry}
 }
 
 func (srv *Server) TopN(_ context.Context, req *pb.TopNRequest) (*pb.TopNResponse, error) {
@@ -53,6 +54,16 @@ func (srv *Server) Trend(_ context.Context, req *pb.TrendRequest) (*pb.TrendResp
 	return resp, nil
 }
 
+func (srv *Server) ListTargets(_ context.Context, _ *pb.ListTargetsRequest) (*pb.ListTargetsResponse, error) {
+	resp := &pb.ListTargetsResponse{}
+	if srv.registry != nil {
+		for _, t := range srv.registry.Targets() {
+			resp.Targets = append(resp.Targets, &pb.TargetInfo{Name: t.Name, Addr: t.Addr})
+		}
+	}
+	return resp, nil
+}
+
 func (srv *Server) StreamSnapshots(_ *pb.SnapshotRequest, stream grpc.ServerStreamingServer[pb.Snapshot]) error {
 	ch := srv.cache.Subscribe()
 	defer srv.cache.Unsubscribe(ch)

cmd/aggregator/subscriber.go

@@ -15,12 +15,13 @@ import (
 // the collector degraded (zeroing its contribution) after 3 consecutive
 // failures.
 type CollectorSub struct {
-	addr   string
-	merger *Merger
+	addr     string
+	merger   *Merger
+	registry *TargetRegistry
 }
 
-func NewCollectorSub(addr string, merger *Merger) *CollectorSub {
-	return &CollectorSub{addr: addr, merger: merger}
+func NewCollectorSub(addr string, merger *Merger, registry *TargetRegistry) *CollectorSub {
+	return &CollectorSub{addr: addr, merger: merger, registry: registry}
 }
 
 // Run blocks until ctx is cancelled.
@@ -92,6 +93,7 @@ func (cs *CollectorSub) stream(ctx context.Context) (bool, error) {
 			return gotOne, err
 		}
 		gotOne = true
+		cs.registry.SetName(cs.addr, snap.Source)
 		cs.merger.Apply(snap)
 	}
 }

cmd/cli/cmd_targets.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
+)
+
+func runTargets(args []string) {
+	fs := flag.NewFlagSet("targets", flag.ExitOnError)
+	sf, target := bindShared(fs)
+	fs.Usage = func() {
+		fmt.Fprintln(os.Stderr, "usage: logtail-cli targets [--target host:port] [--json]")
+		fs.PrintDefaults()
+	}
+	fs.Parse(args)
+	sf.resolve(*target)
+
+	for _, addr := range sf.targets {
+		conn, client, err := dial(addr)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "targets: cannot connect to %s: %v\n", addr, err)
+			continue
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		resp, err := client.ListTargets(ctx, &pb.ListTargetsRequest{})
+		cancel()
+		conn.Close()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "targets: %s: %v\n", addr, err)
+			continue
+		}
+
+		if sf.jsonOut {
+			type row struct {
+				QueryTarget string `json:"query_target"`
+				Name        string `json:"name"`
+				Addr        string `json:"addr"`
+			}
+			for _, t := range resp.Targets {
+				json.NewEncoder(os.Stdout).Encode(row{QueryTarget: addr, Name: t.Name, Addr: t.Addr})
+			}
+		} else {
+			if len(sf.targets) > 1 {
+				fmt.Println(targetHeader(addr, "", len(sf.targets)))
+			}
+			for _, t := range resp.Targets {
+				addrCol := t.Addr
+				if addrCol == "" {
+					addrCol = "(self)"
+				}
+				fmt.Printf("%-40s  %s\n", t.Name, addrCol)
+			}
+		}
+	}
+}

cmd/cli/main.go

@@ -11,6 +11,7 @@ Usage:
   logtail-cli topn    [flags]   ranked label → count list
   logtail-cli trend   [flags]   per-minute time series
   logtail-cli stream  [flags]   live snapshot feed
+  logtail-cli targets [flags]   list targets known to the queried endpoint
 
 Subcommand flags (all subcommands):
   --target  host:port[,host:port,...]  endpoints to query (default: localhost:9090)
@@ -43,6 +44,8 @@ func main() {
 		runTrend(os.Args[2:])
 	case "stream":
 		runStream(os.Args[2:])
+	case "targets":
+		runTargets(os.Args[2:])
 	case "-h", "--help", "help":
 		fmt.Print(usage)
 	default:

cmd/collector/server.go

@@ -56,6 +56,12 @@ func (srv *Server) Trend(_ context.Context, req *pb.TrendRequest) (*pb.TrendResp
 	return resp, nil
 }
 
+func (srv *Server) ListTargets(_ context.Context, _ *pb.ListTargetsRequest) (*pb.ListTargetsResponse, error) {
+	return &pb.ListTargetsResponse{
+		Targets: []*pb.TargetInfo{{Name: srv.source, Addr: ""}},
+	}, nil
+}
+
 func (srv *Server) StreamSnapshots(req *pb.SnapshotRequest, stream grpc.ServerStreamingServer[pb.Snapshot]) error {
 	ch := srv.store.Subscribe()
 	defer srv.store.Unsubscribe(ch)

cmd/frontend/handler.go

@@ -76,6 +76,7 @@ type PageData struct {
 	Breadcrumbs    []Crumb
 	Windows        []Tab
 	GroupBys       []Tab
+	Targets        []Tab // source/target picker; empty when only one target available
 	RefreshSecs    int
 	Error          string
 	FilterExpr     string // current filter serialised to mini-language for the input box
@@ -340,6 +341,38 @@ func buildGroupByTabs(p QueryParams) []Tab {
 	return tabs
 }
 
+// buildTargetTabs builds the source/target picker tabs from a ListTargets response.
+// Returns nil (hide picker) when only one endpoint is reachable.
+func (h *Handler) buildTargetTabs(p QueryParams, lt *pb.ListTargetsResponse) []Tab {
+	// "all" always points at the configured aggregator default.
+	allTab := Tab{
+		Label:  "all",
+		URL:    p.buildURL(map[string]string{"target": h.defaultTarget}),
+		Active: p.Target == h.defaultTarget,
+	}
+
+	var collectorTabs []Tab
+	if lt != nil {
+		for _, t := range lt.Targets {
+			addr := t.Addr
+			if addr == "" {
+				addr = p.Target // collector reporting itself; addr is the current target
+			}
+			collectorTabs = append(collectorTabs, Tab{
+				Label:  t.Name,
+				URL:    p.buildURL(map[string]string{"target": addr}),
+				Active: p.Target == addr,
+			})
+		}
+	}
+
+	// Only render the picker when there is more than one choice.
+	if len(collectorTabs) == 0 {
+		return nil
+	}
+	return append([]Tab{allTab}, collectorTabs...)
+}
+
 func buildTableRows(entries []*pb.TopNEntry, p QueryParams) ([]TableRow, int64) {
 	if len(entries) == 0 {
 		return nil, 0
@@ -410,6 +443,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	topNCh := make(chan topNResult, 1)
 	trendCh := make(chan trendResult, 1)
+	ltCh := make(chan *pb.ListTargetsResponse, 1)
 
 	go func() {
 		resp, err := client.TopN(ctx, &pb.TopNRequest{
@@ -427,9 +461,18 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		})
 		trendCh <- trendResult{resp, err}
 	}()
+	go func() {
+		resp, err := client.ListTargets(ctx, &pb.ListTargetsRequest{})
+		if err != nil {
+			ltCh <- nil
+		} else {
+			ltCh <- resp
+		}
+	}()
 
 	tn := <-topNCh
 	tr := <-trendCh
+	lt := <-ltCh
 
 	if tn.err != nil {
 		h.render(w, http.StatusBadGateway, h.errorPage(params,
@@ -459,6 +502,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Breadcrumbs:    buildCrumbs(params),
 		Windows:        buildWindowTabs(params),
 		GroupBys:       buildGroupByTabs(params),
+		Targets:        h.buildTargetTabs(params, lt),
 		RefreshSecs:    h.refreshSecs,
 		FilterExpr:     filterExprInput,
 		FilterErr:      filterErr,

cmd/frontend/templates/base.html

@@ -34,6 +34,8 @@ a:hover { text-decoration: underline; }
 .error { color: #c00; border: 1px solid #fbb; background: #fff5f5; padding: 0.7em 1em; margin: 1em 0; border-radius: 3px; }
 .nodata { color: #999; margin: 2em 0; font-style: italic; }
 footer { margin-top: 2em; padding-top: 0.6em; border-top: 1px solid #e0e0e0; font-size: 0.8em; color: #999; }
+.tabs-targets { margin-top: -0.4em; }
+.tabs-label { font-size: 0.85em; color: #888; margin-right: 0.2em; align-self: center; }
 .filter-form { display: flex; gap: 0.4em; align-items: center; margin-bottom: 0.7em; }
 .filter-input { flex: 1; font-family: monospace; font-size: 13px; padding: 0.25em 0.5em; border: 1px solid #aaa; }
 .filter-form button { padding: 0.25em 0.8em; border: 1px solid #aaa; background: #f4f4f4; cursor: pointer; font-family: monospace; }

cmd/frontend/templates/index.html

@@ -13,6 +13,13 @@
 {{- end}}
 </div>
 
+{{if .Targets}}<div class="tabs tabs-targets">
+  <span class="tabs-label">source:</span>
+{{- range .Targets}}
+  <a href="{{.URL}}"{{if .Active}} class="active"{{end}}>{{.Label}}</a>
+{{- end}}
+</div>{{end}}
+
 <form class="filter-form" method="get" action="/">
   <input type="hidden" name="target" value="{{.Params.Target}}">
   <input type="hidden" name="w" value="{{.Params.WindowS}}">