Pim van Pelt
143aad9063
Build and release tooling:
- Makefile with help as default; targets: build/build-amd64/build-arm64,
test, lint, proto, pkg-deb, docker, docker-push, clean, plus
install-deps (+ three sub-targets for apt / Go toolchain / Go tools).
- internal/version package; -ldflags -X injects Version/Commit/Date into
every binary. -version flag on all four binaries (nginx-logtail version
for the CLI).
- Dockerfile takes VERSION/COMMIT/DATE build-args and forwards them.
- .deb output lands in build/; .gitignore ignores /build/.
Debian package:
- debian/build-deb.sh packages all four static binaries into a single
nginx-logtail_<ver>_<arch>.deb using dpkg-deb.
- Binary layout: /usr/sbin/nginx-logtail-{collector,aggregator,frontend}
and /usr/bin/nginx-logtail.
- nginx-logtail(8) manpage.
- Three systemd units (collector, aggregator, frontend) shipped under
/lib/systemd/system/. Installed but never enabled or started — the
operator opts in per host.
- Collector runs as _logtail:www-data (log access); aggregator and
frontend as _logtail:_logtail. postinst creates the system user/group
idempotently.
- Single shared env file /etc/default/nginx-logtail rendered from a
template at first install with %HOSTNAME% substituted. Sensible
defaults for every COLLECTOR_*, AGGREGATOR_*, FRONTEND_* variable;
plus COLLECTOR_ARGS / AGGREGATOR_ARGS / FRONTEND_ARGS escape hatches
appended to ExecStart. Not a dpkg conffile: operator edits survive
upgrades and dpkg --purge removes it.
Versioned UDP wire format:
- ParseUDPLine dispatches on a leading "v<N>\t" tag; v1 routes to the
existing 12-field parser. Unknown/missing versions fail closed so
future v2 parsers can land before emitters are upgraded.
- Tests updated; design.md FR-2.2 rewritten to make the version tag
normative.
Docs:
- README.md gains a Quick Start (Debian / Docker Compose / from source).
- user-guide.md rewritten around Installation and Configuration: full
env-var table, UDP-only default explained, precise file/UDP log_format
layouts, note that operators can emit "0" for unknown \$is_tor / \$asn.
- Drilldown cycle, frontend filter table, and CLI --group-by list all
include source_tag. UDP counters documented in the Prometheus section.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
94 lines
2.4 KiB
Go
94 lines
2.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"git.ipng.ch/ipng/nginx-logtail/internal/version"
|
|
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
func main() {
|
|
listen := flag.String("listen", envOr("AGGREGATOR_LISTEN", ":9091"), "gRPC listen address (env: AGGREGATOR_LISTEN)")
|
|
collectors := flag.String("collectors", envOr("AGGREGATOR_COLLECTORS", ""), "comma-separated collector host:port addresses (env: AGGREGATOR_COLLECTORS)")
|
|
source := flag.String("source", envOr("AGGREGATOR_SOURCE", hostname()), "name for this aggregator in responses (env: AGGREGATOR_SOURCE, default: hostname)")
|
|
showVersion := flag.Bool("version", false, "print version and exit")
|
|
flag.Parse()
|
|
|
|
if *showVersion {
|
|
fmt.Printf("aggregator %s\n", version.String())
|
|
return
|
|
}
|
|
|
|
if *collectors == "" {
|
|
log.Fatal("aggregator: --collectors / AGGREGATOR_COLLECTORS is required")
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
var collectorAddrs []string
|
|
for _, addr := range strings.Split(*collectors, ",") {
|
|
addr = strings.TrimSpace(addr)
|
|
if addr != "" {
|
|
collectorAddrs = append(collectorAddrs, addr)
|
|
}
|
|
}
|
|
|
|
merger := NewMerger()
|
|
cache := NewCache(merger, *source)
|
|
registry := NewTargetRegistry(collectorAddrs)
|
|
|
|
lis, err := net.Listen("tcp", *listen)
|
|
if err != nil {
|
|
log.Fatalf("aggregator: failed to listen on %s: %v", *listen, err)
|
|
}
|
|
grpcServer := grpc.NewServer()
|
|
pb.RegisterLogtailServiceServer(grpcServer, NewServer(cache, *source, registry))
|
|
|
|
go func() {
|
|
log.Printf("aggregator: gRPC listening on %s (source=%s)", *listen, *source)
|
|
if err := grpcServer.Serve(lis); err != nil {
|
|
log.Printf("aggregator: gRPC server stopped: %v", err)
|
|
}
|
|
}()
|
|
|
|
go cache.Run(ctx)
|
|
|
|
for _, addr := range collectorAddrs {
|
|
sub := NewCollectorSub(addr, merger, registry)
|
|
go sub.Run(ctx)
|
|
log.Printf("aggregator: subscribing to collector %s", addr)
|
|
}
|
|
|
|
log.Printf("aggregator: backfilling from %d collector(s)", len(collectorAddrs))
|
|
go Backfill(ctx, collectorAddrs, cache)
|
|
|
|
<-ctx.Done()
|
|
log.Printf("aggregator: shutting down")
|
|
grpcServer.GracefulStop()
|
|
}
|
|
|
|
func hostname() string {
|
|
h, err := os.Hostname()
|
|
if err != nil {
|
|
return "unknown"
|
|
}
|
|
return h
|
|
}
|
|
|
|
func envOr(key, def string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|