From d2dcd88c4b6004761889596237af90e7a10b0a26 Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Wed, 25 Mar 2026 06:41:13 +0100 Subject: [PATCH] Add Docker setup, add environment vars for each flag --- Dockerfile | 14 +++++++++ README.md | 65 ++++++++++++++++++++++++++++++++++++++++++ cmd/aggregator/main.go | 15 +++++++--- cmd/collector/main.go | 44 ++++++++++++++++++++++------ cmd/frontend/main.go | 26 ++++++++++++++--- docker-compose.yml | 26 +++++++++++++++++ 6 files changed, 174 insertions(+), 16 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..63ae17e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/collector ./cmd/collector && \ + CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/aggregator ./cmd/aggregator && \ + CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/frontend ./cmd/frontend && \ + CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/cli ./cmd/cli + +FROM scratch +COPY --from=builder /out/ /usr/local/bin/ diff --git a/README.md b/README.md index 7d695ac..d8030a5 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,71 @@ Programs are written in Go. No CGO, no external runtime dependencies. --- +DEPLOYMENT + +## Docker + +All four binaries are published in a single image: `git.ipng.ch/ipng/nginx-logtail`. + +The image is built with a two-stage Dockerfile: a `golang:1.24-alpine` builder produces +statically-linked, stripped binaries (`CGO_ENABLED=0`, `-trimpath -ldflags="-s -w"`); the final +stage is `scratch` — no OS, no shell, no runtime dependencies. Each binary is invoked explicitly +via the container `command`. + +### Build and push + +``` +docker compose build --push +``` + +### Running aggregator + frontend + +The `docker-compose.yml` in the repo root runs the aggregator and frontend together. At minimum, +set `AGGREGATOR_COLLECTORS` to the comma-separated `host:port` list of your collector(s): + +```sh +AGGREGATOR_COLLECTORS=nginx1:9090,nginx2:9090 docker compose up -d +``` + +The frontend reaches the aggregator at `aggregator:9091` via Docker's internal DNS. The frontend +UI is available on port `8080`. + +### Environment variables + +All flags have environment variable equivalents. CLI flags take precedence over env vars. + +**collector** (runs on each nginx host, not in Docker): + +| Env var | Flag | Default | +|--------------------------|-------------------|-------------| +| `COLLECTOR_LISTEN` | `-listen` | `:9090` | +| `COLLECTOR_PROM_LISTEN` | `-prom-listen` | `:9100` | +| `COLLECTOR_LOGS` | `-logs` | — | +| `COLLECTOR_LOGS_FILE` | `-logs-file` | — | +| `COLLECTOR_SOURCE` | `-source` | hostname | +| `COLLECTOR_V4PREFIX` | `-v4prefix` | `24` | +| `COLLECTOR_V6PREFIX` | `-v6prefix` | `48` | +| `COLLECTOR_SCAN_INTERVAL`| `-scan-interval` | `10s` | + +**aggregator**: + +| Env var | Flag | Default | +|--------------------------|---------------|-------------| +| `AGGREGATOR_LISTEN` | `-listen` | `:9091` | +| `AGGREGATOR_COLLECTORS` | `-collectors` | — (required)| +| `AGGREGATOR_SOURCE` | `-source` | hostname | + +**frontend**: + +| Env var | Flag | Default | +|------------------|------------|-------------------| +| `FRONTEND_LISTEN`| `-listen` | `:8080` | +| `FRONTEND_TARGET`| `-target` | `localhost:9091` | +| `FRONTEND_N` | `-n` | `25` | +| `FRONTEND_REFRESH`| `-refresh`| `30` | + +--- + DESIGN ## Directory Layout diff --git a/cmd/aggregator/main.go b/cmd/aggregator/main.go index 51d87d4..930837a 100644 --- a/cmd/aggregator/main.go +++ b/cmd/aggregator/main.go @@ -15,13 +15,13 @@ import ( ) func main() { - listen := flag.String("listen", ":9091", "gRPC listen address") - collectors := flag.String("collectors", "", "comma-separated collector host:port addresses") - source := flag.String("source", hostname(), "name for this aggregator in responses") + listen := flag.String("listen", envOr("AGGREGATOR_LISTEN", ":9091"), "gRPC listen address (env: AGGREGATOR_LISTEN)") + collectors := flag.String("collectors", envOr("AGGREGATOR_COLLECTORS", ""), "comma-separated collector host:port addresses (env: AGGREGATOR_COLLECTORS)") + source := flag.String("source", envOr("AGGREGATOR_SOURCE", hostname()), "name for this aggregator in responses (env: AGGREGATOR_SOURCE, default: hostname)") flag.Parse() if *collectors == "" { - log.Fatal("aggregator: --collectors is required") + log.Fatal("aggregator: --collectors / AGGREGATOR_COLLECTORS is required") } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) @@ -72,3 +72,10 @@ func hostname() string { } return h } + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/cmd/collector/main.go b/cmd/collector/main.go index 59d967f..cc1dd35 100644 --- a/cmd/collector/main.go +++ b/cmd/collector/main.go @@ -10,6 +10,7 @@ import ( "os" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -19,14 +20,14 @@ import ( ) func main() { - listen := flag.String("listen", ":9090", "gRPC listen address") - promListen := flag.String("prom-listen", ":9100", "Prometheus metrics listen address (empty to disable)") - logPaths := flag.String("logs", "", "comma-separated log file paths/globs to tail") - logsFile := flag.String("logs-file", "", "file containing one log path/glob per line") - source := flag.String("source", hostname(), "name for this collector (default: hostname)") - v4prefix := flag.Int("v4prefix", 24, "IPv4 prefix length for client bucketing") - v6prefix := flag.Int("v6prefix", 48, "IPv6 prefix length for client bucketing") - scanInterval := flag.Duration("scan-interval", 10*time.Second, "how often to rescan glob patterns for new/removed files") + listen := flag.String("listen", envOr("COLLECTOR_LISTEN", ":9090"), "gRPC listen address (env: COLLECTOR_LISTEN)") + promListen := flag.String("prom-listen", envOr("COLLECTOR_PROM_LISTEN", ":9100"), "Prometheus metrics listen address, empty to disable (env: COLLECTOR_PROM_LISTEN)") + logPaths := flag.String("logs", envOr("COLLECTOR_LOGS", ""), "comma-separated log file paths/globs to tail (env: COLLECTOR_LOGS)") + logsFile := flag.String("logs-file", envOr("COLLECTOR_LOGS_FILE", ""), "file containing one log path/glob per line (env: COLLECTOR_LOGS_FILE)") + source := flag.String("source", envOr("COLLECTOR_SOURCE", hostname()), "name for this collector (env: COLLECTOR_SOURCE, default: hostname)") + v4prefix := flag.Int("v4prefix", envOrInt("COLLECTOR_V4PREFIX", 24), "IPv4 prefix length for client bucketing (env: COLLECTOR_V4PREFIX)") + v6prefix := flag.Int("v6prefix", envOrInt("COLLECTOR_V6PREFIX", 48), "IPv6 prefix length for client bucketing (env: COLLECTOR_V6PREFIX)") + scanInterval := flag.Duration("scan-interval", envOrDuration("COLLECTOR_SCAN_INTERVAL", 10*time.Second), "how often to rescan glob patterns for new/removed files (env: COLLECTOR_SCAN_INTERVAL)") flag.Parse() patterns := collectPatterns(*logPaths, *logsFile) @@ -154,3 +155,30 @@ func hostname() string { } return h } + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envOrInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + log.Printf("collector: invalid int for %s=%q, using default %d", key, v, def) + } + return def +} + +func envOrDuration(key string, def time.Duration) time.Duration { + if v := os.Getenv(key); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + log.Printf("collector: invalid duration for %s=%q, using default %s", key, v, def) + } + return def +} diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go index bb32522..5dcaa86 100644 --- a/cmd/frontend/main.go +++ b/cmd/frontend/main.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" ) @@ -16,10 +17,10 @@ import ( var templatesFS embed.FS func main() { - listen := flag.String("listen", ":8080", "HTTP listen address") - target := flag.String("target", "localhost:9091", "default gRPC endpoint (aggregator or collector)") - n := flag.Int("n", 25, "default number of table rows") - refresh := flag.Int("refresh", 30, "meta-refresh interval in seconds (0 = disabled)") + listen := flag.String("listen", envOr("FRONTEND_LISTEN", ":8080"), "HTTP listen address (env: FRONTEND_LISTEN)") + target := flag.String("target", envOr("FRONTEND_TARGET", "localhost:9091"), "default gRPC endpoint, aggregator or collector (env: FRONTEND_TARGET)") + n := flag.Int("n", envOrInt("FRONTEND_N", 25), "default number of table rows (env: FRONTEND_N)") + refresh := flag.Int("refresh", envOrInt("FRONTEND_REFRESH", 30), "meta-refresh interval in seconds, 0 to disable (env: FRONTEND_REFRESH)") flag.Parse() funcMap := template.FuncMap{"fmtCount": fmtCount} @@ -51,3 +52,20 @@ func main() { log.Printf("frontend: shutting down") srv.Shutdown(context.Background()) } + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envOrInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + log.Printf("frontend: invalid int for %s=%q, using default %d", key, v, def) + } + return def +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d1d87a4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + aggregator: + build: . + image: git.ipng.ch/ipng/nginx-logtail + command: ["/usr/local/bin/aggregator"] + restart: unless-stopped + environment: + AGGREGATOR_LISTEN: ":9091" + AGGREGATOR_COLLECTORS: "" # e.g. "collector1:9090,collector2:9090" + AGGREGATOR_SOURCE: "" # defaults to container hostname + ports: + - "9091:9091" + + frontend: + image: git.ipng.ch/ipng/nginx-logtail + command: ["/usr/local/bin/frontend"] + restart: unless-stopped + environment: + FRONTEND_LISTEN: ":8080" + FRONTEND_TARGET: "aggregator:9091" + FRONTEND_N: "25" + FRONTEND_REFRESH: "30" + ports: + - "8080:8080" + depends_on: + - aggregator