diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8e5d080 --- /dev/null +++ b/.env.example @@ -0,0 +1,127 @@ +# .env.example — docker-compose.yml environment for nginx-logtail. +# +# Copy to .env and edit. `.env` is gitignored so local edits stay local: +# +# cp .env.example .env +# $EDITOR .env +# docker compose up -d + +# --------------------------------------------------------------------------- +# Which containers start +# --------------------------------------------------------------------------- +# Docker Compose "profiles" decide which services actually start when you +# run `docker compose up`. Set COMPOSE_PROFILES below to a comma-separated +# list of the services you want. Valid values: +# +# collector — ingests nginx logs (file + UDP); runs on each nginx host +# aggregator — merges collectors into a central view +# frontend — HTTP dashboard (reads aggregator or a single collector) +# +# Leave empty (or delete the line) to start nothing. + +# Default: central host stack (aggregator + frontend), mirrors the +# shipped docker-compose.yml layout. +COMPOSE_PROFILES=aggregator,frontend + +# Examples: +#COMPOSE_PROFILES=collector,aggregator,frontend # single-host all-in-one +#COMPOSE_PROFILES=collector # nginx host running only the collector +#COMPOSE_PROFILES=frontend # frontend pointing at a remote aggregator +#COMPOSE_PROFILES= # nothing + +# --------------------------------------------------------------------------- +# Collector (nginx-logtail-collector) +# --------------------------------------------------------------------------- +# The variable names below mirror /etc/default/nginx-logtail on a Debian +# install, so an operator moving between bare-metal and containers can +# reuse muscle memory. Each variable is read by the collector via an +# envOr(..., COLLECTOR_*, ...) fallback in cmd/collector/main.go. + +# gRPC listen address inside the container. Host-networked, so this is +# also the address the aggregator dials. +COLLECTOR_LISTEN=:9090 + +# Prometheus /metrics listen address. Set to "" to disable the endpoint. +COLLECTOR_PROM_LISTEN=:9100 + +# Comma-separated log file paths or glob patterns. At least one of +# COLLECTOR_LOGS, COLLECTOR_LOGS_FILE, or COLLECTOR_LOGTAIL_PORT > 0 must +# be set; otherwise the collector refuses to start. Leave empty to run +# UDP-only. The shipped compose file bind-mounts /var/log/nginx:ro so +# paths under that tree are readable. +COLLECTOR_LOGS= + +# File containing one path/glob per line. +COLLECTOR_LOGS_FILE= + +# Name for this collector in query responses and ListTargets. Docker's +# default container hostname is the short container id; set something +# stable here if you want meaningful source names across restarts. +#COLLECTOR_SOURCE=nginx1 + +# IPv4 / IPv6 prefix lengths for client address bucketing. +COLLECTOR_V4PREFIX=24 +COLLECTOR_V6PREFIX=48 + +# How often to rescan COLLECTOR_LOGS globs for new/removed files. +COLLECTOR_SCAN_INTERVAL=10s + +# UDP port for ipng_stats_logtail datagrams from nginx-ipng-stats-plugin. +# Set to 0 to disable the UDP listener entirely. +COLLECTOR_LOGTAIL_PORT=9514 + +# UDP bind address. Host-networked default accepts localhost traffic +# from host nginx; change to 0.0.0.0 if packets come from off-host. +COLLECTOR_LOGTAIL_BIND=127.0.0.1 + +# Extra arguments appended to the collector argv. Useful for temporary +# overrides or flags without an env-var form. +COLLECTOR_ARGS= + +# --------------------------------------------------------------------------- +# Aggregator (nginx-logtail-aggregator) +# --------------------------------------------------------------------------- + +# gRPC listen address. The compose file publishes 9091 regardless. +AGGREGATOR_LISTEN=:9091 + +# Comma-separated collector addresses (MANDATORY). How you spell them +# depends on whether the collectors are in the same compose stack: +# +# same-host all-in-one (COMPOSE_PROFILES=collector,aggregator,frontend): +# the collector uses network_mode: host, so reach it via the host's +# address — "host.docker.internal:9090" on Docker Desktop, or the +# host's LAN address on Linux. +# +# central aggregator pointing at remote collectors: +# list each remote host, e.g. "nginx1:9090,nginx2:9090,nginx3:9090". +AGGREGATOR_COLLECTORS=nginx1:9090 + +# Display name for this aggregator. Uncomment to override the short +# container id that os.Hostname() returns inside Docker. +#AGGREGATOR_SOURCE=agg-prod + +# Extra arguments appended to the aggregator argv. +AGGREGATOR_ARGS= + +# --------------------------------------------------------------------------- +# Frontend (nginx-logtail-frontend) +# --------------------------------------------------------------------------- + +# HTTP listen address. The compose file publishes 8080 regardless. +FRONTEND_LISTEN=:8080 + +# Default gRPC endpoint the dashboard queries. When the aggregator runs +# in the same compose stack, Docker's internal DNS resolves the service +# name to the bridge IP — "aggregator:9091" just works. Point at a +# remote aggregator instead for a frontend-only deployment. +FRONTEND_TARGET=aggregator:9091 + +# Default number of table rows. Override per-URL with ?n=N. +FRONTEND_N=25 + +# Meta-refresh interval (seconds). Set 0 to disable auto-refresh. +FRONTEND_REFRESH=30 + +# Extra arguments appended to the frontend argv. +FRONTEND_ARGS= diff --git a/.gitignore b/.gitignore index 39a00c2..00e73b1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ # Build output — per-arch binaries and .deb packages, all under build/. /build/ +# Local docker-compose overrides (copy .env.example -> .env). +/.env + # Editor .idea/ .vscode/ diff --git a/Dockerfile b/Dockerfile index 845ca44..5519ef9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,64 @@ -FROM golang:1.24-alpine AS builder +# syntax=docker/dockerfile:1.6 +# +# Multi-stage, multi-arch build for nginx-logtail. +# +# A single `docker buildx build --platform linux/amd64,linux/arm64` +# produces a proper multi-arch manifest without a qemu-emulated builder: +# the builder runs on the host's $BUILDPLATFORM and Go cross-compiles +# to each requested $TARGETARCH via the Makefile's build-amd64 / +# build-arm64 targets. Both BUILDPLATFORM and TARGETARCH are populated +# automatically by buildx. +# +# One runtime image contains all four binaries; docker-compose.yml +# picks which one each service runs via `command:`. +# +# Driven from the Makefile: +# make docker # native arch, --load into local daemon +# make docker-push # multi-arch manifest, pushed to the registry +# ============================================================================= +# Builder — compiles all four binaries on the host's native arch. +# ============================================================================= +FROM --platform=$BUILDPLATFORM golang:1.24-alpine AS builder + +# TARGETARCH (amd64, arm64, ...) is supplied by buildx; VERSION/COMMIT/DATE +# come in as --build-arg from `make docker` / `make docker-push`. +ARG TARGETARCH ARG VERSION=dev ARG COMMIT=unknown ARG DATE=unknown -ENV CGO_ENABLED=0 \ - LDFLAGS="-s -w -X git.ipng.ch/ipng/nginx-logtail/internal/version.Version=${VERSION} -X git.ipng.ch/ipng/nginx-logtail/internal/version.Commit=${COMMIT} -X git.ipng.ch/ipng/nginx-logtail/internal/version.Date=${DATE}" +# make drives the build. git lets the Makefile stamp a commit hash via +# `git rev-parse` when --build-arg COMMIT isn't supplied; when it is, +# the command-line override below wins and git is only needed to avoid +# the fallback branch warning. +RUN apk add --no-cache make git WORKDIR /src + +# Cache Go modules so code-only rebuilds skip the download. COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go build -trimpath -ldflags="${LDFLAGS}" -o /out/collector ./cmd/collector && \ - go build -trimpath -ldflags="${LDFLAGS}" -o /out/aggregator ./cmd/aggregator && \ - go build -trimpath -ldflags="${LDFLAGS}" -o /out/frontend ./cmd/frontend && \ - go build -trimpath -ldflags="${LDFLAGS}" -o /out/cli ./cmd/cli +# Generated proto files are checked in, but `docker COPY` resets mtimes, +# which can make `make` think they are stale relative to the .proto +# source. Touch them so the regen rule is skipped. +RUN touch proto/logtailpb/logtail.pb.go proto/logtailpb/logtail_grpc.pb.go + +# The Makefile's build- target sets GOOS=linux GOARCH= +# CGO_ENABLED=0 and emits build// with the usual -ldflags +# version injection. Forwarding VERSION/COMMIT_HASH/DATE on the command +# line stamps the values supplied by --build-arg into the binaries. +RUN make VERSION=$VERSION COMMIT_HASH=$COMMIT DATE=$DATE build-$TARGETARCH + +# ============================================================================= +# Runtime — scratch, no OS, no shell. All four binaries. +# ============================================================================= FROM scratch -COPY --from=builder /out/ /usr/local/bin/ +ARG TARGETARCH +COPY --from=builder /src/build/$TARGETARCH/collector /usr/local/bin/collector +COPY --from=builder /src/build/$TARGETARCH/aggregator /usr/local/bin/aggregator +COPY --from=builder /src/build/$TARGETARCH/frontend /usr/local/bin/frontend +COPY --from=builder /src/build/$TARGETARCH/cli /usr/local/bin/cli diff --git a/docker-compose.yml b/docker-compose.yml index d1d87a4..d84a450 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,69 @@ +# docker-compose.yml — nginx-logtail container stack +# +# One multi-arch image ships all four binaries (built by the companion +# Dockerfile). Each service below picks which binary it runs via +# `command:`. Services are opt-in via Docker Compose "profiles": +# +# cp .env.example .env +# $EDITOR .env # set COMPOSE_PROFILES and tunables +# docker compose up -d +# +# Valid profile names: collector, aggregator, frontend. +# +# Typical deployments: +# +# COMPOSE_PROFILES=aggregator,frontend # central host (default) +# COMPOSE_PROFILES=collector # nginx host running only the collector in Docker +# COMPOSE_PROFILES=collector,aggregator,frontend # single-host all-in-one +# +# See .env.example for every tunable. + services: - aggregator: - build: . - image: git.ipng.ch/ipng/nginx-logtail - command: ["/usr/local/bin/aggregator"] + collector: + profiles: [collector] + build: + context: . + dockerfile: Dockerfile + image: git.ipng.ch/ipng/nginx-logtail:latest + container_name: nginx-logtail-collector restart: unless-stopped - environment: - AGGREGATOR_LISTEN: ":9091" - AGGREGATOR_COLLECTORS: "" # e.g. "collector1:9090,collector2:9090" - AGGREGATOR_SOURCE: "" # defaults to container hostname + command: ["/usr/local/bin/collector"] + env_file: .env + + # Host networking is the simplest way to accept UDP datagrams from a + # host nginx on 127.0.0.1:9514 — bridge networking with port mapping + # does not round-trip packets that never leave loopback. Mac/Windows + # Docker Desktop users must switch to a bridge + explicit + # COLLECTOR_LOGTAIL_BIND. + network_mode: host + + # Mount nginx log directory so the file tailer can read it. Harmless + # when COLLECTOR_LOGS is empty (the collector simply runs UDP-only). + volumes: + - /var/log/nginx:/var/log/nginx:ro + + aggregator: + profiles: [aggregator] + build: + context: . + dockerfile: Dockerfile + image: git.ipng.ch/ipng/nginx-logtail:latest + container_name: nginx-logtail-aggregator + restart: unless-stopped + command: ["/usr/local/bin/aggregator"] + env_file: .env ports: - - "9091:9091" + - "9091:9091" # gRPC frontend: - image: git.ipng.ch/ipng/nginx-logtail - command: ["/usr/local/bin/frontend"] + profiles: [frontend] + build: + context: . + dockerfile: Dockerfile + image: git.ipng.ch/ipng/nginx-logtail:latest + container_name: nginx-logtail-frontend restart: unless-stopped - environment: - FRONTEND_LISTEN: ":8080" - FRONTEND_TARGET: "aggregator:9091" - FRONTEND_N: "25" - FRONTEND_REFRESH: "30" + command: ["/usr/local/bin/frontend"] + env_file: .env ports: - - "8080:8080" - depends_on: - - aggregator + - "8080:8080" # HTTP dashboard