PRE-RELEASE 0.9.1: Makefile, Debian packaging, versioned UDP

Build and release tooling:
- Makefile with help as default; targets: build/build-amd64/build-arm64,
  test, lint, proto, pkg-deb, docker, docker-push, clean, plus
  install-deps (+ three sub-targets for apt / Go toolchain / Go tools).
- internal/version package; -ldflags -X injects Version/Commit/Date into
  every binary. -version flag on all four binaries (nginx-logtail version
  for the CLI).
- Dockerfile takes VERSION/COMMIT/DATE build-args and forwards them.
- .deb output lands in build/; .gitignore ignores /build/.

Debian package:
- debian/build-deb.sh packages all four static binaries into a single
  nginx-logtail_<ver>_<arch>.deb using dpkg-deb.
- Binary layout: /usr/sbin/nginx-logtail-{collector,aggregator,frontend}
  and /usr/bin/nginx-logtail.
- nginx-logtail(8) manpage.
- Three systemd units (collector, aggregator, frontend) shipped under
  /lib/systemd/system/. Installed but never enabled or started — the
  operator opts in per host.
- Collector runs as _logtail:www-data (log access); aggregator and
  frontend as _logtail:_logtail. postinst creates the system user/group
  idempotently.
- Single shared env file /etc/default/nginx-logtail rendered from a
  template at first install with %HOSTNAME% substituted. Sensible
  defaults for every COLLECTOR_*, AGGREGATOR_*, FRONTEND_* variable;
  plus COLLECTOR_ARGS / AGGREGATOR_ARGS / FRONTEND_ARGS escape hatches
  appended to ExecStart. Not a dpkg conffile: operator edits survive
  upgrades and dpkg --purge removes it.

Versioned UDP wire format:
- ParseUDPLine dispatches on a leading "v<N>\t" tag; v1 routes to the
  existing 12-field parser. Unknown/missing versions fail closed so
  future v2 parsers can land before emitters are upgraded.
- Tests updated; design.md FR-2.2 rewritten to make the version tag
  normative.

Docs:
- README.md gains a Quick Start (Debian / Docker Compose / from source).
- user-guide.md rewritten around Installation and Configuration: full
  env-var table, UDP-only default explained, precise file/UDP log_format
  layouts, note that operators can emit "0" for unknown \$is_tor / \$asn.
- Drilldown cycle, frontend filter table, and CLI --group-by list all
  include source_tag. UDP counters documented in the Prometheus section.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-17 10:35:08 +02:00
parent 577ed3dad5
commit 143aad9063
23 changed files with 1214 additions and 114 deletions

View File

@@ -4,6 +4,7 @@ import (
"bufio"
"context"
"flag"
"fmt"
"log"
"net"
"net/http"
@@ -15,6 +16,7 @@ import (
"syscall"
"time"
"git.ipng.ch/ipng/nginx-logtail/internal/version"
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
"google.golang.org/grpc"
)
@@ -30,8 +32,14 @@ func main() {
scanInterval := flag.Duration("scan-interval", envOrDuration("COLLECTOR_SCAN_INTERVAL", 10*time.Second), "how often to rescan glob patterns for new/removed files (env: COLLECTOR_SCAN_INTERVAL)")
logtailPort := flag.Int("logtail-port", envOrInt("COLLECTOR_LOGTAIL_PORT", 0), "UDP port to receive nginx ipng_stats_logtail packets, 0 to disable (env: COLLECTOR_LOGTAIL_PORT)")
logtailBind := flag.String("logtail-bind", envOr("COLLECTOR_LOGTAIL_BIND", "127.0.0.1"), "UDP bind address for the logtail listener (env: COLLECTOR_LOGTAIL_BIND)")
showVersion := flag.Bool("version", false, "print version and exit")
flag.Parse()
if *showVersion {
fmt.Printf("collector %s\n", version.String())
return
}
patterns := collectPatterns(*logPaths, *logsFile)
if len(patterns) == 0 && *logtailPort == 0 {
log.Fatal("collector: no inputs configured; use --logs, --logs-file, or --logtail-port")

View File

@@ -63,17 +63,33 @@ func ParseLine(line string, v4bits, v6bits int) (LogRecord, bool) {
}, true
}
// ParseUDPLine parses a tab-separated logtail log line from the UDP listener:
// ParseUDPLine dispatches on the version prefix emitted by
// nginx-ipng-stats-plugin's ipng_stats_logtail directive. The wire format is
// "v<N>\t<payload>", where <payload> is version-specific. Unknown or missing
// versions return false so operators can roll out a v2 parser before
// upgrading emitters.
func ParseUDPLine(line string, v4bits, v6bits int) (LogRecord, bool) {
i := strings.IndexByte(line, '\t')
if i < 0 {
return LogRecord{}, false
}
switch line[:i] {
case "v1":
return parseUDPLineV1(line[i+1:], v4bits, v6bits)
default:
return LogRecord{}, false
}
}
// parseUDPLineV1 parses the v1 payload (12 tab-separated fields):
//
// $host \t $remote_addr \t $request_method \t $request_uri \t $status \t
// $body_bytes_sent \t $request_time \t $is_tor \t $asn \t
// $ipng_source_tag \t $server_addr \t $scheme
//
// All 12 fields are required. server_addr and scheme are consumed but not
// propagated. Returns false for any malformed packet (wrong field count,
// bad IP).
func ParseUDPLine(line string, v4bits, v6bits int) (LogRecord, bool) {
fields := strings.Split(line, "\t")
// server_addr and scheme are parsed but discarded.
func parseUDPLineV1(payload string, v4bits, v6bits int) (LogRecord, bool) {
fields := strings.Split(payload, "\t")
if len(fields) != 12 {
return LogRecord{}, false
}

View File

@@ -213,9 +213,9 @@ func TestParseLine(t *testing.T) {
}
func TestParseUDPLine(t *testing.T) {
// host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
// v1 \t host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
// is_tor \t asn \t source_tag \t server_addr \t scheme
good := "www.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
good := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
tests := []struct {
name string
@@ -224,7 +224,7 @@ func TestParseUDPLine(t *testing.T) {
want LogRecord
}{
{
name: "all 12 fields parsed, query stripped, extras dropped",
name: "v1 payload parsed, query stripped, extras dropped",
line: good,
wantOK: true,
want: LogRecord{
@@ -241,8 +241,8 @@ func TestParseUDPLine(t *testing.T) {
},
},
{
name: "is_tor=1, tag direct, IPv6",
line: "h\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
name: "v1 IPv6 tor=1 direct tag",
line: "v1\th\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
wantOK: true,
want: LogRecord{
Website: "h",
@@ -258,18 +258,33 @@ func TestParseUDPLine(t *testing.T) {
},
},
{
name: "11 fields rejected",
line: "h\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1",
name: "v1 payload with 11 fields rejected",
line: "v1\th\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1",
wantOK: false,
},
{
name: "13 fields rejected",
name: "v1 payload with 13 fields rejected",
line: good + "\textra",
wantOK: false,
},
{
name: "bad IP rejected",
line: "h\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
name: "v1 bad IP rejected",
line: "v1\th\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "unknown version rejected (future v2)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "missing version prefix rejected (legacy 12-field line)",
line: "www.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "no tab at all rejected",
line: "v1",
wantOK: false,
},
}

View File

@@ -34,7 +34,7 @@ func TestUDPListenerRoundTrip(t *testing.T) {
defer conn.Close()
// The listener is started asynchronously; retry for up to 1s.
good := "www.example.com\t1.2.3.4\tGET\t/\t200\t42\t0.010\t0\t12345\tdirect\t10.0.0.1\thttps"
good := "v1\twww.example.com\t1.2.3.4\tGET\t/\t200\t42\t0.010\t0\t12345\tdirect\t10.0.0.1\thttps"
bad := "not enough\tfields"
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {