Pim van Pelt
143aad9063
Build and release tooling:
- Makefile with help as default; targets: build/build-amd64/build-arm64,
test, lint, proto, pkg-deb, docker, docker-push, clean, plus
install-deps (+ three sub-targets for apt / Go toolchain / Go tools).
- internal/version package; -ldflags -X injects Version/Commit/Date into
every binary. -version flag on all four binaries (nginx-logtail version
for the CLI).
- Dockerfile takes VERSION/COMMIT/DATE build-args and forwards them.
- .deb output lands in build/; .gitignore ignores /build/.
Debian package:
- debian/build-deb.sh packages all four static binaries into a single
nginx-logtail_<ver>_<arch>.deb using dpkg-deb.
- Binary layout: /usr/sbin/nginx-logtail-{collector,aggregator,frontend}
and /usr/bin/nginx-logtail.
- nginx-logtail(8) manpage.
- Three systemd units (collector, aggregator, frontend) shipped under
/lib/systemd/system/. Installed but never enabled or started — the
operator opts in per host.
- Collector runs as _logtail:www-data (log access); aggregator and
frontend as _logtail:_logtail. postinst creates the system user/group
idempotently.
- Single shared env file /etc/default/nginx-logtail rendered from a
template at first install with %HOSTNAME% substituted. Sensible
defaults for every COLLECTOR_*, AGGREGATOR_*, FRONTEND_* variable;
plus COLLECTOR_ARGS / AGGREGATOR_ARGS / FRONTEND_ARGS escape hatches
appended to ExecStart. Not a dpkg conffile: operator edits survive
upgrades and dpkg --purge removes it.
Versioned UDP wire format:
- ParseUDPLine dispatches on a leading "v<N>\t" tag; v1 routes to the
existing 12-field parser. Unknown/missing versions fail closed so
future v2 parsers can land before emitters are upgraded.
- Tests updated; design.md FR-2.2 rewritten to make the version tag
normative.
Docs:
- README.md gains a Quick Start (Debian / Docker Compose / from source).
- user-guide.md rewritten around Installation and Configuration: full
env-var table, UDP-only default explained, precise file/UDP log_format
layouts, note that operators can emit "0" for unknown \$is_tor / \$asn.
- Drilldown cycle, frontend filter table, and CLI --group-by list all
include source_tag. UDP counters documented in the Prometheus section.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
330 lines
7.8 KiB
Go
330 lines
7.8 KiB
Go
package main
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestParseLine(t *testing.T) {
|
|
good := "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api/v1/search?q=foo&x=1\t200\t1452\t0.043"
|
|
|
|
tests := []struct {
|
|
name string
|
|
line string
|
|
wantOK bool
|
|
want LogRecord
|
|
}{
|
|
{
|
|
name: "normal IPv4 line strips query string",
|
|
line: good,
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/api/v1/search",
|
|
Status: "200",
|
|
Method: "GET",
|
|
BodyBytesSent: 1452,
|
|
RequestTime: 0.043,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "URI with no query string",
|
|
line: "host\t10.0.0.1\t0\tPOST\t/submit\t201\t0\t0.001",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "host",
|
|
ClientPrefix: "10.0.0.0/24",
|
|
URI: "/submit",
|
|
Status: "201",
|
|
Method: "POST",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "IPv6 address truncated to /48",
|
|
line: "host\t2001:db8:cafe::1\t0\tGET\t/\t200\t0\t0.001",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "host",
|
|
ClientPrefix: "2001:db8:cafe::/48",
|
|
URI: "/",
|
|
Status: "200",
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "too few fields returns false",
|
|
line: "host\t1.2.3.4\t0\tGET\t/",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "empty line returns false",
|
|
line: "",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "invalid IP returns false",
|
|
line: "host\tnot-an-ip\t0\tGET\t/\t200\t0\t0.001",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "status 429",
|
|
line: "api.example.com\t5.6.7.8\t0\tGET\t/rate-limited\t429\t0\t0.001",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "api.example.com",
|
|
ClientPrefix: "5.6.7.0/24",
|
|
URI: "/rate-limited",
|
|
Status: "429",
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "is_tor=1 sets IsTor true",
|
|
line: "tor.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "tor.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: true,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "is_tor=0 sets IsTor false",
|
|
line: "normal.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "normal.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: false,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "missing is_tor field defaults to false (backward compat)",
|
|
line: "old.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "old.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: false,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "asn field parsed",
|
|
line: "asn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\t12345",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "asn.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: false,
|
|
ASN: 12345,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "asn field with is_tor=1",
|
|
line: "both.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1\t65535",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "both.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: true,
|
|
ASN: 65535,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "missing asn field defaults to 0 (backward compat)",
|
|
line: "noasn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "noasn.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: true,
|
|
ASN: 0,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "invalid asn field defaults to 0",
|
|
line: "badann.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\tnot-a-number",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "badann.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: false,
|
|
ASN: 0,
|
|
Method: "GET",
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := ParseLine(tc.line, 24, 48)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ParseLine ok=%v, want %v", ok, tc.wantOK)
|
|
}
|
|
if !tc.wantOK {
|
|
return
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("got %+v, want %+v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseUDPLine(t *testing.T) {
|
|
// v1 \t host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
|
|
// is_tor \t asn \t source_tag \t server_addr \t scheme
|
|
good := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
|
|
|
|
tests := []struct {
|
|
name string
|
|
line string
|
|
wantOK bool
|
|
want LogRecord
|
|
}{
|
|
{
|
|
name: "v1 payload parsed, query stripped, extras dropped",
|
|
line: good,
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/api/v1/search",
|
|
Status: "200",
|
|
IsTor: false,
|
|
ASN: 12345,
|
|
Method: "GET",
|
|
BodyBytesSent: 1452,
|
|
RequestTime: 0.043,
|
|
SourceTag: "cdn",
|
|
},
|
|
},
|
|
{
|
|
name: "v1 IPv6 tor=1 direct tag",
|
|
line: "v1\th\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "h",
|
|
ClientPrefix: "2001:db8::/48",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: true,
|
|
ASN: 65535,
|
|
Method: "GET",
|
|
BodyBytesSent: 0,
|
|
RequestTime: 0,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "v1 payload with 11 fields rejected",
|
|
line: "v1\th\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v1 payload with 13 fields rejected",
|
|
line: good + "\textra",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v1 bad IP rejected",
|
|
line: "v1\th\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "unknown version rejected (future v2)",
|
|
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "missing version prefix rejected (legacy 12-field line)",
|
|
line: "www.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "no tab at all rejected",
|
|
line: "v1",
|
|
wantOK: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := ParseUDPLine(tc.line, 24, 48)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ParseUDPLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
|
|
}
|
|
if !tc.wantOK {
|
|
return
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("got %+v, want %+v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTruncateIP(t *testing.T) {
|
|
tests := []struct {
|
|
addr string
|
|
want string
|
|
}{
|
|
{"1.2.3.4", "1.2.3.0/24"},
|
|
{"192.168.100.200", "192.168.100.0/24"},
|
|
{"2001:db8:cafe:babe::1", "2001:db8:cafe::/48"},
|
|
{"::1", "::/48"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
got, ok := truncateIP(tc.addr, 24, 48)
|
|
if !ok {
|
|
t.Errorf("truncateIP(%q) returned not-ok", tc.addr)
|
|
continue
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("truncateIP(%q) = %q, want %q", tc.addr, got, tc.want)
|
|
}
|
|
}
|
|
}
|