ipng/nginx-logtail
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

RELEASE 1.0.1: v2 log format, source_tag-labeled metrics, lint cleanup

Browse Source 
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).

Prometheus exposition is rewritten:

  - nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
    a source_tag label.
  - nginx_http_requests_by_source_total gains status_class.
  - New v2-only metrics: nginx_http_request_bytes,
    nginx_http_upstream_duration_seconds,
    nginx_http_upstream_requests_total{status_class}.
  - Dropped nginx_http_response_body_bytes_by_source (subsumed by the
    dual-labeled bytes_sent metric).

Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).

Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
This commit is contained in:
Pim van Pelt
2026-05-01 15:40:53 +02:00
parent d1a21a7a62
commit 6647f95be4
28 changed files with 931 additions and 724 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/collector/parser_test.go
+165 -246
View File
@@ -4,215 +4,7 @@ import (
"testing"
)
func TestParseLine(t *testing.T) {
good := "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api/v1/search?q=foo&x=1\t200\t1452\t0.043"
tests := []struct {
name string
line string
wantOK bool
want LogRecord
}{
{
name: "normal IPv4 line strips query string",
line: good,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
Method: "GET",
BodyBytesSent: 1452,
RequestTime: 0.043,
SourceTag: "direct",
},
},
{
name: "URI with no query string",
line: "host\t10.0.0.1\t0\tPOST\t/submit\t201\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "host",
ClientPrefix: "10.0.0.0/24",
URI: "/submit",
Status: "201",
Method: "POST",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "IPv6 address truncated to /48",
line: "host\t2001:db8:cafe::1\t0\tGET\t/\t200\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "host",
ClientPrefix: "2001:db8:cafe::/48",
URI: "/",
Status: "200",
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "too few fields returns false",
line: "host\t1.2.3.4\t0\tGET\t/",
wantOK: false,
},
{
name: "empty line returns false",
line: "",
wantOK: false,
},
{
name: "invalid IP returns false",
line: "host\tnot-an-ip\t0\tGET\t/\t200\t0\t0.001",
wantOK: false,
},
{
name: "status 429",
line: "api.example.com\t5.6.7.8\t0\tGET\t/rate-limited\t429\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "api.example.com",
ClientPrefix: "5.6.7.0/24",
URI: "/rate-limited",
Status: "429",
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "is_tor=1 sets IsTor true",
line: "tor.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
wantOK: true,
want: LogRecord{
Website: "tor.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "is_tor=0 sets IsTor false",
line: "normal.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0",
wantOK: true,
want: LogRecord{
Website: "normal.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "missing is_tor field defaults to false (backward compat)",
line: "old.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "old.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "asn field parsed",
line: "asn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\t12345",
wantOK: true,
want: LogRecord{
Website: "asn.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "asn field with is_tor=1",
line: "both.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1\t65535",
wantOK: true,
want: LogRecord{
Website: "both.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "missing asn field defaults to 0 (backward compat)",
line: "noasn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
wantOK: true,
want: LogRecord{
Website: "noasn.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
ASN: 0,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "invalid asn field defaults to 0",
line: "badann.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\tnot-a-number",
wantOK: true,
want: LogRecord{
Website: "badann.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
ASN: 0,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseLine ok=%v, want %v", ok, tc.wantOK)
}
if !tc.wantOK {
return
}
if got != tc.want {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
}
}
func TestParseUDPLine(t *testing.T) {
func TestParseLineV1(t *testing.T) {
// v1 \t host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
// is_tor \t asn \t source_tag \t server_addr \t scheme
good := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
@@ -228,16 +20,16 @@ func TestParseUDPLine(t *testing.T) {
line: good,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
BodyBytesSent: 1452,
RequestTime: 0.043,
SourceTag: "cdn",
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
BytesSent: 1452,
RequestTime: 0.043,
SourceTag: "cdn",
},
},
{
@@ -245,16 +37,16 @@ func TestParseUDPLine(t *testing.T) {
line: "v1\th\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
wantOK: true,
want: LogRecord{
Website: "h",
ClientPrefix: "2001:db8::/48",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
BodyBytesSent: 0,
RequestTime: 0,
SourceTag: "direct",
Website: "h",
ClientPrefix: "2001:db8::/48",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
BytesSent: 0,
RequestTime: 0,
SourceTag: "direct",
},
},
{
@@ -272,28 +64,13 @@ func TestParseUDPLine(t *testing.T) {
line: "v1\th\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "unknown version rejected (future v2)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "missing version prefix rejected (legacy 12-field line)",
line: "www.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "no tab at all rejected",
line: "v1",
wantOK: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseUDPLine(tc.line, 24, 48)
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseUDPLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
}
if !tc.wantOK {
return
@@ -305,6 +82,148 @@ func TestParseUDPLine(t *testing.T) {
}
}
func TestParseLineV2(t *testing.T) {
// v2 \t host \t remote_addr \t method \t uri \t status \t bytes_sent \t request_length \t
// request_time \t upstream_response_time \t upstream_status \t is_tor \t asn \t
// source_tag \t server_addr \t scheme
full := "v2\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1500\t421\t0.043\t0.012\t200\t0\t12345\tcdn\t10.0.0.1\thttps"
tests := []struct {
name string
line string
wantOK bool
want LogRecord
}{
{
name: "v2 with upstream",
line: full,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
ASN: 12345,
Method: "GET",
BytesSent: 1500,
RequestLength: 421,
RequestTime: 0.043,
UpstreamResponseTime: 0.012,
UpstreamStatus: "200",
HasUpstream: true,
SourceTag: "cdn",
},
},
{
name: "v2 no upstream (dash sentinels)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/static.html\t200\t900\t300\t0.001\t-\t-\t0\t0\tdirect\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/static.html",
Status: "200",
Method: "GET",
BytesSent: 900,
RequestLength: 300,
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "v2 no upstream (empty fields)",
line: "v2\thh\t1.2.3.4\tGET\t/\t301\t200\t100\t0\t\t\t0\t0\tdirect\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "hh",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "301",
Method: "GET",
BytesSent: 200,
RequestLength: 100,
SourceTag: "direct",
},
},
{
name: "v2 retried upstreams (comma-separated, last wins)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/api\t502\t900\t300\t1.500\t0.500, 1.000\t504, 502\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api",
Status: "502",
Method: "GET",
BytesSent: 900,
RequestLength: 300,
RequestTime: 1.500,
UpstreamResponseTime: 1.000,
UpstreamStatus: "502",
HasUpstream: true,
SourceTag: "cdn",
},
},
{
name: "v2 wrong field count (14) rejected",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1",
wantOK: false,
},
{
name: "v2 bad IP rejected",
line: "v2\thh\tnope\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: false,
},
{
name: "v2 bad upstream time leaves HasUpstream=false",
line: "v2\thh\t1.2.3.4\tGET\t/\t200\t0\t0\t0\tnotanumber\t200\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "hh",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
Method: "GET",
SourceTag: "cdn",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
}
if !tc.wantOK {
return
}
if got != tc.want {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
}
}
func TestParseLineRejections(t *testing.T) {
tests := []struct {
name string
line string
}{
{"empty line", ""},
{"no tab at all", "v1"},
{"unknown version v3", "v3\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp"},
{"missing version prefix (legacy file format)", "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api\t200\t1452\t0.043"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, ok := ParseLine(tc.line, 24, 48); ok {
t.Errorf("expected rejection for %q", tc.line)
}
})
}
}
func TestTruncateIP(t *testing.T) {
tests := []struct {
addr string