pim
6647f95be4
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).
Prometheus exposition is rewritten:
- nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
a source_tag label.
- nginx_http_requests_by_source_total gains status_class.
- New v2-only metrics: nginx_http_request_bytes,
nginx_http_upstream_duration_seconds,
nginx_http_upstream_requests_total{status_class}.
- Dropped nginx_http_response_body_bytes_by_source (subsumed by the
dual-labeled bytes_sent metric).
Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).
Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
249 lines
6.4 KiB
Go
249 lines
6.4 KiB
Go
package main
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestParseLineV1(t *testing.T) {
|
|
// v1 \t host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
|
|
// is_tor \t asn \t source_tag \t server_addr \t scheme
|
|
good := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
|
|
|
|
tests := []struct {
|
|
name string
|
|
line string
|
|
wantOK bool
|
|
want LogRecord
|
|
}{
|
|
{
|
|
name: "v1 payload parsed, query stripped, extras dropped",
|
|
line: good,
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/api/v1/search",
|
|
Status: "200",
|
|
IsTor: false,
|
|
ASN: 12345,
|
|
Method: "GET",
|
|
BytesSent: 1452,
|
|
RequestTime: 0.043,
|
|
SourceTag: "cdn",
|
|
},
|
|
},
|
|
{
|
|
name: "v1 IPv6 tor=1 direct tag",
|
|
line: "v1\th\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "h",
|
|
ClientPrefix: "2001:db8::/48",
|
|
URI: "/",
|
|
Status: "200",
|
|
IsTor: true,
|
|
ASN: 65535,
|
|
Method: "GET",
|
|
BytesSent: 0,
|
|
RequestTime: 0,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "v1 payload with 11 fields rejected",
|
|
line: "v1\th\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v1 payload with 13 fields rejected",
|
|
line: good + "\textra",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v1 bad IP rejected",
|
|
line: "v1\th\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
|
|
wantOK: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := ParseLine(tc.line, 24, 48)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
|
|
}
|
|
if !tc.wantOK {
|
|
return
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("got %+v, want %+v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseLineV2(t *testing.T) {
|
|
// v2 \t host \t remote_addr \t method \t uri \t status \t bytes_sent \t request_length \t
|
|
// request_time \t upstream_response_time \t upstream_status \t is_tor \t asn \t
|
|
// source_tag \t server_addr \t scheme
|
|
full := "v2\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1500\t421\t0.043\t0.012\t200\t0\t12345\tcdn\t10.0.0.1\thttps"
|
|
|
|
tests := []struct {
|
|
name string
|
|
line string
|
|
wantOK bool
|
|
want LogRecord
|
|
}{
|
|
{
|
|
name: "v2 with upstream",
|
|
line: full,
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/api/v1/search",
|
|
Status: "200",
|
|
ASN: 12345,
|
|
Method: "GET",
|
|
BytesSent: 1500,
|
|
RequestLength: 421,
|
|
RequestTime: 0.043,
|
|
UpstreamResponseTime: 0.012,
|
|
UpstreamStatus: "200",
|
|
HasUpstream: true,
|
|
SourceTag: "cdn",
|
|
},
|
|
},
|
|
{
|
|
name: "v2 no upstream (dash sentinels)",
|
|
line: "v2\twww.example.com\t1.2.3.4\tGET\t/static.html\t200\t900\t300\t0.001\t-\t-\t0\t0\tdirect\t10.0.0.1\thttps",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/static.html",
|
|
Status: "200",
|
|
Method: "GET",
|
|
BytesSent: 900,
|
|
RequestLength: 300,
|
|
RequestTime: 0.001,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "v2 no upstream (empty fields)",
|
|
line: "v2\thh\t1.2.3.4\tGET\t/\t301\t200\t100\t0\t\t\t0\t0\tdirect\t10.0.0.1\thttps",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "hh",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "301",
|
|
Method: "GET",
|
|
BytesSent: 200,
|
|
RequestLength: 100,
|
|
SourceTag: "direct",
|
|
},
|
|
},
|
|
{
|
|
name: "v2 retried upstreams (comma-separated, last wins)",
|
|
line: "v2\twww.example.com\t1.2.3.4\tGET\t/api\t502\t900\t300\t1.500\t0.500, 1.000\t504, 502\t0\t0\tcdn\t10.0.0.1\thttps",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "www.example.com",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/api",
|
|
Status: "502",
|
|
Method: "GET",
|
|
BytesSent: 900,
|
|
RequestLength: 300,
|
|
RequestTime: 1.500,
|
|
UpstreamResponseTime: 1.000,
|
|
UpstreamStatus: "502",
|
|
HasUpstream: true,
|
|
SourceTag: "cdn",
|
|
},
|
|
},
|
|
{
|
|
name: "v2 wrong field count (14) rejected",
|
|
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v2 bad IP rejected",
|
|
line: "v2\thh\tnope\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1\thttps",
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "v2 bad upstream time leaves HasUpstream=false",
|
|
line: "v2\thh\t1.2.3.4\tGET\t/\t200\t0\t0\t0\tnotanumber\t200\t0\t0\tcdn\t10.0.0.1\thttps",
|
|
wantOK: true,
|
|
want: LogRecord{
|
|
Website: "hh",
|
|
ClientPrefix: "1.2.3.0/24",
|
|
URI: "/",
|
|
Status: "200",
|
|
Method: "GET",
|
|
SourceTag: "cdn",
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := ParseLine(tc.line, 24, 48)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
|
|
}
|
|
if !tc.wantOK {
|
|
return
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("got %+v, want %+v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseLineRejections(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
line string
|
|
}{
|
|
{"empty line", ""},
|
|
{"no tab at all", "v1"},
|
|
{"unknown version v3", "v3\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp"},
|
|
{"missing version prefix (legacy file format)", "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api\t200\t1452\t0.043"},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if _, ok := ParseLine(tc.line, 24, 48); ok {
|
|
t.Errorf("expected rejection for %q", tc.line)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTruncateIP(t *testing.T) {
|
|
tests := []struct {
|
|
addr string
|
|
want string
|
|
}{
|
|
{"1.2.3.4", "1.2.3.0/24"},
|
|
{"192.168.100.200", "192.168.100.0/24"},
|
|
{"2001:db8:cafe:babe::1", "2001:db8:cafe::/48"},
|
|
{"::1", "::/48"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
got, ok := truncateIP(tc.addr, 24, 48)
|
|
if !ok {
|
|
t.Errorf("truncateIP(%q) returned not-ok", tc.addr)
|
|
continue
|
|
}
|
|
if got != tc.want {
|
|
t.Errorf("truncateIP(%q) = %q, want %q", tc.addr, got, tc.want)
|
|
}
|
|
}
|
|
}
|