RELEASE 1.0.1: v2 log format, source_tag-labeled metrics, lint cleanup
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).
Prometheus exposition is rewritten:
- nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
a source_tag label.
- nginx_http_requests_by_source_total gains status_class.
- New v2-only metrics: nginx_http_request_bytes,
nginx_http_upstream_duration_seconds,
nginx_http_upstream_requests_total{status_class}.
- Dropped nginx_http_response_body_bytes_by_source (subsumed by the
dual-labeled bytes_sent metric).
Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).
Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
This commit is contained in:
+122
-32
@@ -6,22 +6,22 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPromStoreIngestBodyBuckets(t *testing.T) {
|
||||
func TestPromStoreIngestBytesBuckets(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// 512 bytes: > 256, ≤ 1024 → bucket[0] stays 0, buckets[1..N] get 1
|
||||
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", BodyBytesSent: 512})
|
||||
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", BytesSent: 512, SourceTag: "direct"})
|
||||
|
||||
ps.mu.Lock()
|
||||
be := ps.body["example.com"]
|
||||
be := ps.bytesSent[hostSourceKey{"example.com", "direct"}]
|
||||
ps.mu.Unlock()
|
||||
|
||||
if be == nil {
|
||||
t.Fatal("expected body entry, got nil")
|
||||
t.Fatal("expected bytes entry, got nil")
|
||||
}
|
||||
if be.buckets[0] != 0 { // le=256: 512 > 256
|
||||
if be.buckets[0] != 0 {
|
||||
t.Errorf("le=256 bucket = %d, want 0", be.buckets[0])
|
||||
}
|
||||
if be.buckets[1] != 1 { // le=1024: 512 ≤ 1024
|
||||
if be.buckets[1] != 1 {
|
||||
t.Errorf("le=1024 bucket = %d, want 1", be.buckets[1])
|
||||
}
|
||||
for i := 2; i <= promNumBodyBounds; i++ {
|
||||
@@ -37,24 +37,21 @@ func TestPromStoreIngestBodyBuckets(t *testing.T) {
|
||||
func TestPromStoreIngestTimeBuckets(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// 0.075s: > 0.05, ≤ 0.1
|
||||
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", RequestTime: 0.075})
|
||||
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", RequestTime: 0.075, SourceTag: "direct"})
|
||||
|
||||
ps.mu.Lock()
|
||||
te := ps.reqTime["example.com"]
|
||||
te := ps.requestDuration[hostSourceKey{"example.com", "direct"}]
|
||||
ps.mu.Unlock()
|
||||
|
||||
if te == nil {
|
||||
t.Fatal("expected time entry, got nil")
|
||||
t.Fatal("expected request_duration entry, got nil")
|
||||
}
|
||||
// le=0.05 (index 3): 0.075 > 0.05 → 0
|
||||
if te.buckets[3] != 0 {
|
||||
t.Errorf("le=0.05 bucket = %d, want 0", te.buckets[3])
|
||||
}
|
||||
// le=0.1 (index 4): 0.075 ≤ 0.1 → 1
|
||||
if te.buckets[4] != 1 {
|
||||
t.Errorf("le=0.1 bucket = %d, want 1", te.buckets[4])
|
||||
}
|
||||
// +Inf (last): always 1
|
||||
if te.buckets[promNumTimeBounds] != 1 {
|
||||
t.Errorf("+Inf bucket = %d, want 1", te.buckets[promNumTimeBounds])
|
||||
}
|
||||
@@ -83,7 +80,7 @@ func TestPromStoreServeHTTP(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
ps.Ingest(LogRecord{
|
||||
Website: "example.com", Method: "GET", Status: "200",
|
||||
BodyBytesSent: 100, RequestTime: 0.042,
|
||||
BytesSent: 100, RequestTime: 0.042, SourceTag: "direct",
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("GET", "/metrics", nil)
|
||||
@@ -95,13 +92,15 @@ func TestPromStoreServeHTTP(t *testing.T) {
|
||||
checks := []string{
|
||||
"# TYPE nginx_http_requests_total counter",
|
||||
`nginx_http_requests_total{host="example.com",method="GET",status="200"} 1`,
|
||||
"# TYPE nginx_http_response_body_bytes histogram",
|
||||
`nginx_http_response_body_bytes_bucket{host="example.com",le="256"} 1`, // 100 ≤ 256
|
||||
`nginx_http_response_body_bytes_count{host="example.com"} 1`,
|
||||
`nginx_http_response_body_bytes_sum{host="example.com"} 100`,
|
||||
"# TYPE nginx_http_bytes_sent histogram",
|
||||
`nginx_http_bytes_sent_bucket{host="example.com",source_tag="direct",le="256"} 1`,
|
||||
`nginx_http_bytes_sent_count{host="example.com",source_tag="direct"} 1`,
|
||||
`nginx_http_bytes_sent_sum{host="example.com",source_tag="direct"} 100`,
|
||||
"# TYPE nginx_http_request_duration_seconds histogram",
|
||||
`nginx_http_request_duration_seconds_bucket{host="example.com",le="0.05"} 1`, // 0.042 ≤ 0.05
|
||||
`nginx_http_request_duration_seconds_count{host="example.com"} 1`,
|
||||
`nginx_http_request_duration_seconds_bucket{host="example.com",source_tag="direct",le="0.05"} 1`,
|
||||
`nginx_http_request_duration_seconds_count{host="example.com",source_tag="direct"} 1`,
|
||||
"# TYPE nginx_http_requests_by_source_total counter",
|
||||
`nginx_http_requests_by_source_total{source_tag="direct",status_class="2xx"} 1`,
|
||||
}
|
||||
for _, want := range checks {
|
||||
if !strings.Contains(body, want) {
|
||||
@@ -110,12 +109,12 @@ func TestPromStoreServeHTTP(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromStoreSourceTagRollup(t *testing.T) {
|
||||
func TestPromStoreSourceTagAndStatusClass(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// same host, two tags; each tag should appear with its own series.
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 100, SourceTag: "direct"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 300, SourceTag: "cdn"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 100, SourceTag: "cdn"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "direct"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 300, SourceTag: "cdn"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "404", BytesSent: 100, SourceTag: "cdn"})
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "500", BytesSent: 50, SourceTag: "cdn"})
|
||||
|
||||
req := httptest.NewRequest("GET", "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
@@ -124,13 +123,17 @@ func TestPromStoreSourceTagRollup(t *testing.T) {
|
||||
|
||||
checks := []string{
|
||||
"# TYPE nginx_http_requests_by_source_total counter",
|
||||
`nginx_http_requests_by_source_total{source_tag="direct"} 1`,
|
||||
`nginx_http_requests_by_source_total{source_tag="cdn"} 2`,
|
||||
"# TYPE nginx_http_response_body_bytes_by_source histogram",
|
||||
`nginx_http_response_body_bytes_by_source_sum{source_tag="direct"} 100`,
|
||||
`nginx_http_response_body_bytes_by_source_sum{source_tag="cdn"} 400`,
|
||||
// host-series totals are unchanged (one row, counting 3 requests).
|
||||
`nginx_http_requests_total{host="h",method="GET",status="200"} 3`,
|
||||
`nginx_http_requests_by_source_total{source_tag="direct",status_class="2xx"} 1`,
|
||||
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="2xx"} 1`,
|
||||
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="4xx"} 1`,
|
||||
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="5xx"} 1`,
|
||||
// dual-labeled histogram subsumes the old by-source bytes metric
|
||||
`nginx_http_bytes_sent_sum{host="h",source_tag="direct"} 100`,
|
||||
`nginx_http_bytes_sent_sum{host="h",source_tag="cdn"} 450`,
|
||||
// host counter unchanged
|
||||
`nginx_http_requests_total{host="h",method="GET",status="200"} 2`,
|
||||
`nginx_http_requests_total{host="h",method="GET",status="404"} 1`,
|
||||
`nginx_http_requests_total{host="h",method="GET",status="500"} 1`,
|
||||
}
|
||||
for _, want := range checks {
|
||||
if !strings.Contains(body, want) {
|
||||
@@ -139,6 +142,79 @@ func TestPromStoreSourceTagRollup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromStoreUpstreamMetrics(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// One upstream-served 200, one upstream-served 502, one no-upstream 200.
|
||||
ps.Ingest(LogRecord{
|
||||
Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "cdn",
|
||||
RequestLength: 500,
|
||||
HasUpstream: true,
|
||||
UpstreamResponseTime: 0.020,
|
||||
UpstreamStatus: "200",
|
||||
})
|
||||
ps.Ingest(LogRecord{
|
||||
Website: "h", Method: "GET", Status: "502", BytesSent: 100, SourceTag: "cdn",
|
||||
RequestLength: 500,
|
||||
HasUpstream: true,
|
||||
UpstreamResponseTime: 0.150,
|
||||
UpstreamStatus: "502",
|
||||
})
|
||||
ps.Ingest(LogRecord{
|
||||
Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "direct",
|
||||
RequestLength: 200,
|
||||
})
|
||||
|
||||
req := httptest.NewRequest("GET", "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
ps.ServeHTTP(rec, req)
|
||||
body := rec.Body.String()
|
||||
|
||||
checks := []string{
|
||||
// upstream counter: only the two upstream-served requests
|
||||
`nginx_http_upstream_requests_total{host="h",source_tag="cdn",status_class="2xx"} 1`,
|
||||
`nginx_http_upstream_requests_total{host="h",source_tag="cdn",status_class="5xx"} 1`,
|
||||
// upstream duration histogram only for upstream-served requests
|
||||
`nginx_http_upstream_duration_seconds_count{host="h",source_tag="cdn"} 2`,
|
||||
// request_bytes only observed when RequestLength > 0 — all three here
|
||||
`nginx_http_request_bytes_count{host="h",source_tag="cdn"} 2`,
|
||||
`nginx_http_request_bytes_count{host="h",source_tag="direct"} 1`,
|
||||
}
|
||||
for _, want := range checks {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q in output:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
|
||||
// no-upstream request must not bump upstream counters
|
||||
if strings.Contains(body, `source_tag="direct",status_class=`) &&
|
||||
strings.Contains(body, "nginx_http_upstream_requests_total") {
|
||||
// Better check: ensure no direct-tagged upstream entry
|
||||
for _, bad := range []string{
|
||||
`nginx_http_upstream_requests_total{host="h",source_tag="direct"`,
|
||||
`nginx_http_upstream_duration_seconds_bucket{host="h",source_tag="direct"`,
|
||||
} {
|
||||
if strings.Contains(body, bad) {
|
||||
t.Errorf("unexpected %q in output:\n%s", bad, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromStoreRequestBytesSkippedWhenZero(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// v1 record — RequestLength=0, so request_bytes histogram should be empty.
|
||||
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "cdn"})
|
||||
|
||||
req := httptest.NewRequest("GET", "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
ps.ServeHTTP(rec, req)
|
||||
body := rec.Body.String()
|
||||
|
||||
if strings.Contains(body, "nginx_http_request_bytes_bucket{") {
|
||||
t.Errorf("expected no request_bytes series for v1 record:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromStoreUDPCounters(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
ps.IncUDPPacket()
|
||||
@@ -167,7 +243,6 @@ func TestPromStoreUDPCounters(t *testing.T) {
|
||||
|
||||
func TestPromStoreCounterCap(t *testing.T) {
|
||||
ps := NewPromStore()
|
||||
// Fill to cap with distinct {host,method,status} combos
|
||||
for i := 0; i < promCounterCap+10; i++ {
|
||||
host := strings.Repeat("x", i%10+1) + ".com"
|
||||
status := "200"
|
||||
@@ -183,3 +258,18 @@ func TestPromStoreCounterCap(t *testing.T) {
|
||||
t.Errorf("counter map size %d exceeds cap %d", n, promCounterCap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusClass(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"200": "2xx", "201": "2xx", "299": "2xx",
|
||||
"301": "3xx",
|
||||
"404": "4xx", "418": "4xx",
|
||||
"500": "5xx", "504": "5xx",
|
||||
"100": "other", "": "other", "abc": "other",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := statusClass(in); got != want {
|
||||
t.Errorf("statusClass(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user