ipng/nginx-logtail
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

RELEASE 1.0.1: v2 log format, source_tag-labeled metrics, lint cleanup

Browse Source 
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).

Prometheus exposition is rewritten:

  - nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
    a source_tag label.
  - nginx_http_requests_by_source_total gains status_class.
  - New v2-only metrics: nginx_http_request_bytes,
    nginx_http_upstream_duration_seconds,
    nginx_http_upstream_requests_total{status_class}.
  - Dropped nginx_http_response_body_bytes_by_source (subsumed by the
    dual-labeled bytes_sent metric).

Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).

Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
This commit is contained in:
Pim van Pelt
2026-05-01 15:40:53 +02:00
parent d1a21a7a62
commit 6647f95be4
28 changed files with 931 additions and 724 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/aggregator/aggregator_test.go
+3 -3
View File
@@ -262,7 +262,7 @@ func startFakeCollector(t *testing.T, snaps []*pb.Snapshot) string {
}
srv := grpc.NewServer()
pb.RegisterLogtailServiceServer(srv, &fakeCollector{snaps: snaps})
go srv.Serve(lis)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.Stop)
return lis.Addr().String()
}
@@ -310,7 +310,7 @@ func TestGRPCEndToEnd(t *testing.T) {
}
grpcSrv := grpc.NewServer()
pb.RegisterLogtailServiceServer(grpcSrv, NewServer(cache, "agg-test", NewTargetRegistry(nil)))
go grpcSrv.Serve(lis)
go func() { _ = grpcSrv.Serve(lis) }()
defer grpcSrv.Stop()
conn, err := grpc.NewClient(lis.Addr().String(),
@@ -318,7 +318,7 @@ func TestGRPCEndToEnd(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
client := pb.NewLogtailServiceClient(conn)
qctx, qcancel := context.WithTimeout(context.Background(), 5*time.Second)
defer qcancel()
cmd/aggregator/backfill.go
+1 -1
View File
@@ -84,7 +84,7 @@ func dumpCollector(ctx context.Context, addr string) (fine, coarse []st.Snapshot
if err != nil {
return nil, nil, err
}
defer conn.Close()
defer func() { _ = conn.Close() }()
client := pb.NewLogtailServiceClient(conn)
stream, err := client.DumpSnapshots(ctx, &pb.DumpSnapshotsRequest{})
cmd/aggregator/cache.go
+2 -6
View File
@@ -97,15 +97,11 @@ func (c *Cache) LoadHistorical(fine, coarse []st.Snapshot) {
c.mu.Lock()
defer c.mu.Unlock()
for i, snap := range fine {
c.fineRing[i] = snap
}
copy(c.fineRing[:], fine)
c.fineFilled = len(fine)
c.fineHead = len(fine) % st.FineRingSize
for i, snap := range coarse {
c.coarseRing[i] = snap
}
copy(c.coarseRing[:], coarse)
c.coarseFilled = len(coarse)
c.coarseHead = len(coarse) % st.CoarseRingSize
}
cmd/aggregator/subscriber.go
+1 -1
View File
@@ -77,7 +77,7 @@ func (cs *CollectorSub) stream(ctx context.Context) (bool, error) {
if err != nil {
return false, err
}
defer conn.Close()
defer func() { _ = conn.Close() }()
client := pb.NewLogtailServiceClient(conn)
stream, err := client.StreamSnapshots(ctx, &pb.SnapshotRequest{})
cmd/cli/cli_test.go
+1 -29
View File
@@ -1,7 +1,6 @@
package main
import (
"bytes"
"context"
"encoding/json"
"net"
@@ -11,7 +10,6 @@ import (
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// --- Unit tests ---
@@ -149,21 +147,11 @@ func startFake(t *testing.T, fs *fakeServer) string {
}
srv := grpc.NewServer()
pb.RegisterLogtailServiceServer(srv, fs)
go srv.Serve(lis)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.GracefulStop)
return lis.Addr().String()
}
func dialTest(t *testing.T, addr string) pb.LogtailServiceClient {
t.Helper()
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.Close() })
return pb.NewLogtailServiceClient(conn)
}
// --- TopN tests ---
func TestTopNSingleTarget(t *testing.T) {
@@ -225,23 +213,7 @@ func TestTopNJSON(t *testing.T) {
})
results := fanOutTopN([]string{addr}, nil, pb.GroupBy_WEBSITE, 10, pb.Window_W5M)
var buf bytes.Buffer
// Redirect stdout not needed; call JSON formatter directly.
r := results[0]
// Build expected JSON by calling printTopNJSON with a captured stdout.
// We test indirectly: marshal manually and compare fields.
type entry struct {
Label string `json:"label"`
Count int64 `json:"count"`
}
type out struct {
Source string `json:"source"`
Target string `json:"target"`
Entries []entry `json:"entries"`
}
_ = buf
_ = r
// Verify the response fields are correct for JSON serialization.
if r.resp.Source != "agg" {
t.Errorf("source = %q", r.resp.Source)
}
cmd/cli/cmd_stream.go
+2 -2
View File
@@ -24,7 +24,7 @@ type streamEvent struct {
func runStream(args []string) {
fs := flag.NewFlagSet("stream", flag.ExitOnError)
sf, targetFlag := bindShared(fs)
fs.Parse(args)
_ = fs.Parse(args) // ExitOnError: only returns nil here
sf.resolve(*targetFlag)
filter := buildFilter(sf)
@@ -85,7 +85,7 @@ func streamOnce(ctx context.Context, addr string, filter *pb.Filter, events chan
if err != nil {
return err
}
defer conn.Close()
defer func() { _ = conn.Close() }()
stream, err := client.StreamSnapshots(ctx, &pb.SnapshotRequest{})
if err != nil {
cmd/cli/cmd_targets.go
+3 -3
View File
@@ -18,7 +18,7 @@ func runTargets(args []string) {
fmt.Fprintln(os.Stderr, "usage: logtail-cli targets [--target host:port] [--json]")
fs.PrintDefaults()
}
fs.Parse(args)
_ = fs.Parse(args) // ExitOnError: only returns nil here
sf.resolve(*target)
for _, addr := range sf.targets {
@@ -30,7 +30,7 @@ func runTargets(args []string) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
resp, err := client.ListTargets(ctx, &pb.ListTargetsRequest{})
cancel()
conn.Close()
_ = conn.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "targets: %s: %v\n", addr, err)
continue
@@ -43,7 +43,7 @@ func runTargets(args []string) {
Addr string `json:"addr"`
}
for _, t := range resp.Targets {
json.NewEncoder(os.Stdout).Encode(row{QueryTarget: addr, Name: t.Name, Addr: t.Addr})
_ = json.NewEncoder(os.Stdout).Encode(row{QueryTarget: addr, Name: t.Name, Addr: t.Addr})
}
} else {
if len(sf.targets) > 1 {
cmd/cli/cmd_topn.go
+2 -2
View File
@@ -24,7 +24,7 @@ func runTopN(args []string) {
n := fs.Int("n", 10, "number of entries")
window := fs.String("window", "5m", "time window: 1m 5m 15m 60m 6h 24h")
groupBy := fs.String("group-by", "website", "group by: website prefix uri status")
fs.Parse(args)
_ = fs.Parse(args) // ExitOnError: only returns nil here
sf.resolve(*targetFlag)
win := parseWindow(*window)
@@ -66,7 +66,7 @@ func fanOutTopN(targets []string, filter *pb.Filter, groupBy pb.GroupBy, n int,
results[i].resp = &pb.TopNResponse{}
return
}
defer conn.Close()
defer func() { _ = conn.Close() }()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.TopN(ctx, &pb.TopNRequest{
cmd/cli/cmd_trend.go
+2 -2
View File
@@ -22,7 +22,7 @@ func runTrend(args []string) {
fs := flag.NewFlagSet("trend", flag.ExitOnError)
sf, targetFlag := bindShared(fs)
window := fs.String("window", "5m", "time window: 1m 5m 15m 60m 6h 24h")
fs.Parse(args)
_ = fs.Parse(args) // ExitOnError: only returns nil here
sf.resolve(*targetFlag)
win := parseWindow(*window)
@@ -63,7 +63,7 @@ func fanOutTrend(targets []string, filter *pb.Filter, window pb.Window) []trendR
results[i].resp = &pb.TrendResponse{}
return
}
defer conn.Close()
defer func() { _ = conn.Close() }()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.Trend(ctx, &pb.TrendRequest{
cmd/cli/flags.go
-6
View File
@@ -167,9 +167,3 @@ func parseGroupBy(s string) pb.GroupBy {
panic("unreachable")
}
}
func dieUsage(fs *flag.FlagSet, msg string) {
fmt.Fprintln(os.Stderr, msg)
fs.PrintDefaults()
os.Exit(1)
}
cmd/cli/format.go
+3 -3
View File
@@ -16,17 +16,17 @@ func printTable(w io.Writer, rows [][]string) {
}
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
for i, row := range rows {
fmt.Fprintln(tw, strings.Join(row, "\t"))
_, _ = fmt.Fprintln(tw, strings.Join(row, "\t"))
if i == 0 {
// Print a divider matching the header width.
dashes := make([]string, len(row))
for j, h := range row {
dashes[j] = strings.Repeat("-", len(h))
}
fmt.Fprintln(tw, strings.Join(dashes, "\t"))
_, _ = fmt.Fprintln(tw, strings.Join(dashes, "\t"))
}
}
tw.Flush()
_ = tw.Flush()
}
// fmtCount formats a count with a space as the thousands separator.
cmd/collector/main.go
+1 -1
View File
@@ -128,7 +128,7 @@ func collectPatterns(logPaths, logsFile string) []string {
if err != nil {
log.Fatalf("collector: cannot open --logs-file %s: %v", logsFile, err)
}
defer f.Close()
defer func() { _ = f.Close() }()
sc := bufio.NewScanner(f)
for sc.Scan() {
if p := strings.TrimSpace(sc.Text()); p != "" && !strings.HasPrefix(p, "#") {
cmd/collector/parser.go
+95 -66
View File
@@ -8,87 +8,57 @@ import (
)
// LogRecord holds the dimensions extracted from a single nginx log line.
//
// BytesSent carries $body_bytes_sent for v1 records and $bytes_sent for v2
// records — operators see a small step up when emitters move to v2 because v2
// includes header overhead.
//
// RequestLength, UpstreamResponseTime, UpstreamStatus, HasUpstream are v2-only.
// In v1 records HasUpstream is always false and the related fields are zero.
type LogRecord struct {
Website string
ClientPrefix string
URI string
Status string
IsTor bool
ASN int32
Method string
BodyBytesSent int64
RequestTime float64
SourceTag string
Website string
ClientPrefix string
URI string
Status string
IsTor bool
ASN int32
Method string
BytesSent int64
RequestLength int64
RequestTime float64
UpstreamResponseTime float64
UpstreamStatus string
HasUpstream bool
SourceTag string
}
// fileSourceTag is the SourceTag assigned to records read from on-disk log
// files, which pre-date the tag concept. Mirrors nginx's fallback label.
const fileSourceTag = "direct"
// ParseLine parses a tab-separated logtail log line from a file:
//
// $host \t $remote_addr \t $msec \t $request_method \t $request_uri \t $status \t $body_bytes_sent \t $request_time \t $is_tor \t $asn
//
// The is_tor (field 9) and asn (field 10) fields are optional for backward
// compatibility with older log files that omit them; they default to false/0
// when absent. SourceTag is always set to "direct" (file origin has no tag).
// Returns false for lines with fewer than 8 fields.
// ParseLine parses a versioned nginx-logtail line. Both file ingest and UDP
// ingest funnel through here. Every line MUST start with "v<N>\t"; unknown or
// missing versions return false so operators can ship a parser update before
// the emitter switches.
func ParseLine(line string, v4bits, v6bits int) (LogRecord, bool) {
fields := strings.SplitN(line, "\t", 10)
if len(fields) < 8 {
return LogRecord{}, false
}
prefix, ok := truncateIP(fields[1], v4bits, v6bits)
if !ok {
return LogRecord{}, false
}
isTor := len(fields) >= 9 && fields[8] == "1"
var asn int32
if len(fields) == 10 {
if n, err := strconv.ParseInt(fields[9], 10, 32); err == nil {
asn = int32(n)
}
}
return LogRecord{
Website: fields[0],
ClientPrefix: prefix,
URI: stripQuery(fields[4]),
Status: fields[5],
IsTor: isTor,
ASN: asn,
Method: fields[3],
BodyBytesSent: parseInt(fields[6]),
RequestTime: parseFloat(fields[7]),
SourceTag: fileSourceTag,
}, true
}
// ParseUDPLine dispatches on the version prefix emitted by
// nginx-ipng-stats-plugin's ipng_stats_logtail directive. The wire format is
// "v<N>\t<payload>", where <payload> is version-specific. Unknown or missing
// versions return false so operators can roll out a v2 parser before
// upgrading emitters.
func ParseUDPLine(line string, v4bits, v6bits int) (LogRecord, bool) {
i := strings.IndexByte(line, '\t')
if i < 0 {
return LogRecord{}, false
}
switch line[:i] {
case "v1":
return parseUDPLineV1(line[i+1:], v4bits, v6bits)
return parseV1(line[i+1:], v4bits, v6bits)
case "v2":
return parseV2(line[i+1:], v4bits, v6bits)
default:
return LogRecord{}, false
}
}
// parseUDPLineV1 parses the v1 payload (12 tab-separated fields):
// parseV1 parses the v1 payload (12 tab-separated fields):
//
// $host \t $remote_addr \t $request_method \t $request_uri \t $status \t
// $body_bytes_sent \t $request_time \t $is_tor \t $asn \t
// $ipng_source_tag \t $server_addr \t $scheme
//
// server_addr and scheme are parsed but discarded.
func parseUDPLineV1(payload string, v4bits, v6bits int) (LogRecord, bool) {
// $server_addr and $scheme are parsed but discarded.
func parseV1(payload string, v4bits, v6bits int) (LogRecord, bool) {
fields := strings.Split(payload, "\t")
if len(fields) != 12 {
return LogRecord{}, false
@@ -102,17 +72,76 @@ func parseUDPLineV1(payload string, v4bits, v6bits int) (LogRecord, bool) {
asn = int32(n)
}
return LogRecord{
Website: fields[0],
ClientPrefix: prefix,
URI: stripQuery(fields[3]),
Status: fields[4],
IsTor: fields[7] == "1",
ASN: asn,
Method: fields[2],
BytesSent: parseInt(fields[5]),
RequestTime: parseFloat(fields[6]),
SourceTag: fields[9],
}, true
}
// parseV2 parses the v2 payload (15 tab-separated fields):
//
// $host \t $remote_addr \t $request_method \t $request_uri \t $status \t
// $bytes_sent \t $request_length \t $request_time \t
// $upstream_response_time \t $upstream_status \t
// $is_tor \t $asn \t $ipng_source_tag \t $server_addr \t $scheme
//
// $upstream_response_time and $upstream_status are "-" (or empty) when nginx
// served the response directly — HasUpstream is left false in that case.
// When nginx retried across multiple upstreams the fields are comma-separated;
// the parser keeps the last entry, since that's the upstream that actually
// served the response. $server_addr and $scheme are parsed but discarded.
func parseV2(payload string, v4bits, v6bits int) (LogRecord, bool) {
fields := strings.Split(payload, "\t")
if len(fields) != 15 {
return LogRecord{}, false
}
prefix, ok := truncateIP(fields[1], v4bits, v6bits)
if !ok {
return LogRecord{}, false
}
var asn int32
if n, err := strconv.ParseInt(fields[11], 10, 32); err == nil {
asn = int32(n)
}
r := LogRecord{
Website: fields[0],
ClientPrefix: prefix,
URI: stripQuery(fields[3]),
Status: fields[4],
IsTor: fields[7] == "1",
IsTor: fields[10] == "1",
ASN: asn,
Method: fields[2],
BodyBytesSent: parseInt(fields[5]),
RequestTime: parseFloat(fields[6]),
SourceTag: fields[9],
}, true
BytesSent: parseInt(fields[5]),
RequestLength: parseInt(fields[6]),
RequestTime: parseFloat(fields[7]),
SourceTag: fields[12],
}
if fields[8] != "-" && fields[8] != "" {
timeStr := lastCommaPart(fields[8])
statusStr := lastCommaPart(fields[9])
if t, err := strconv.ParseFloat(timeStr, 64); err == nil {
r.UpstreamResponseTime = t
r.UpstreamStatus = statusStr
r.HasUpstream = true
}
}
return r, true
}
// lastCommaPart returns the substring after the last ", " (nginx's separator
// for retried upstreams). Plain values pass through unchanged.
func lastCommaPart(s string) string {
if i := strings.LastIndex(s, ", "); i >= 0 {
return s[i+2:]
}
return s
}
func stripQuery(uri string) string {
cmd/collector/parser_test.go
+165 -246
View File
@@ -4,215 +4,7 @@ import (
"testing"
)
func TestParseLine(t *testing.T) {
good := "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api/v1/search?q=foo&x=1\t200\t1452\t0.043"
tests := []struct {
name string
line string
wantOK bool
want LogRecord
}{
{
name: "normal IPv4 line strips query string",
line: good,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
Method: "GET",
BodyBytesSent: 1452,
RequestTime: 0.043,
SourceTag: "direct",
},
},
{
name: "URI with no query string",
line: "host\t10.0.0.1\t0\tPOST\t/submit\t201\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "host",
ClientPrefix: "10.0.0.0/24",
URI: "/submit",
Status: "201",
Method: "POST",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "IPv6 address truncated to /48",
line: "host\t2001:db8:cafe::1\t0\tGET\t/\t200\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "host",
ClientPrefix: "2001:db8:cafe::/48",
URI: "/",
Status: "200",
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "too few fields returns false",
line: "host\t1.2.3.4\t0\tGET\t/",
wantOK: false,
},
{
name: "empty line returns false",
line: "",
wantOK: false,
},
{
name: "invalid IP returns false",
line: "host\tnot-an-ip\t0\tGET\t/\t200\t0\t0.001",
wantOK: false,
},
{
name: "status 429",
line: "api.example.com\t5.6.7.8\t0\tGET\t/rate-limited\t429\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "api.example.com",
ClientPrefix: "5.6.7.0/24",
URI: "/rate-limited",
Status: "429",
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "is_tor=1 sets IsTor true",
line: "tor.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
wantOK: true,
want: LogRecord{
Website: "tor.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "is_tor=0 sets IsTor false",
line: "normal.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0",
wantOK: true,
want: LogRecord{
Website: "normal.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "missing is_tor field defaults to false (backward compat)",
line: "old.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001",
wantOK: true,
want: LogRecord{
Website: "old.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "asn field parsed",
line: "asn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\t12345",
wantOK: true,
want: LogRecord{
Website: "asn.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "asn field with is_tor=1",
line: "both.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1\t65535",
wantOK: true,
want: LogRecord{
Website: "both.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "missing asn field defaults to 0 (backward compat)",
line: "noasn.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t1",
wantOK: true,
want: LogRecord{
Website: "noasn.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: true,
ASN: 0,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "invalid asn field defaults to 0",
line: "badann.example.com\t1.2.3.4\t0\tGET\t/\t200\t0\t0.001\t0\tnot-a-number",
wantOK: true,
want: LogRecord{
Website: "badann.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
IsTor: false,
ASN: 0,
Method: "GET",
RequestTime: 0.001,
SourceTag: "direct",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseLine ok=%v, want %v", ok, tc.wantOK)
}
if !tc.wantOK {
return
}
if got != tc.want {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
}
}
func TestParseUDPLine(t *testing.T) {
func TestParseLineV1(t *testing.T) {
// v1 \t host \t remote_addr \t method \t uri \t status \t body_bytes \t req_time \t
// is_tor \t asn \t source_tag \t server_addr \t scheme
good := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
@@ -228,16 +20,16 @@ func TestParseUDPLine(t *testing.T) {
line: good,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
BodyBytesSent: 1452,
RequestTime: 0.043,
SourceTag: "cdn",
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
IsTor: false,
ASN: 12345,
Method: "GET",
BytesSent: 1452,
RequestTime: 0.043,
SourceTag: "cdn",
},
},
{
@@ -245,16 +37,16 @@ func TestParseUDPLine(t *testing.T) {
line: "v1\th\t2001:db8::1\tGET\t/\t200\t0\t0\t1\t65535\tdirect\t::1\thttp",
wantOK: true,
want: LogRecord{
Website: "h",
ClientPrefix: "2001:db8::/48",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
BodyBytesSent: 0,
RequestTime: 0,
SourceTag: "direct",
Website: "h",
ClientPrefix: "2001:db8::/48",
URI: "/",
Status: "200",
IsTor: true,
ASN: 65535,
Method: "GET",
BytesSent: 0,
RequestTime: 0,
SourceTag: "direct",
},
},
{
@@ -272,28 +64,13 @@ func TestParseUDPLine(t *testing.T) {
line: "v1\th\tnope\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "unknown version rejected (future v2)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "missing version prefix rejected (legacy 12-field line)",
line: "www.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp",
wantOK: false,
},
{
name: "no tab at all rejected",
line: "v1",
wantOK: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseUDPLine(tc.line, 24, 48)
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseUDPLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
}
if !tc.wantOK {
return
@@ -305,6 +82,148 @@ func TestParseUDPLine(t *testing.T) {
}
}
func TestParseLineV2(t *testing.T) {
// v2 \t host \t remote_addr \t method \t uri \t status \t bytes_sent \t request_length \t
// request_time \t upstream_response_time \t upstream_status \t is_tor \t asn \t
// source_tag \t server_addr \t scheme
full := "v2\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1500\t421\t0.043\t0.012\t200\t0\t12345\tcdn\t10.0.0.1\thttps"
tests := []struct {
name string
line string
wantOK bool
want LogRecord
}{
{
name: "v2 with upstream",
line: full,
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api/v1/search",
Status: "200",
ASN: 12345,
Method: "GET",
BytesSent: 1500,
RequestLength: 421,
RequestTime: 0.043,
UpstreamResponseTime: 0.012,
UpstreamStatus: "200",
HasUpstream: true,
SourceTag: "cdn",
},
},
{
name: "v2 no upstream (dash sentinels)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/static.html\t200\t900\t300\t0.001\t-\t-\t0\t0\tdirect\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/static.html",
Status: "200",
Method: "GET",
BytesSent: 900,
RequestLength: 300,
RequestTime: 0.001,
SourceTag: "direct",
},
},
{
name: "v2 no upstream (empty fields)",
line: "v2\thh\t1.2.3.4\tGET\t/\t301\t200\t100\t0\t\t\t0\t0\tdirect\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "hh",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "301",
Method: "GET",
BytesSent: 200,
RequestLength: 100,
SourceTag: "direct",
},
},
{
name: "v2 retried upstreams (comma-separated, last wins)",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/api\t502\t900\t300\t1.500\t0.500, 1.000\t504, 502\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "www.example.com",
ClientPrefix: "1.2.3.0/24",
URI: "/api",
Status: "502",
Method: "GET",
BytesSent: 900,
RequestLength: 300,
RequestTime: 1.500,
UpstreamResponseTime: 1.000,
UpstreamStatus: "502",
HasUpstream: true,
SourceTag: "cdn",
},
},
{
name: "v2 wrong field count (14) rejected",
line: "v2\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1",
wantOK: false,
},
{
name: "v2 bad IP rejected",
line: "v2\thh\tnope\tGET\t/\t200\t0\t0\t0\t-\t-\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: false,
},
{
name: "v2 bad upstream time leaves HasUpstream=false",
line: "v2\thh\t1.2.3.4\tGET\t/\t200\t0\t0\t0\tnotanumber\t200\t0\t0\tcdn\t10.0.0.1\thttps",
wantOK: true,
want: LogRecord{
Website: "hh",
ClientPrefix: "1.2.3.0/24",
URI: "/",
Status: "200",
Method: "GET",
SourceTag: "cdn",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, ok := ParseLine(tc.line, 24, 48)
if ok != tc.wantOK {
t.Fatalf("ParseLine ok=%v, want %v; got=%+v", ok, tc.wantOK, got)
}
if !tc.wantOK {
return
}
if got != tc.want {
t.Errorf("got %+v, want %+v", got, tc.want)
}
})
}
}
func TestParseLineRejections(t *testing.T) {
tests := []struct {
name string
line string
}{
{"empty line", ""},
{"no tab at all", "v1"},
{"unknown version v3", "v3\twww.example.com\t1.2.3.4\tGET\t/\t200\t0\t0\t0\t0\ttag\t10.0.0.1\thttp"},
{"missing version prefix (legacy file format)", "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api\t200\t1452\t0.043"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, ok := ParseLine(tc.line, 24, 48); ok {
t.Errorf("expected rejection for %q", tc.line)
}
})
}
}
func TestTruncateIP(t *testing.T) {
tests := []struct {
addr string
cmd/collector/prom.go
+252 -135
View File
@@ -14,27 +14,46 @@ const promNumBodyBounds = 7
var promBodyBounds = [promNumBodyBounds]int64{256, 1024, 4096, 16384, 65536, 262144, 1048576}
// Request-time histogram bucket upper bounds in seconds (standard Prometheus defaults).
// Duration histogram bucket upper bounds in seconds (Prometheus defaults).
const promNumTimeBounds = 11
var promTimeBounds = [promNumTimeBounds]float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
const promCounterCap = 250_000 // safety cap on {host,method,status} counter entries
// promCounterKey is the label set for per-request counters.
// promCounterKey is the label set for the per-request counter.
type promCounterKey struct {
Host string
Method string
Status string
}
// promBodyEntry holds the body_bytes_sent histogram for one host.
// hostSourceKey labels histograms by {host, source_tag}.
type hostSourceKey struct {
Host string
SourceTag string
}
// sourceClassKey labels the source-tag rollup counter.
type sourceClassKey struct {
SourceTag string
StatusClass string
}
// upstreamKey labels the upstream-only request counter.
type upstreamKey struct {
Host string
SourceTag string
StatusClass string // class of $upstream_status
}
// promBodyEntry holds one body-size histogram (one label-set worth).
type promBodyEntry struct {
buckets [promNumBodyBounds + 1]int64 // indices 0..N-1: le=bound[i]; index N: le=+Inf
sum int64
}
// promTimeEntry holds the request_time histogram for one host.
// promTimeEntry holds one duration histogram (one label-set worth).
type promTimeEntry struct {
buckets [promNumTimeBounds + 1]int64
sum float64
@@ -45,30 +64,31 @@ type promTimeEntry struct {
// Ingest must be called from exactly one goroutine (the store's Run goroutine).
// ServeHTTP may be called from any number of goroutines concurrently.
type PromStore struct {
mu sync.Mutex
counters map[promCounterKey]int64
body map[string]*promBodyEntry // keyed by host
reqTime map[string]*promTimeEntry // keyed by host
mu sync.Mutex
counters map[promCounterKey]int64
bytesSent map[hostSourceKey]*promBodyEntry
requestDuration map[hostSourceKey]*promTimeEntry
requestBytes map[hostSourceKey]*promBodyEntry // v2 only
upstreamDuration map[hostSourceKey]*promTimeEntry // v2 only
upstreamCounters map[upstreamKey]int64 // v2 only
sourceCounters map[sourceClassKey]int64
// per-source_tag rollups (parallel series, not a cross-product with host)
sourceCounters map[string]int64 // keyed by source_tag
sourceBody map[string]*promBodyEntry // keyed by source_tag
// UDP ingest counters — protected by their own atomic-friendly lock.
udpMu sync.Mutex
udpPacketsReceived int64 // datagrams read off the socket
udpLoglinesSuccess int64 // successfully parsed
udpLoglinesConsumed int64 // successfully forwarded to the store channel
udpMu sync.Mutex
udpPacketsReceived int64
udpLoglinesSuccess int64
udpLoglinesConsumed int64
}
// NewPromStore returns an empty PromStore ready for use.
func NewPromStore() *PromStore {
return &PromStore{
counters: make(map[promCounterKey]int64, 1024),
body: make(map[string]*promBodyEntry, 64),
reqTime: make(map[string]*promTimeEntry, 64),
sourceCounters: make(map[string]int64, 32),
sourceBody: make(map[string]*promBodyEntry, 32),
counters: make(map[promCounterKey]int64, 1024),
bytesSent: make(map[hostSourceKey]*promBodyEntry, 64),
requestDuration: make(map[hostSourceKey]*promTimeEntry, 64),
requestBytes: make(map[hostSourceKey]*promBodyEntry, 64),
upstreamDuration: make(map[hostSourceKey]*promTimeEntry, 64),
upstreamCounters: make(map[upstreamKey]int64, 64),
sourceCounters: make(map[sourceClassKey]int64, 32),
}
}
@@ -76,8 +96,11 @@ func NewPromStore() *PromStore {
// Must be called from a single goroutine.
func (p *PromStore) Ingest(r LogRecord) {
p.mu.Lock()
defer p.mu.Unlock()
// --- per-{host,method,status} request counter ---
hsk := hostSourceKey{Host: r.Website, SourceTag: r.SourceTag}
// nginx_http_requests_total{host,method,status} — capped.
ck := promCounterKey{Host: r.Website, Method: r.Method, Status: r.Status}
if _, ok := p.counters[ck]; ok {
p.counters[ck]++
@@ -85,37 +108,54 @@ func (p *PromStore) Ingest(r LogRecord) {
p.counters[ck] = 1
}
// --- body_bytes_sent histogram (keyed by host only) ---
observeBody(p.body, r.Website, r.BodyBytesSent)
// --- request_time histogram (keyed by host only) ---
te, ok := p.reqTime[r.Website]
if !ok {
te = &promTimeEntry{}
p.reqTime[r.Website] = te
observeBody(p.bytesSent, hsk, r.BytesSent)
observeTime(p.requestDuration, hsk, r.RequestTime)
if r.RequestLength > 0 {
observeBody(p.requestBytes, hsk, r.RequestLength)
}
for i, bound := range promTimeBounds {
if r.RequestTime <= bound {
te.buckets[i]++
}
p.sourceCounters[sourceClassKey{
SourceTag: r.SourceTag,
StatusClass: statusClass(r.Status),
}]++
if r.HasUpstream {
observeTime(p.upstreamDuration, hsk, r.UpstreamResponseTime)
p.upstreamCounters[upstreamKey{
Host: r.Website,
SourceTag: r.SourceTag,
StatusClass: statusClass(r.UpstreamStatus),
}]++
}
te.buckets[promNumTimeBounds]++ // +Inf
te.sum += r.RequestTime
// --- per-source_tag rollups ---
p.sourceCounters[r.SourceTag]++
observeBody(p.sourceBody, r.SourceTag, r.BodyBytesSent)
p.mu.Unlock()
}
// IncUDPPacket, IncUDPSuccess, and IncUDPConsumed bump their respective
// UDP ingest counters. They are called from the UDP listener goroutine.
// IncUDPPacket, IncUDPSuccess, IncUDPConsumed bump UDP-ingest counters from
// the listener goroutine.
func (p *PromStore) IncUDPPacket() { p.udpMu.Lock(); p.udpPacketsReceived++; p.udpMu.Unlock() }
func (p *PromStore) IncUDPSuccess() { p.udpMu.Lock(); p.udpLoglinesSuccess++; p.udpMu.Unlock() }
func (p *PromStore) IncUDPConsumed() { p.udpMu.Lock(); p.udpLoglinesConsumed++; p.udpMu.Unlock() }
func observeBody(m map[string]*promBodyEntry, key string, bytes int64) {
// statusClass folds an HTTP status code into 2xx/3xx/4xx/5xx, with anything
// else falling to "other" (including empty input).
func statusClass(status string) string {
if status == "" {
return "other"
}
switch status[0] {
case '2':
return "2xx"
case '3':
return "3xx"
case '4':
return "4xx"
case '5':
return "5xx"
default:
return "other"
}
}
func observeBody(m map[hostSourceKey]*promBodyEntry, key hostSourceKey, bytes int64) {
e, ok := m[key]
if !ok {
e = &promBodyEntry{}
@@ -126,53 +166,77 @@ func observeBody(m map[string]*promBodyEntry, key string, bytes int64) {
e.buckets[i]++
}
}
e.buckets[promNumBodyBounds]++ // +Inf
e.buckets[promNumBodyBounds]++
e.sum += bytes
}
func observeTime(m map[hostSourceKey]*promTimeEntry, key hostSourceKey, seconds float64) {
e, ok := m[key]
if !ok {
e = &promTimeEntry{}
m[key] = e
}
for i, bound := range promTimeBounds {
if seconds <= bound {
e.buckets[i]++
}
}
e.buckets[promNumTimeBounds]++
e.sum += seconds
}
// ServeHTTP renders all metrics in the Prometheus text exposition format (0.0.4).
func (p *PromStore) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
// Snapshot everything under the lock, then render without holding it.
p.mu.Lock()
type counterSnap struct {
k promCounterKey
v int64
}
type bodySnap struct {
k hostSourceKey
e promBodyEntry
}
type timeSnap struct {
k hostSourceKey
e promTimeEntry
}
type upstreamCounterSnap struct {
k upstreamKey
v int64
}
type sourceCounterSnap struct {
k sourceClassKey
v int64
}
p.mu.Lock()
counters := make([]counterSnap, 0, len(p.counters))
for k, v := range p.counters {
counters = append(counters, counterSnap{k, v})
}
type bodySnap struct {
label string
e promBodyEntry
bytesSnaps := make([]bodySnap, 0, len(p.bytesSent))
for k, e := range p.bytesSent {
bytesSnaps = append(bytesSnaps, bodySnap{k, *e})
}
bodySnaps := make([]bodySnap, 0, len(p.body))
for h, e := range p.body {
bodySnaps = append(bodySnaps, bodySnap{h, *e})
requestBytesSnaps := make([]bodySnap, 0, len(p.requestBytes))
for k, e := range p.requestBytes {
requestBytesSnaps = append(requestBytesSnaps, bodySnap{k, *e})
}
type timeSnap struct {
host string
e promTimeEntry
requestDurationSnaps := make([]timeSnap, 0, len(p.requestDuration))
for k, e := range p.requestDuration {
requestDurationSnaps = append(requestDurationSnaps, timeSnap{k, *e})
}
timeSnaps := make([]timeSnap, 0, len(p.reqTime))
for h, e := range p.reqTime {
timeSnaps = append(timeSnaps, timeSnap{h, *e})
upstreamDurationSnaps := make([]timeSnap, 0, len(p.upstreamDuration))
for k, e := range p.upstreamDuration {
upstreamDurationSnaps = append(upstreamDurationSnaps, timeSnap{k, *e})
}
type sourceCounterSnap struct {
tag string
v int64
upstreamCounters := make([]upstreamCounterSnap, 0, len(p.upstreamCounters))
for k, v := range p.upstreamCounters {
upstreamCounters = append(upstreamCounters, upstreamCounterSnap{k, v})
}
sourceCounters := make([]sourceCounterSnap, 0, len(p.sourceCounters))
for t, v := range p.sourceCounters {
sourceCounters = append(sourceCounters, sourceCounterSnap{t, v})
}
sourceBodySnaps := make([]bodySnap, 0, len(p.sourceBody))
for t, e := range p.sourceBody {
sourceBodySnaps = append(sourceBodySnaps, bodySnap{t, *e})
for k, v := range p.sourceCounters {
sourceCounters = append(sourceCounters, sourceCounterSnap{k, v})
}
p.mu.Unlock()
@@ -183,7 +247,6 @@ func (p *PromStore) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
udpConsumed := p.udpLoglinesConsumed
p.udpMu.Unlock()
// Sort for stable, human-readable output.
sort.Slice(counters, func(i, j int) bool {
a, b := counters[i].k, counters[j].k
if a.Host != b.Host {
@@ -194,85 +257,139 @@ func (p *PromStore) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
}
return a.Status < b.Status
})
sort.Slice(bodySnaps, func(i, j int) bool { return bodySnaps[i].label < bodySnaps[j].label })
sort.Slice(timeSnaps, func(i, j int) bool { return timeSnaps[i].host < timeSnaps[j].host })
sort.Slice(sourceCounters, func(i, j int) bool { return sourceCounters[i].tag < sourceCounters[j].tag })
sort.Slice(sourceBodySnaps, func(i, j int) bool { return sourceBodySnaps[i].label < sourceBodySnaps[j].label })
sortBody := func(s []bodySnap) {
sort.Slice(s, func(i, j int) bool {
a, b := s[i].k, s[j].k
if a.Host != b.Host {
return a.Host < b.Host
}
return a.SourceTag < b.SourceTag
})
}
sortTime := func(s []timeSnap) {
sort.Slice(s, func(i, j int) bool {
a, b := s[i].k, s[j].k
if a.Host != b.Host {
return a.Host < b.Host
}
return a.SourceTag < b.SourceTag
})
}
sortBody(bytesSnaps)
sortBody(requestBytesSnaps)
sortTime(requestDurationSnaps)
sortTime(upstreamDurationSnaps)
sort.Slice(upstreamCounters, func(i, j int) bool {
a, b := upstreamCounters[i].k, upstreamCounters[j].k
if a.Host != b.Host {
return a.Host < b.Host
}
if a.SourceTag != b.SourceTag {
return a.SourceTag < b.SourceTag
}
return a.StatusClass < b.StatusClass
})
sort.Slice(sourceCounters, func(i, j int) bool {
a, b := sourceCounters[i].k, sourceCounters[j].k
if a.SourceTag != b.SourceTag {
return a.SourceTag < b.SourceTag
}
return a.StatusClass < b.StatusClass
})
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
bw := bufio.NewWriterSize(w, 256*1024)
// nginx_http_requests_total
fmt.Fprintln(bw, "# HELP nginx_http_requests_total Total number of HTTP requests processed.")
fmt.Fprintln(bw, "# TYPE nginx_http_requests_total counter")
// pf, pln are short helpers so the metric block reads cleanly. Errors on a
// bufio writer wrapping http.ResponseWriter mean the client disconnected;
// there's nothing useful to do mid-write — the next call will simply no-op.
pf := func(format string, a ...any) { _, _ = fmt.Fprintf(bw, format, a...) }
pln := func(s string) { _, _ = fmt.Fprintln(bw, s) }
pln("# HELP nginx_http_requests_total Total number of HTTP requests processed.")
pln("# TYPE nginx_http_requests_total counter")
for _, c := range counters {
fmt.Fprintf(bw, "nginx_http_requests_total{host=%q,method=%q,status=%q} %d\n",
pf("nginx_http_requests_total{host=%q,method=%q,status=%q} %d\n",
c.k.Host, c.k.Method, c.k.Status, c.v)
}
// nginx_http_response_body_bytes (histogram, labeled by host)
fmt.Fprintln(bw, "# HELP nginx_http_response_body_bytes HTTP response body size distribution in bytes.")
fmt.Fprintln(bw, "# TYPE nginx_http_response_body_bytes histogram")
for _, s := range bodySnaps {
writeBodyHistogram(bw, "nginx_http_response_body_bytes", "host", s.label, s.e)
pln("# HELP nginx_http_bytes_sent HTTP response size distribution in bytes (body for v1 records, full wire bytes for v2).")
pln("# TYPE nginx_http_bytes_sent histogram")
for _, s := range bytesSnaps {
writeBodyHistogramHS(bw, "nginx_http_bytes_sent", s.k, s.e)
}
// nginx_http_request_duration_seconds (histogram, labeled by host)
fmt.Fprintln(bw, "# HELP nginx_http_request_duration_seconds HTTP request processing time in seconds.")
fmt.Fprintln(bw, "# TYPE nginx_http_request_duration_seconds histogram")
for _, s := range timeSnaps {
for i, bound := range promTimeBounds {
fmt.Fprintf(bw, "nginx_http_request_duration_seconds_bucket{host=%q,le=%q} %d\n",
s.host, formatFloat(bound), s.e.buckets[i])
}
fmt.Fprintf(bw, "nginx_http_request_duration_seconds_bucket{host=%q,le=\"+Inf\"} %d\n",
s.host, s.e.buckets[promNumTimeBounds])
fmt.Fprintf(bw, "nginx_http_request_duration_seconds_count{host=%q} %d\n",
s.host, s.e.buckets[promNumTimeBounds])
fmt.Fprintf(bw, "nginx_http_request_duration_seconds_sum{host=%q} %g\n",
s.host, s.e.sum)
pln("# HELP nginx_http_request_bytes HTTP request size distribution in bytes (v2 emitters only).")
pln("# TYPE nginx_http_request_bytes histogram")
for _, s := range requestBytesSnaps {
writeBodyHistogramHS(bw, "nginx_http_request_bytes", s.k, s.e)
}
// nginx_http_requests_by_source_total (counter, labeled by source_tag)
fmt.Fprintln(bw, "# HELP nginx_http_requests_by_source_total HTTP requests rolled up by nginx source tag.")
fmt.Fprintln(bw, "# TYPE nginx_http_requests_by_source_total counter")
pln("# HELP nginx_http_request_duration_seconds HTTP request processing time in seconds.")
pln("# TYPE nginx_http_request_duration_seconds histogram")
for _, s := range requestDurationSnaps {
writeTimeHistogramHS(bw, "nginx_http_request_duration_seconds", s.k, s.e)
}
pln("# HELP nginx_http_upstream_duration_seconds Upstream response time in seconds (v2 emitters only).")
pln("# TYPE nginx_http_upstream_duration_seconds histogram")
for _, s := range upstreamDurationSnaps {
writeTimeHistogramHS(bw, "nginx_http_upstream_duration_seconds", s.k, s.e)
}
pln("# HELP nginx_http_upstream_requests_total Requests served via an upstream, by upstream-status class (v2 emitters only).")
pln("# TYPE nginx_http_upstream_requests_total counter")
for _, c := range upstreamCounters {
pf("nginx_http_upstream_requests_total{host=%q,source_tag=%q,status_class=%q} %d\n",
c.k.Host, c.k.SourceTag, c.k.StatusClass, c.v)
}
pln("# HELP nginx_http_requests_by_source_total HTTP requests rolled up by source_tag and status class.")
pln("# TYPE nginx_http_requests_by_source_total counter")
for _, c := range sourceCounters {
fmt.Fprintf(bw, "nginx_http_requests_by_source_total{source_tag=%q} %d\n", c.tag, c.v)
pf("nginx_http_requests_by_source_total{source_tag=%q,status_class=%q} %d\n",
c.k.SourceTag, c.k.StatusClass, c.v)
}
// nginx_http_response_body_bytes_by_source (histogram, labeled by source_tag)
fmt.Fprintln(bw, "# HELP nginx_http_response_body_bytes_by_source HTTP response body size distribution by nginx source tag.")
fmt.Fprintln(bw, "# TYPE nginx_http_response_body_bytes_by_source histogram")
for _, s := range sourceBodySnaps {
writeBodyHistogram(bw, "nginx_http_response_body_bytes_by_source", "source_tag", s.label, s.e)
}
pln("# HELP logtail_udp_packets_received_total Datagrams read from the UDP socket.")
pln("# TYPE logtail_udp_packets_received_total counter")
pf("logtail_udp_packets_received_total %d\n", udpPackets)
pln("# HELP logtail_udp_loglines_success_total UDP loglines that parsed successfully.")
pln("# TYPE logtail_udp_loglines_success_total counter")
pf("logtail_udp_loglines_success_total %d\n", udpSuccess)
pln("# HELP logtail_udp_loglines_consumed_total UDP loglines forwarded to the store (not dropped).")
pln("# TYPE logtail_udp_loglines_consumed_total counter")
pf("logtail_udp_loglines_consumed_total %d\n", udpConsumed)
// UDP ingest counters — lets operators distinguish parse failures
// (received - success) from channel-full drops (success - consumed).
fmt.Fprintln(bw, "# HELP logtail_udp_packets_received_total Datagrams read from the UDP socket.")
fmt.Fprintln(bw, "# TYPE logtail_udp_packets_received_total counter")
fmt.Fprintf(bw, "logtail_udp_packets_received_total %d\n", udpPackets)
fmt.Fprintln(bw, "# HELP logtail_udp_loglines_success_total UDP loglines that parsed successfully.")
fmt.Fprintln(bw, "# TYPE logtail_udp_loglines_success_total counter")
fmt.Fprintf(bw, "logtail_udp_loglines_success_total %d\n", udpSuccess)
fmt.Fprintln(bw, "# HELP logtail_udp_loglines_consumed_total UDP loglines forwarded to the store (not dropped).")
fmt.Fprintln(bw, "# TYPE logtail_udp_loglines_consumed_total counter")
fmt.Fprintf(bw, "logtail_udp_loglines_consumed_total %d\n", udpConsumed)
bw.Flush()
_ = bw.Flush()
}
func writeBodyHistogram(bw *bufio.Writer, metric, labelName, labelValue string, e promBodyEntry) {
func writeBodyHistogramHS(bw *bufio.Writer, metric string, k hostSourceKey, e promBodyEntry) {
pf := func(format string, a ...any) { _, _ = fmt.Fprintf(bw, format, a...) }
for i, bound := range promBodyBounds {
fmt.Fprintf(bw, "%s_bucket{%s=%q,le=%q} %d\n",
metric, labelName, labelValue, fmt.Sprintf("%d", bound), e.buckets[i])
pf("%s_bucket{host=%q,source_tag=%q,le=\"%d\"} %d\n",
metric, k.Host, k.SourceTag, bound, e.buckets[i])
}
fmt.Fprintf(bw, "%s_bucket{%s=%q,le=\"+Inf\"} %d\n",
metric, labelName, labelValue, e.buckets[promNumBodyBounds])
fmt.Fprintf(bw, "%s_count{%s=%q} %d\n",
metric, labelName, labelValue, e.buckets[promNumBodyBounds])
fmt.Fprintf(bw, "%s_sum{%s=%q} %d\n",
metric, labelName, labelValue, e.sum)
pf("%s_bucket{host=%q,source_tag=%q,le=\"+Inf\"} %d\n",
metric, k.Host, k.SourceTag, e.buckets[promNumBodyBounds])
pf("%s_count{host=%q,source_tag=%q} %d\n",
metric, k.Host, k.SourceTag, e.buckets[promNumBodyBounds])
pf("%s_sum{host=%q,source_tag=%q} %d\n",
metric, k.Host, k.SourceTag, e.sum)
}
func writeTimeHistogramHS(bw *bufio.Writer, metric string, k hostSourceKey, e promTimeEntry) {
pf := func(format string, a ...any) { _, _ = fmt.Fprintf(bw, format, a...) }
for i, bound := range promTimeBounds {
pf("%s_bucket{host=%q,source_tag=%q,le=%q} %d\n",
metric, k.Host, k.SourceTag, formatFloat(bound), e.buckets[i])
}
pf("%s_bucket{host=%q,source_tag=%q,le=\"+Inf\"} %d\n",
metric, k.Host, k.SourceTag, e.buckets[promNumTimeBounds])
pf("%s_count{host=%q,source_tag=%q} %d\n",
metric, k.Host, k.SourceTag, e.buckets[promNumTimeBounds])
pf("%s_sum{host=%q,source_tag=%q} %g\n",
metric, k.Host, k.SourceTag, e.sum)
}
// formatFloat renders a float64 bucket bound without trailing zeros but always
@@ -280,7 +397,7 @@ func writeBodyHistogram(bw *bufio.Writer, metric, labelName, labelValue string,
func formatFloat(f float64) string {
s := fmt.Sprintf("%g", f)
if !strings.Contains(s, ".") && !strings.Contains(s, "e") {
s += ".0" // ensure it looks like a float, not an integer
s += ".0"
}
return s
}
cmd/collector/prom_test.go
+122 -32
View File
@@ -6,22 +6,22 @@ import (
"testing"
)
func TestPromStoreIngestBodyBuckets(t *testing.T) {
func TestPromStoreIngestBytesBuckets(t *testing.T) {
ps := NewPromStore()
// 512 bytes: > 256, ≤ 1024 → bucket[0] stays 0, buckets[1..N] get 1
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", BodyBytesSent: 512})
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", BytesSent: 512, SourceTag: "direct"})
ps.mu.Lock()
be := ps.body["example.com"]
be := ps.bytesSent[hostSourceKey{"example.com", "direct"}]
ps.mu.Unlock()
if be == nil {
t.Fatal("expected body entry, got nil")
t.Fatal("expected bytes entry, got nil")
}
if be.buckets[0] != 0 { // le=256: 512 > 256
if be.buckets[0] != 0 {
t.Errorf("le=256 bucket = %d, want 0", be.buckets[0])
}
if be.buckets[1] != 1 { // le=1024: 512 ≤ 1024
if be.buckets[1] != 1 {
t.Errorf("le=1024 bucket = %d, want 1", be.buckets[1])
}
for i := 2; i <= promNumBodyBounds; i++ {
@@ -37,24 +37,21 @@ func TestPromStoreIngestBodyBuckets(t *testing.T) {
func TestPromStoreIngestTimeBuckets(t *testing.T) {
ps := NewPromStore()
// 0.075s: > 0.05, ≤ 0.1
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", RequestTime: 0.075})
ps.Ingest(LogRecord{Website: "example.com", Method: "GET", Status: "200", RequestTime: 0.075, SourceTag: "direct"})
ps.mu.Lock()
te := ps.reqTime["example.com"]
te := ps.requestDuration[hostSourceKey{"example.com", "direct"}]
ps.mu.Unlock()
if te == nil {
t.Fatal("expected time entry, got nil")
t.Fatal("expected request_duration entry, got nil")
}
// le=0.05 (index 3): 0.075 > 0.05 → 0
if te.buckets[3] != 0 {
t.Errorf("le=0.05 bucket = %d, want 0", te.buckets[3])
}
// le=0.1 (index 4): 0.075 ≤ 0.1 → 1
if te.buckets[4] != 1 {
t.Errorf("le=0.1 bucket = %d, want 1", te.buckets[4])
}
// +Inf (last): always 1
if te.buckets[promNumTimeBounds] != 1 {
t.Errorf("+Inf bucket = %d, want 1", te.buckets[promNumTimeBounds])
}
@@ -83,7 +80,7 @@ func TestPromStoreServeHTTP(t *testing.T) {
ps := NewPromStore()
ps.Ingest(LogRecord{
Website: "example.com", Method: "GET", Status: "200",
BodyBytesSent: 100, RequestTime: 0.042,
BytesSent: 100, RequestTime: 0.042, SourceTag: "direct",
})
req := httptest.NewRequest("GET", "/metrics", nil)
@@ -95,13 +92,15 @@ func TestPromStoreServeHTTP(t *testing.T) {
checks := []string{
"# TYPE nginx_http_requests_total counter",
`nginx_http_requests_total{host="example.com",method="GET",status="200"} 1`,
"# TYPE nginx_http_response_body_bytes histogram",
`nginx_http_response_body_bytes_bucket{host="example.com",le="256"} 1`, // 100 ≤ 256
`nginx_http_response_body_bytes_count{host="example.com"} 1`,
`nginx_http_response_body_bytes_sum{host="example.com"} 100`,
"# TYPE nginx_http_bytes_sent histogram",
`nginx_http_bytes_sent_bucket{host="example.com",source_tag="direct",le="256"} 1`,
`nginx_http_bytes_sent_count{host="example.com",source_tag="direct"} 1`,
`nginx_http_bytes_sent_sum{host="example.com",source_tag="direct"} 100`,
"# TYPE nginx_http_request_duration_seconds histogram",
`nginx_http_request_duration_seconds_bucket{host="example.com",le="0.05"} 1`, // 0.042 ≤ 0.05
`nginx_http_request_duration_seconds_count{host="example.com"} 1`,
`nginx_http_request_duration_seconds_bucket{host="example.com",source_tag="direct",le="0.05"} 1`,
`nginx_http_request_duration_seconds_count{host="example.com",source_tag="direct"} 1`,
"# TYPE nginx_http_requests_by_source_total counter",
`nginx_http_requests_by_source_total{source_tag="direct",status_class="2xx"} 1`,
}
for _, want := range checks {
if !strings.Contains(body, want) {
@@ -110,12 +109,12 @@ func TestPromStoreServeHTTP(t *testing.T) {
}
}
func TestPromStoreSourceTagRollup(t *testing.T) {
func TestPromStoreSourceTagAndStatusClass(t *testing.T) {
ps := NewPromStore()
// same host, two tags; each tag should appear with its own series.
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 100, SourceTag: "direct"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 300, SourceTag: "cdn"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BodyBytesSent: 100, SourceTag: "cdn"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "direct"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 300, SourceTag: "cdn"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "404", BytesSent: 100, SourceTag: "cdn"})
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "500", BytesSent: 50, SourceTag: "cdn"})
req := httptest.NewRequest("GET", "/metrics", nil)
rec := httptest.NewRecorder()
@@ -124,13 +123,17 @@ func TestPromStoreSourceTagRollup(t *testing.T) {
checks := []string{
"# TYPE nginx_http_requests_by_source_total counter",
`nginx_http_requests_by_source_total{source_tag="direct"} 1`,
`nginx_http_requests_by_source_total{source_tag="cdn"} 2`,
"# TYPE nginx_http_response_body_bytes_by_source histogram",
`nginx_http_response_body_bytes_by_source_sum{source_tag="direct"} 100`,
`nginx_http_response_body_bytes_by_source_sum{source_tag="cdn"} 400`,
// host-series totals are unchanged (one row, counting 3 requests).
`nginx_http_requests_total{host="h",method="GET",status="200"} 3`,
`nginx_http_requests_by_source_total{source_tag="direct",status_class="2xx"} 1`,
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="2xx"} 1`,
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="4xx"} 1`,
`nginx_http_requests_by_source_total{source_tag="cdn",status_class="5xx"} 1`,
// dual-labeled histogram subsumes the old by-source bytes metric
`nginx_http_bytes_sent_sum{host="h",source_tag="direct"} 100`,
`nginx_http_bytes_sent_sum{host="h",source_tag="cdn"} 450`,
// host counter unchanged
`nginx_http_requests_total{host="h",method="GET",status="200"} 2`,
`nginx_http_requests_total{host="h",method="GET",status="404"} 1`,
`nginx_http_requests_total{host="h",method="GET",status="500"} 1`,
}
for _, want := range checks {
if !strings.Contains(body, want) {
@@ -139,6 +142,79 @@ func TestPromStoreSourceTagRollup(t *testing.T) {
}
}
func TestPromStoreUpstreamMetrics(t *testing.T) {
ps := NewPromStore()
// One upstream-served 200, one upstream-served 502, one no-upstream 200.
ps.Ingest(LogRecord{
Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "cdn",
RequestLength: 500,
HasUpstream: true,
UpstreamResponseTime: 0.020,
UpstreamStatus: "200",
})
ps.Ingest(LogRecord{
Website: "h", Method: "GET", Status: "502", BytesSent: 100, SourceTag: "cdn",
RequestLength: 500,
HasUpstream: true,
UpstreamResponseTime: 0.150,
UpstreamStatus: "502",
})
ps.Ingest(LogRecord{
Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "direct",
RequestLength: 200,
})
req := httptest.NewRequest("GET", "/metrics", nil)
rec := httptest.NewRecorder()
ps.ServeHTTP(rec, req)
body := rec.Body.String()
checks := []string{
// upstream counter: only the two upstream-served requests
`nginx_http_upstream_requests_total{host="h",source_tag="cdn",status_class="2xx"} 1`,
`nginx_http_upstream_requests_total{host="h",source_tag="cdn",status_class="5xx"} 1`,
// upstream duration histogram only for upstream-served requests
`nginx_http_upstream_duration_seconds_count{host="h",source_tag="cdn"} 2`,
// request_bytes only observed when RequestLength > 0 — all three here
`nginx_http_request_bytes_count{host="h",source_tag="cdn"} 2`,
`nginx_http_request_bytes_count{host="h",source_tag="direct"} 1`,
}
for _, want := range checks {
if !strings.Contains(body, want) {
t.Errorf("missing %q in output:\n%s", want, body)
}
}
// no-upstream request must not bump upstream counters
if strings.Contains(body, `source_tag="direct",status_class=`) &&
strings.Contains(body, "nginx_http_upstream_requests_total") {
// Better check: ensure no direct-tagged upstream entry
for _, bad := range []string{
`nginx_http_upstream_requests_total{host="h",source_tag="direct"`,
`nginx_http_upstream_duration_seconds_bucket{host="h",source_tag="direct"`,
} {
if strings.Contains(body, bad) {
t.Errorf("unexpected %q in output:\n%s", bad, body)
}
}
}
}
func TestPromStoreRequestBytesSkippedWhenZero(t *testing.T) {
ps := NewPromStore()
// v1 record — RequestLength=0, so request_bytes histogram should be empty.
ps.Ingest(LogRecord{Website: "h", Method: "GET", Status: "200", BytesSent: 100, SourceTag: "cdn"})
req := httptest.NewRequest("GET", "/metrics", nil)
rec := httptest.NewRecorder()
ps.ServeHTTP(rec, req)
body := rec.Body.String()
if strings.Contains(body, "nginx_http_request_bytes_bucket{") {
t.Errorf("expected no request_bytes series for v1 record:\n%s", body)
}
}
func TestPromStoreUDPCounters(t *testing.T) {
ps := NewPromStore()
ps.IncUDPPacket()
@@ -167,7 +243,6 @@ func TestPromStoreUDPCounters(t *testing.T) {
func TestPromStoreCounterCap(t *testing.T) {
ps := NewPromStore()
// Fill to cap with distinct {host,method,status} combos
for i := 0; i < promCounterCap+10; i++ {
host := strings.Repeat("x", i%10+1) + ".com"
status := "200"
@@ -183,3 +258,18 @@ func TestPromStoreCounterCap(t *testing.T) {
t.Errorf("counter map size %d exceeds cap %d", n, promCounterCap)
}
}
func TestStatusClass(t *testing.T) {
cases := map[string]string{
"200": "2xx", "201": "2xx", "299": "2xx",
"301": "3xx",
"404": "4xx", "418": "4xx",
"500": "5xx", "504": "5xx",
"100": "other", "": "other", "abc": "other",
}
for in, want := range cases {
if got := statusClass(in); got != want {
t.Errorf("statusClass(%q) = %q, want %q", in, got, want)
}
}
}
cmd/collector/smoke_test.go
+13 -4
View File
@@ -32,9 +32,18 @@ func BenchmarkIngest(b *testing.B) {
}
}
// BenchmarkParseLine measures parser throughput.
// BenchmarkParseLine measures parser throughput on a v1 line.
func BenchmarkParseLine(b *testing.B) {
line := "www.example.com\t1.2.3.4\t1741954800.123\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043"
line := "v1\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1452\t0.043\t0\t12345\tcdn\t10.0.0.1\thttps"
b.ResetTimer()
for i := 0; i < b.N; i++ {
ParseLine(line, 24, 48)
}
}
// BenchmarkParseLineV2 measures parser throughput on a v2 line with upstream.
func BenchmarkParseLineV2(b *testing.B) {
line := "v2\twww.example.com\t1.2.3.4\tGET\t/api/v1/search?q=foo\t200\t1500\t421\t0.043\t0.012\t200\t0\t12345\tcdn\t10.0.0.1\thttps"
b.ResetTimer()
for i := 0; i < b.N; i++ {
ParseLine(line, 24, 48)
@@ -118,7 +127,7 @@ func TestGRPCEndToEnd(t *testing.T) {
}
grpcSrv := grpc.NewServer()
pb.RegisterLogtailServiceServer(grpcSrv, NewServer(store, "e2e-test"))
go grpcSrv.Serve(lis)
go func() { _ = grpcSrv.Serve(lis) }()
defer grpcSrv.Stop()
// Dial it
@@ -127,7 +136,7 @@ func TestGRPCEndToEnd(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
client := pb.NewLogtailServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
cmd/collector/tailer.go
+6 -6
View File
@@ -52,7 +52,7 @@ func (mt *MultiTailer) Run(ctx context.Context) {
if err != nil {
log.Fatalf("tailer: failed to create watcher: %v", err)
}
defer watcher.Close()
defer func() { _ = watcher.Close() }()
files := make(map[string]*fileState)
retrying := make(map[string]struct{}) // paths currently in a retryOpen goroutine
@@ -85,7 +85,7 @@ func (mt *MultiTailer) Run(ctx context.Context) {
select {
case <-ctx.Done():
for _, fs := range files {
fs.f.Close()
_ = fs.f.Close()
}
return
@@ -117,7 +117,7 @@ func (mt *MultiTailer) Run(ctx context.Context) {
if event.Has(fsnotify.Rename) || event.Has(fsnotify.Remove) {
// Drain remaining bytes in the old fd before it disappears.
mt.readLines(fs.reader)
fs.f.Close()
_ = fs.f.Close()
delete(files, event.Name)
_ = watcher.Remove(event.Name)
startRetry(event.Name)
@@ -167,7 +167,7 @@ func (mt *MultiTailer) rescan(
for path, fs := range files {
if _, matched := current[path]; !matched {
mt.readLines(fs.reader)
fs.f.Close()
_ = fs.f.Close()
_ = watcher.Remove(path)
delete(files, path)
log.Printf("tailer: retired %s (no longer matched by any pattern)", path)
@@ -182,11 +182,11 @@ func openAndSeekEOF(path string, watcher *fsnotify.Watcher) (*fileState, error)
return nil, err
}
if _, err := f.Seek(0, io.SeekEnd); err != nil {
f.Close()
_ = f.Close()
return nil, err
}
if err := watcher.Add(path); err != nil {
f.Close()
_ = f.Close()
return nil, err
}
return &fileState{f: f, reader: bufio.NewReader(f)}, nil
cmd/collector/tailer_test.go
+8 -7
View File
@@ -11,7 +11,8 @@ import (
func writeLine(t *testing.T, f *os.File, website string) {
t.Helper()
_, err := fmt.Fprintf(f, "%s\t1.2.3.4\t0\tGET\t/path\t200\t0\t0.001\n", website)
// v1 layout: 12 payload fields after the v1 prefix.
_, err := fmt.Fprintf(f, "v1\t%s\t1.2.3.4\tGET\t/path\t200\t0\t0.001\t0\t0\tdirect\t10.0.0.1\thttps\n", website)
if err != nil {
t.Fatalf("writeLine: %v", err)
}
@@ -25,7 +26,7 @@ func TestMultiTailerReadsLines(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer func() { _ = f.Close() }()
ch := make(chan LogRecord, 100)
mt := NewMultiTailer([]string{path}, time.Hour, 24, 48, ch)
@@ -62,7 +63,7 @@ func TestMultiTailerMultipleFiles(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer func() { _ = f.Close() }()
files[i] = f
}
@@ -108,7 +109,7 @@ func TestMultiTailerLogRotation(t *testing.T) {
// Simulate logrotate: rename the old file, create a new one
rotated := filepath.Join(dir, "access.log.1")
f.Close()
_ = f.Close()
if err := os.Rename(path, rotated); err != nil {
t.Fatal(err)
}
@@ -121,7 +122,7 @@ func TestMultiTailerLogRotation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer newF.Close()
defer func() { _ = newF.Close() }()
// Allow retry goroutine to pick it up
time.Sleep(300 * time.Millisecond)
@@ -137,7 +138,7 @@ func TestExpandGlobs(t *testing.T) {
dir := t.TempDir()
for _, name := range []string{"a.log", "b.log", "other.txt"} {
f, _ := os.Create(filepath.Join(dir, name))
f.Close()
_ = f.Close()
}
pattern := filepath.Join(dir, "*.log")
@@ -151,7 +152,7 @@ func TestExpandGlobsDeduplication(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "access.log")
f, _ := os.Create(p)
f.Close()
_ = f.Close()
// Same file listed twice via explicit path and glob
paths := expandGlobs([]string{p, filepath.Join(dir, "*.log")})
cmd/collector/udp.go
+4 -4
View File
@@ -16,7 +16,7 @@ const udpReadBufBytes = 4 << 20
const udpPacketBuf = 64 << 10
// UDPListener receives nginx_ipng_stats_logtail datagrams on a local socket,
// parses each packet as one log line, and forwards LogRecords to ch.
// parses each line through the versioned ParseLine, and forwards LogRecords to ch.
type UDPListener struct {
addr string
v4bits int
@@ -50,7 +50,7 @@ func (u *UDPListener) Run(ctx context.Context) {
if err != nil {
log.Fatalf("udp: listen %s: %v", u.addr, err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
if err := conn.SetReadBuffer(udpReadBufBytes); err != nil {
log.Printf("udp: SetReadBuffer(%d): %v", udpReadBufBytes, err)
}
@@ -58,7 +58,7 @@ func (u *UDPListener) Run(ctx context.Context) {
go func() {
<-ctx.Done()
conn.Close()
_ = conn.Close()
}()
buf := make([]byte, udpPacketBuf)
@@ -84,7 +84,7 @@ func (u *UDPListener) Run(ctx context.Context) {
if line == "" {
continue
}
rec, ok := ParseUDPLine(line, u.v4bits, u.v6bits)
rec, ok := ParseLine(line, u.v4bits, u.v6bits)
if !ok {
continue
}
cmd/collector/udp_test.go
+8 -8
View File
@@ -17,7 +17,7 @@ func TestUDPListenerRoundTrip(t *testing.T) {
t.Fatalf("listen probe: %v", err)
}
addr := pc.LocalAddr().String()
pc.Close() // release; listener will re-bind
_ = pc.Close() // release; listener will re-bind
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -31,15 +31,15 @@ func TestUDPListenerRoundTrip(t *testing.T) {
if err != nil {
t.Fatalf("dial: %v", err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
// The listener is started asynchronously; retry for up to 1s.
good := "v1\twww.example.com\t1.2.3.4\tGET\t/\t200\t42\t0.010\t0\t12345\tdirect\t10.0.0.1\thttps"
bad := "not enough\tfields"
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
conn.Write([]byte(good))
conn.Write([]byte(bad))
_, _ = conn.Write([]byte(good))
_, _ = conn.Write([]byte(bad))
select {
case rec := <-ch:
if rec.Website != "www.example.com" || rec.SourceTag != "direct" {
@@ -80,7 +80,7 @@ func TestUDPListenerBatchedDatagram(t *testing.T) {
t.Fatalf("listen probe: %v", err)
}
addr := pc.LocalAddr().(*net.UDPAddr)
pc.Close()
_ = pc.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -100,7 +100,7 @@ func TestUDPListenerBatchedDatagram(t *testing.T) {
if err != nil {
t.Fatalf("src listen: %v", err)
}
defer src.Close()
defer func() { _ = src.Close() }()
// Drive the listener with retries until all three records land.
got := make(map[string]bool)
@@ -148,7 +148,7 @@ func TestUDPListenerMultipleSources(t *testing.T) {
t.Fatalf("listen probe: %v", err)
}
addr := pc.LocalAddr().(*net.UDPAddr)
pc.Close()
_ = pc.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -166,7 +166,7 @@ func TestUDPListenerMultipleSources(t *testing.T) {
if err != nil {
t.Fatalf("%s listen: %v", tag, err)
}
defer src.Close()
defer func() { _ = src.Close() }()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if _, err := src.WriteTo(good, addr); err != nil {
cmd/frontend/filter.go
+1 -1
View File
@@ -51,7 +51,7 @@ func applyTerm(term string, fs *filterState) error {
// Find the first operator character: ~, !, >, <, =
opIdx := strings.IndexAny(term, "~!><=")
if opIdx <= 0 {
return fmt.Errorf("invalid term %q: expected field=value, field>=value, field~=regex, etc.", term)
return fmt.Errorf("invalid term %q: expected field=value, field>=value, field~=regex, etc", term)
}
field := strings.ToLower(strings.TrimSpace(term[:opIdx]))
cmd/frontend/frontend_test.go
+2 -2
View File
@@ -52,7 +52,7 @@ func startFake(t *testing.T, fs *fakeServer) string {
}
srv := grpc.NewServer()
pb.RegisterLogtailServiceServer(srv, fs)
go srv.Serve(lis)
go func() { _ = srv.Serve(lis) }()
t.Cleanup(srv.GracefulStop)
return lis.Addr().String()
}
@@ -541,7 +541,7 @@ func TestDialFake(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer conn.Close()
defer func() { _ = conn.Close() }()
_ = client
// If we got here without error, the fake server is reachable.
cmd/frontend/handler.go
+3 -3
View File
@@ -542,7 +542,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fmt.Sprintf("cannot connect to %s: %v", params.Target, err)))
return
}
defer conn.Close()
defer func() { _ = conn.Close() }()
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()
@@ -589,7 +589,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
resp, err := ltClient.ListTargets(ctx, &pb.ListTargetsRequest{})
if ltConn != nil {
ltConn.Close()
_ = ltConn.Close()
}
if err != nil {
ltCh <- nil
@@ -683,5 +683,5 @@ func writeRawJSON(w http.ResponseWriter, params QueryParams, resp *pb.TopNRespon
o.Entries[i] = entry{Label: e.Label, Count: e.Count}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(o)
_ = json.NewEncoder(w).Encode(o)
}
cmd/frontend/main.go
+1 -1
View File
@@ -59,7 +59,7 @@ func main() {
<-ctx.Done()
log.Printf("frontend: shutting down")
srv.Shutdown(context.Background())
_ = srv.Shutdown(context.Background())
}
func envOr(key, def string) string {