pim
6647f95be4
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).
Prometheus exposition is rewritten:
- nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
a source_tag label.
- nginx_http_requests_by_source_total gains status_class.
- New v2-only metrics: nginx_http_request_bytes,
nginx_http_upstream_duration_seconds,
nginx_http_upstream_requests_total{status_class}.
- Dropped nginx_http_response_body_bytes_by_source (subsumed by the
dual-labeled bytes_sent metric).
Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).
Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
105 lines
3.1 KiB
Go
105 lines
3.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"net"
|
|
"strings"
|
|
)
|
|
|
|
// udpReadBufBytes is the SO_RCVBUF size requested. Bursts of ~10K lines/sec at
|
|
// ~200B each comfortably fit; the kernel may cap below this.
|
|
const udpReadBufBytes = 4 << 20
|
|
|
|
// udpPacketBuf is the per-read buffer. A single nginx log line easily fits in
|
|
// a few KB; 64K is the practical UDP datagram ceiling.
|
|
const udpPacketBuf = 64 << 10
|
|
|
|
// UDPListener receives nginx_ipng_stats_logtail datagrams on a local socket,
|
|
// parses each line through the versioned ParseLine, and forwards LogRecords to ch.
|
|
type UDPListener struct {
|
|
addr string
|
|
v4bits int
|
|
v6bits int
|
|
ch chan<- LogRecord
|
|
prom *PromStore // optional; bumps UDP ingest counters
|
|
}
|
|
|
|
func NewUDPListener(addr string, v4bits, v6bits int, ch chan<- LogRecord) *UDPListener {
|
|
return &UDPListener{addr: addr, v4bits: v4bits, v6bits: v6bits, ch: ch}
|
|
}
|
|
|
|
// SetProm wires a PromStore so the listener can report received/success/consumed counts.
|
|
func (u *UDPListener) SetProm(p *PromStore) { u.prom = p }
|
|
|
|
// Run listens until ctx is cancelled.
|
|
//
|
|
// The socket is unconnected (ListenUDP + ReadFromUDP), so every datagram is
|
|
// accepted regardless of its source address. This matters across nginx
|
|
// reloads: the old worker processes hold their own ephemeral send sockets,
|
|
// and the fresh worker set opens brand-new ones. The listener reads them
|
|
// all. (Contrast with `nc -k -u -l`, which latches onto the first peer's
|
|
// address and silently drops packets from anyone else — that is an `nc`
|
|
// quirk, not a kernel behaviour, and does not apply here.)
|
|
func (u *UDPListener) Run(ctx context.Context) {
|
|
laddr, err := net.ResolveUDPAddr("udp", u.addr)
|
|
if err != nil {
|
|
log.Fatalf("udp: resolve %s: %v", u.addr, err)
|
|
}
|
|
conn, err := net.ListenUDP("udp", laddr)
|
|
if err != nil {
|
|
log.Fatalf("udp: listen %s: %v", u.addr, err)
|
|
}
|
|
defer func() { _ = conn.Close() }()
|
|
if err := conn.SetReadBuffer(udpReadBufBytes); err != nil {
|
|
log.Printf("udp: SetReadBuffer(%d): %v", udpReadBufBytes, err)
|
|
}
|
|
log.Printf("udp: listening on %s", conn.LocalAddr())
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
_ = conn.Close()
|
|
}()
|
|
|
|
buf := make([]byte, udpPacketBuf)
|
|
for {
|
|
n, _, err := conn.ReadFromUDP(buf)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
log.Printf("udp: read: %v", err)
|
|
continue
|
|
}
|
|
if u.prom != nil {
|
|
u.prom.IncUDPPacket()
|
|
}
|
|
// nginx-ipng-stats-plugin batches log lines into a single UDP
|
|
// datagram (default buffer=64k flush=1s), so one packet may carry
|
|
// many lines. nginx's log_format always ends a rendered line with
|
|
// '\n'; split on that and process each line independently.
|
|
payload := strings.TrimRight(string(buf[:n]), "\r\n")
|
|
for _, line := range strings.Split(payload, "\n") {
|
|
line = strings.TrimSuffix(line, "\r")
|
|
if line == "" {
|
|
continue
|
|
}
|
|
rec, ok := ParseLine(line, u.v4bits, u.v6bits)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if u.prom != nil {
|
|
u.prom.IncUDPSuccess()
|
|
}
|
|
select {
|
|
case u.ch <- rec:
|
|
if u.prom != nil {
|
|
u.prom.IncUDPConsumed()
|
|
}
|
|
default:
|
|
// Channel full — drop rather than block the read loop.
|
|
}
|
|
}
|
|
}
|
|
}
|