pim
6647f95be4
Wire-format and metric overhaul. Both file and UDP ingest now share one
versioned ParseLine that dispatches on the v<N>\t prefix; v1 stays
unchanged, v2 adds $bytes_sent (replacing $body_bytes_sent),
$request_length, $upstream_response_time, and $upstream_status. File
ingest gains the same versioning, and the legacy positional file format
is removed (no live deployments).
Prometheus exposition is rewritten:
- nginx_http_bytes_sent and nginx_http_request_duration_seconds gain
a source_tag label.
- nginx_http_requests_by_source_total gains status_class.
- New v2-only metrics: nginx_http_request_bytes,
nginx_http_upstream_duration_seconds,
nginx_http_upstream_requests_total{status_class}.
- Dropped nginx_http_response_body_bytes_by_source (subsumed by the
dual-labeled bytes_sent metric).
Adds 'make fixstyle' (gofmt -w) and clears all golangci-lint findings
across the repo (errcheck, S1001, ST1005, unused).
Docs in design.md FR-2/FR-8 and user-guide.md are rewritten to present
v2 as the recommended log format.
131 lines
2.9 KiB
Go
131 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
|
|
pb "git.ipng.ch/ipng/nginx-logtail/proto/logtailpb"
|
|
)
|
|
|
|
type topNResult struct {
|
|
target string
|
|
resp *pb.TopNResponse
|
|
err error
|
|
}
|
|
|
|
func runTopN(args []string) {
|
|
fs := flag.NewFlagSet("topn", flag.ExitOnError)
|
|
sf, targetFlag := bindShared(fs)
|
|
n := fs.Int("n", 10, "number of entries")
|
|
window := fs.String("window", "5m", "time window: 1m 5m 15m 60m 6h 24h")
|
|
groupBy := fs.String("group-by", "website", "group by: website prefix uri status")
|
|
_ = fs.Parse(args) // ExitOnError: only returns nil here
|
|
sf.resolve(*targetFlag)
|
|
|
|
win := parseWindow(*window)
|
|
grp := parseGroupBy(*groupBy)
|
|
filter := buildFilter(sf)
|
|
|
|
results := fanOutTopN(sf.targets, filter, grp, *n, win)
|
|
|
|
if sf.jsonOut {
|
|
printTopNJSONArray(results)
|
|
return
|
|
}
|
|
for _, r := range results {
|
|
if hdr := targetHeader(r.target, r.resp.GetSource(), len(sf.targets)); hdr != "" {
|
|
fmt.Println(hdr)
|
|
}
|
|
if r.err != nil {
|
|
fmt.Fprintf(os.Stderr, "error from %s: %v\n", r.target, r.err)
|
|
continue
|
|
}
|
|
printTopNTable(r)
|
|
if len(sf.targets) > 1 {
|
|
fmt.Println()
|
|
}
|
|
}
|
|
}
|
|
|
|
func fanOutTopN(targets []string, filter *pb.Filter, groupBy pb.GroupBy, n int, window pb.Window) []topNResult {
|
|
results := make([]topNResult, len(targets))
|
|
var wg sync.WaitGroup
|
|
for i, t := range targets {
|
|
wg.Add(1)
|
|
go func(i int, addr string) {
|
|
defer wg.Done()
|
|
results[i].target = addr
|
|
conn, client, err := dial(addr)
|
|
if err != nil {
|
|
results[i].err = err
|
|
results[i].resp = &pb.TopNResponse{}
|
|
return
|
|
}
|
|
defer func() { _ = conn.Close() }()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
resp, err := client.TopN(ctx, &pb.TopNRequest{
|
|
Filter: filter,
|
|
GroupBy: groupBy,
|
|
N: int32(n),
|
|
Window: window,
|
|
})
|
|
results[i].resp = resp
|
|
results[i].err = err
|
|
}(i, t)
|
|
}
|
|
wg.Wait()
|
|
return results
|
|
}
|
|
|
|
func printTopNTable(r topNResult) {
|
|
if len(r.resp.Entries) == 0 {
|
|
fmt.Println("(no data)")
|
|
return
|
|
}
|
|
rows := [][]string{{"RANK", "COUNT", "LABEL"}}
|
|
for i, e := range r.resp.Entries {
|
|
rows = append(rows, []string{
|
|
fmt.Sprintf("%4d", i+1),
|
|
fmtCount(e.Count),
|
|
e.Label,
|
|
})
|
|
}
|
|
printTable(os.Stdout, rows)
|
|
}
|
|
|
|
func printTopNJSONArray(results []topNResult) {
|
|
type entry struct {
|
|
Label string `json:"label"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
type out struct {
|
|
Source string `json:"source"`
|
|
Target string `json:"target"`
|
|
Entries []entry `json:"entries"`
|
|
}
|
|
rows := make([]out, 0, len(results))
|
|
for _, r := range results {
|
|
if r.err != nil {
|
|
fmt.Fprintf(os.Stderr, "error from %s: %v\n", r.target, r.err)
|
|
continue
|
|
}
|
|
o := out{
|
|
Source: r.resp.Source,
|
|
Target: r.target,
|
|
Entries: make([]entry, len(r.resp.Entries)),
|
|
}
|
|
for i, e := range r.resp.Entries {
|
|
o.Entries[i] = entry{Label: e.Label, Count: e.Count}
|
|
}
|
|
rows = append(rows, o)
|
|
}
|
|
b, _ := json.Marshal(rows)
|
|
fmt.Println(string(b))
|
|
}
|