Refactor docs; Add 'ipng_source_tag', add udp listener for nginx-ipng-stats plugin

cmd/collector/parser.go

@@ -18,65 +18,104 @@ type LogRecord struct {
 	Method        string
 	BodyBytesSent int64
 	RequestTime   float64
+	SourceTag     string
 }
 
-// ParseLine parses a tab-separated logtail log line:
+// fileSourceTag is the SourceTag assigned to records read from on-disk log
+// files, which pre-date the tag concept. Mirrors nginx's fallback label.
+const fileSourceTag = "direct"
+
+// ParseLine parses a tab-separated logtail log line from a file:
 //
 //	$host \t $remote_addr \t $msec \t $request_method \t $request_uri \t $status \t $body_bytes_sent \t $request_time \t $is_tor \t $asn
 //
 // The is_tor (field 9) and asn (field 10) fields are optional for backward
 // compatibility with older log files that omit them; they default to false/0
-// when absent.
+// when absent. SourceTag is always set to "direct" (file origin has no tag).
 // Returns false for lines with fewer than 8 fields.
 func ParseLine(line string, v4bits, v6bits int) (LogRecord, bool) {
-	// SplitN caps allocations; we need up to 10 fields.
 	fields := strings.SplitN(line, "\t", 10)
 	if len(fields) < 8 {
 		return LogRecord{}, false
 	}
-
-	uri := fields[4]
-	if i := strings.IndexByte(uri, '?'); i >= 0 {
-		uri = uri[:i]
-	}
-
 	prefix, ok := truncateIP(fields[1], v4bits, v6bits)
 	if !ok {
 		return LogRecord{}, false
 	}
-
 	isTor := len(fields) >= 9 && fields[8] == "1"
-
 	var asn int32
 	if len(fields) == 10 {
 		if n, err := strconv.ParseInt(fields[9], 10, 32); err == nil {
 			asn = int32(n)
 		}
 	}
-
-	var bodyBytes int64
-	if n, err := strconv.ParseInt(fields[6], 10, 64); err == nil {
-		bodyBytes = n
-	}
-
-	var reqTime float64
-	if f, err := strconv.ParseFloat(fields[7], 64); err == nil {
-		reqTime = f
-	}
-
 	return LogRecord{
 		Website:       fields[0],
 		ClientPrefix:  prefix,
-		URI:           uri,
+		URI:           stripQuery(fields[4]),
 		Status:        fields[5],
 		IsTor:         isTor,
 		ASN:           asn,
 		Method:        fields[3],
-		BodyBytesSent: bodyBytes,
-		RequestTime:   reqTime,
+		BodyBytesSent: parseInt(fields[6]),
+		RequestTime:   parseFloat(fields[7]),
+		SourceTag:     fileSourceTag,
 	}, true
 }
 
+// ParseUDPLine parses a tab-separated logtail log line from the UDP listener:
+//
+//	$host \t $remote_addr \t $request_method \t $request_uri \t $status \t
+//	$body_bytes_sent \t $request_time \t $is_tor \t $asn \t
+//	$ipng_source_tag \t $server_addr \t $scheme
+//
+// All 12 fields are required. server_addr and scheme are consumed but not
+// propagated. Returns false for any malformed packet (wrong field count,
+// bad IP).
+func ParseUDPLine(line string, v4bits, v6bits int) (LogRecord, bool) {
+	fields := strings.Split(line, "\t")
+	if len(fields) != 12 {
+		return LogRecord{}, false
+	}
+	prefix, ok := truncateIP(fields[1], v4bits, v6bits)
+	if !ok {
+		return LogRecord{}, false
+	}
+	var asn int32
+	if n, err := strconv.ParseInt(fields[8], 10, 32); err == nil {
+		asn = int32(n)
+	}
+	return LogRecord{
+		Website:       fields[0],
+		ClientPrefix:  prefix,
+		URI:           stripQuery(fields[3]),
+		Status:        fields[4],
+		IsTor:         fields[7] == "1",
+		ASN:           asn,
+		Method:        fields[2],
+		BodyBytesSent: parseInt(fields[5]),
+		RequestTime:   parseFloat(fields[6]),
+		SourceTag:     fields[9],
+	}, true
+}
+
+func stripQuery(uri string) string {
+	if i := strings.IndexByte(uri, '?'); i >= 0 {
+		return uri[:i]
+	}
+	return uri
+}
+
+func parseInt(s string) int64 {
+	n, _ := strconv.ParseInt(s, 10, 64)
+	return n
+}
+
+func parseFloat(s string) float64 {
+	f, _ := strconv.ParseFloat(s, 64)
+	return f
+}
+
 // truncateIP masks addr to the given prefix length depending on IP version.
 // Returns the CIDR string (e.g. "1.2.3.0/24") and true on success.
 func truncateIP(addr string, v4bits, v6bits int) (string, bool) {