diff --git a/docs/config-guide.md b/docs/config-guide.md index 7453224..6a465af 100644 --- a/docs/config-guide.md +++ b/docs/config-guide.md @@ -108,6 +108,21 @@ same set applies to every `(source, vip)` key in the module (v0.1 does not suppo See FR-2.3, FR-5.4. +### `ipng_stats_byte_buckets ...` + +**Context:** `http`. + +**Value:** two or more strictly increasing sizes (nginx size spec: `100`, `1k`, `1m`, ...) representing byte-size histogram upper +bounds. + +**Default:** `100 1000 10000 100000 1000000 10000000`, plus an implicit `+Inf` bucket. + +**Effect:** overrides the default bucket boundaries for the `nginx_ipng_bytes_in` and `nginx_ipng_bytes_out` histograms. Pick values +that match your traffic mix — these bucket bounds feed the scrape output only, not the per-`(source, vip, class)` byte counters, which +are exact. + +See FR-2.3. + ### `ipng_stats on | off` **Context:** `http`, `server`, `location`. @@ -231,17 +246,29 @@ See FR-3.1, FR-3.2, FR-3.3, FR-3.4, FR-3.5. For Prometheus, the module exports under the `nginx_ipng_` prefix. +The `code` label is a class bucket — one of `1xx`, `2xx`, `3xx`, `4xx`, `5xx`, or `unknown` (for codes outside `[100, 599]`). This +keeps per-`(source, vip)` counter cardinality bounded at six lanes regardless of how many distinct three-digit responses nginx serves. +Histogram series do not carry `code` — they aggregate across all classes for a given `(source, vip)`. Operators who need a full +per-three-digit-code breakdown should enable `ipng_stats_logtail` and derive it from the access-log stream off the hot path. + | metric | type | labels | meaning | | --- | --- | --- | --- | -| `nginx_ipng_requests_total` | counter | `source_tag`, `vip`, `code` | Request count per `(source, vip, status_code)`. | +| `nginx_ipng_requests_total` | counter | `source_tag`, `vip`, `code` | Request count per `(source, vip, class)`. | | `nginx_ipng_bytes_in_total` | counter | `source_tag`, `vip`, `code` | Request bytes received (request line + headers + body). | | `nginx_ipng_bytes_out_total` | counter | `source_tag`, `vip`, `code` | Response bytes sent (status line + headers + body). | -| `nginx_ipng_request_duration_seconds_bucket` | histogram bucket | `source_tag`, `vip`, `le` | Request duration histogram (Prometheus shape). | +| `nginx_ipng_latency_total` | counter | `source_tag`, `vip`, `code` | Sum of request durations, in seconds. Divide by `_requests_total` for mean latency per class. | +| `nginx_ipng_request_duration_seconds_bucket` | histogram bucket | `source_tag`, `vip`, `le` | Request duration histogram, aggregated across classes. | | `nginx_ipng_request_duration_seconds_sum` | histogram sum | `source_tag`, `vip` | Sum of observed durations in seconds. | | `nginx_ipng_request_duration_seconds_count` | histogram count | `source_tag`, `vip` | Count of observations. | | `nginx_ipng_upstream_response_seconds_bucket` | histogram bucket | `source_tag`, `vip`, `le` | Upstream response time histogram. | | `nginx_ipng_upstream_response_seconds_sum` | histogram sum | `source_tag`, `vip` | | | `nginx_ipng_upstream_response_seconds_count` | histogram count | `source_tag`, `vip` | | +| `nginx_ipng_bytes_in_bucket` | histogram bucket | `source_tag`, `vip`, `le` | Request-size histogram (bytes). | +| `nginx_ipng_bytes_in_sum` | histogram sum | `source_tag`, `vip` | Sum of request bytes (equals `bytes_in_total` summed over classes). | +| `nginx_ipng_bytes_in_count` | histogram count | `source_tag`, `vip` | Observations. | +| `nginx_ipng_bytes_out_bucket` | histogram bucket | `source_tag`, `vip`, `le` | Response-size histogram (bytes). | +| `nginx_ipng_bytes_out_sum` | histogram sum | `source_tag`, `vip` | Sum of response bytes. | +| `nginx_ipng_bytes_out_count` | histogram count | `source_tag`, `vip` | Observations. | | `nginx_ipng_rate_1s` | gauge | `source_tag`, `vip` | EWMA requests/sec, 1-second decay. | | `nginx_ipng_rate_10s` | gauge | `source_tag`, `vip` | EWMA requests/sec, 10-second decay. | | `nginx_ipng_rate_60s` | gauge | `source_tag`, `vip` | EWMA requests/sec, 60-second decay. | @@ -258,39 +285,31 @@ See FR-2.*, FR-3.7. ```json { - "schema": 1, - "by_source": { - "mg1": { - "vips": { - "192.0.2.10": { - "rate_1s": 42.3, - "rate_10s": 40.1, - "rate_60s": 39.8, - "codes": { - "200": { "requests": 12345, "bytes_in": 9876543, "bytes_out": 54321098 }, - "404": { "requests": 17, "bytes_in": 2048, "bytes_out": 9216 } - }, - "request_duration_ms": { - "buckets": { "1": 10, "5": 40, "10": 120, "25": 350, "50": 870, "100": 2100, - "250": 3400, "500": 4000, "1000": 4100, "2500": 4120, - "5000": 4123, "10000": 4124, "+Inf": 4124 }, - "sum_ms": 87654, - "count": 4124 - }, - "upstream_response_ms": { "...": "..." } - } - } + "schema": 2, + "records": [ + { + "source_tag": "mg1", + "vip": "192.0.2.10", + "classes": { + "2xx": { "requests": 12345, "bytes_in": 9876543, "bytes_out": 54321098, + "latency_ms": 87654, "upstream_latency_ms": 61234 }, + "4xx": { "requests": 17, "bytes_in": 2048, "bytes_out": 9216, + "latency_ms": 102, "upstream_latency_ms": 0 } + }, + "request_duration_ms": { + "sum": 87756, "count": 12362, + "buckets": { "1": 10, "5": 40, "10": 120, "+Inf": 12362 } + }, + "upstream_response_ms": { "sum": 61234, "count": 12345, "buckets": { "...": "..." } }, + "bytes_in": { "count": 12362, "buckets": { "100": 200, "1000": 9000, "+Inf": 12362 } }, + "bytes_out": { "count": 12362, "buckets": { "...": "..." } } } - }, - "meta": { - "zone_bytes_used": 131072, - "zone_bytes_total": 4194304, - "zone_full_events": 0 - } + ] } ``` -The top-level `schema` field is versioned — breaking changes bump it, additive changes don't. Consumers SHOULD check `schema` +The top-level `schema` field is versioned — breaking changes bump it, additive changes don't. Schema `2` collapses status codes to +class buckets and moves histograms out of the per-class records to a per-`(source, vip)` record. Consumers SHOULD check `schema` before parsing. See FR-3.6. @@ -303,6 +322,7 @@ See FR-3.6. | `ipng_stats_flush_interval` | ✅ | — | — | — | | `ipng_stats_default_source` | ✅ | — | — | — | | `ipng_stats_buckets` | ✅ | — | — | — | +| `ipng_stats_byte_buckets` | ✅ | — | — | — | | `ipng_stats_logtail` | ✅ | — | — | — | | `ipng_stats on\|off` | ✅ | ✅ | ✅ | — | | `ipng_stats;` (handler) | — | — | ✅ | — | diff --git a/docs/design.md b/docs/design.md index 13943bf..bd0a275 100644 --- a/docs/design.md +++ b/docs/design.md @@ -90,14 +90,18 @@ Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) so that lat **FR-2 Counters** -- **FR-2.1** The module MUST maintain, for every observed `(source, vip, status_code)` tuple, the following counters: total requests, +- **FR-2.1** The module MUST maintain, for every observed `(source, vip, status_class)` tuple, the following counters: total requests, total bytes received (sum of request bytes including request line, headers, and body), total bytes sent (sum of response bytes - including status line, headers, and body), and a fixed-bucket histogram of request duration in milliseconds. + including status line, headers, and body), and sum of request durations in milliseconds (exported as `nginx_ipng_latency_total`). + The module MUST additionally maintain, per `(source, vip)` pair (no `code` label), fixed-bucket histograms of request duration in + milliseconds and of request/response sizes in bytes. - **FR-2.2** When an upstream is used to serve the request, the module MUST additionally maintain a fixed-bucket histogram of upstream response time in milliseconds, keyed by the same `(source, vip)` pair. -- **FR-2.3** The histogram bucket boundaries MUST be fixed at module initialization and MUST be the same for every `(source, vip)` key. - The default boundaries are `{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}` milliseconds plus an implicit `+Inf` bucket. - Operators MAY override the boundaries via the `ipng_stats_buckets` directive at the `http` level. +- **FR-2.3** The duration histogram bucket boundaries MUST be fixed at module initialization and MUST be the same for every `(source, + vip)` key. The default boundaries are `{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}` milliseconds plus an implicit + `+Inf` bucket. Operators MAY override the boundaries via the `ipng_stats_buckets` directive at the `http` level. The byte-size + histograms (request and response bodies) use independent bounds defaulting to `{100, 1000, 10000, 100000, 1000000, 10000000}` bytes; + `ipng_stats_byte_buckets` overrides them. - **FR-2.4** The module MUST additionally maintain, per `(source, vip)` pair, exponentially-weighted moving averages for instantaneous request rate with decay windows of 1 second, 10 seconds, and 60 seconds. EWMAs are updated from the periodic flush tick (see FR-4.2), not from the request path. @@ -105,8 +109,11 @@ Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) so that lat IPv4, RFC 5952 lowercase-compressed form for IPv6). IPv6 zone identifiers (scope-ids), if any, MUST be stripped during canonicalization; link-local VIPs (which are not expected in practice) are attributed under their scope-less textual form. Port is not part of the key; a VIP that listens on both 80 and 443 MUST be aggregated. -- **FR-2.6** The `status_code` dimension MUST be the full three-digit HTTP status code as recorded by nginx at log phase. The module MUST - NOT bucket codes into classes (2xx/3xx/4xx/5xx); bucketing is the consumer's job. +- **FR-2.6** The `status_code` dimension MUST be bucketed into a single class label: `1xx`, `2xx`, `3xx`, `4xx`, `5xx`, or `unknown` for + codes outside `[100, 599]`. This bounds per-`(source, vip)` cardinality to six lanes regardless of how many distinct three-digit + codes are observed. Operators who need a full per-code breakdown SHOULD enable `ipng_stats_logtail` (FR-8) and derive the per-code + view from the access-log stream off the hot path; the stats zone intentionally trades that resolution for a much smaller scrape + response. **FR-3 Scrape endpoint** @@ -122,8 +129,10 @@ Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) so that lat - **FR-3.5** Both filters MAY be supplied together; their effect is the intersection. - **FR-3.6** The JSON schema MUST be documented in `docs/scrape-api.md` and MUST version via a top-level `schema` field so that breaking changes can be made additively without bricking existing consumers. -- **FR-3.7** The Prometheus text output MUST use stable metric names prefixed with `nginx_ipng_` and MUST label every series with `source_tag` - and `vip`. Counter metrics additionally carry a `code` label. +- **FR-3.7** The Prometheus text output MUST use stable metric names prefixed with `nginx_ipng_` and MUST label every series with + `source_tag` and `vip`. Counter metrics (`nginx_ipng_requests_total`, `nginx_ipng_bytes_{in,out}_total`, `nginx_ipng_latency_total`) + additionally carry a `code` label with a class value (`1xx`..`5xx`/`unknown`). Histogram series (duration, upstream response, + request/response byte size) MUST NOT carry a `code` label — they aggregate across all classes for a given `(source, vip)` pair. **FR-4 Hot path and flush** @@ -220,7 +229,7 @@ Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) so that lat (cached on the connection struct), a constant-time status-code index computation, a constant number of integer increments, and a `O(log B)` histogram binary search where `B` is the number of buckets. No syscalls, no allocations, no locks. - **NFR-2.2** The per-flush cost per worker MUST be bounded by `O(K)` atomic adds, where `K` is the number of distinct - `(source, vip, code)` keys touched by that worker since the last flush. Keys untouched during an interval MUST NOT be visited. + `(source, vip, class)` keys touched by that worker since the last flush. Keys untouched during an interval MUST NOT be visited. - **NFR-2.3** The scrape cost MUST be bounded by `O(K_total)` reads from the shared zone plus `O(K_total)` string format operations, where `K_total` is the number of distinct keys in the zone. @@ -231,9 +240,9 @@ Each requirement carries a unique identifier (`FR-X.Y` or `NFR-X.Y`) so that lat level no more than once per minute per worker. - **NFR-3.2** The per-worker private counter table MUST be bounded by the same total key count the shared zone admits. A worker MUST NOT accumulate private state that exceeds the shared-zone capacity. -- **NFR-3.3** The set of distinct status codes observed is small (typically ≤ 60) and MUST NOT be allowed to explode due to non-standard - responses; the module MUST clamp any observed code `< 100` or `>= 600` into a single bucket labeled `code="unknown"` rather than - allocating a new key. +- **NFR-3.3** Status codes are collapsed to six classes (`1xx`..`5xx`/`unknown`) at counter-update time (FR-2.6), bounding per-`(source, + vip)` counter cardinality at six lanes regardless of how many three-digit codes are observed. Any code outside `[100, 599]` falls + into `code="unknown"`. Per-code resolution is available via `ipng_stats_logtail` (FR-8), which operates off the hot path. **NFR-4 Reload neutrality** @@ -332,7 +341,7 @@ dynamic-module ABI. - Parse new `listen` parameters `device=` and `ipng_source_tag=` and attach their values to each listening socket's config (FR-1.1, FR-1.2). - Call `setsockopt(SO_BINDTODEVICE)` in the master process at bind time for listeners that set `device=` (FR-1.1, NFR-6.1). -- Maintain per-worker private counter tables keyed by `(source_id, vip_id, status_code)` (FR-2.1, NFR-1.1). +- Maintain per-worker private counter tables keyed by `(source_id, vip_id, status_class)` (FR-2.1, NFR-1.1). - Run a per-worker flush timer that moves deltas into the shared-memory zone atomically (FR-4.2, NFR-1.2). - Update EWMAs at flush time (FR-2.4). - Serve the scrape endpoint with content negotiation and optional filters (FR-3). @@ -359,17 +368,22 @@ such an interface is silently misattributed to that interface's source tag. This #### Counter Data Model -Counters are stored as a flat hash table in a shared-memory zone. The key is the tuple `(source_id, vip_id, status_code)` where -`source_id` and `vip_id` are small integers assigned at first observation and reused thereafter. The value is a fixed-size record -containing: +Counters are stored as a flat hash table in a shared-memory zone. The key is the tuple `(source_id, vip_id, status_class)` where +`source_id` and `vip_id` are small integers assigned at first observation and `status_class` is one of six values (`0=unknown`, +`1..5` for `1xx`..`5xx`). The value is a fixed-size record containing: - `requests` (u64) - `bytes_in` (u64) - `bytes_out` (u64) -- `duration_hist` — `B+1` u64 lanes (one per bucket plus the `+Inf` bucket) -- `duration_sum_ms` (u64) -- `upstream_hist` — same shape, only updated when an upstream served the request +- `duration_sum_ms` (u64) — exported as `nginx_ipng_latency_total` (per class) - `upstream_sum_ms` (u64) +- `duration_hist` — `B+1` u64 lanes (one per bucket plus the `+Inf` bucket) +- `upstream_hist` — same shape, only updated when an upstream served the request +- `bytes_in_hist`, `bytes_out_hist` — `Bb+1` u64 lanes over the byte-size bucket bounds + +Histogram lanes are kept per `(source, vip, class)` in storage, then summed across classes at scrape time to produce one +`_bucket`/`_sum`/`_count` series per `(source, vip)` — the Prometheus exposition never carries a `code` label on histogram series +(FR-3.7). A parallel table keyed by `(source_id, vip_id)` — one row per VIP — holds the EWMAs for instantaneous rate. EWMAs are floats but updated only from the flush tick, so there is no float contention on the request path. @@ -379,8 +393,9 @@ endpoint can recover the original strings without re-parsing configuration. String interning is capacity-bounded: the zone is sized by the operator, and once capacity is exhausted new keys are dropped with a counter bump and an infrequent log line (NFR-3.1). In practice, the number of distinct VIPs on a single nginx host is small (tens, maybe -low hundreds), and the number of distinct source tags is the number of attributed interfaces (single digits). The dominant factor is -`status_code`; ~60 keys per VIP is a typical steady state. +low hundreds), and the number of distinct source tags is the number of attributed interfaces (single digits). Because status codes are +collapsed to six classes (FR-2.6), the `status_class` dimension contributes at most 6× the `(source, vip)` count — a ~10× reduction +from the per-three-digit-code model considered and discarded. #### Hot Path @@ -393,7 +408,7 @@ ipng_stats_log_handler(ngx_http_request_t *r) ipng_listen_ctx_t *lctx; ipng_counter_t *counter; ngx_msec_int_t elapsed_ms; - ngx_uint_t code_idx; + ngx_uint_t class_idx; if (!ipng_stats_enabled(r)) { return NGX_OK; @@ -403,8 +418,8 @@ ipng_stats_log_handler(ngx_http_request_t *r) /* lctx contains source_id and the cached VIP id, or resolves VIP lazily on first seen address */ - code_idx = ipng_status_to_index(r->headers_out.status); - counter = ipng_worker_slot(lctx, r->connection->local_sockaddr, code_idx); + class_idx = ipng_status_to_class(r->headers_out.status); /* 0..5 */ + counter = ipng_worker_slot(lctx, r->connection->local_sockaddr, class_idx); counter->requests++; counter->bytes_in += r->request_length; @@ -425,7 +440,7 @@ ipng_stats_log_handler(ngx_http_request_t *r) ``` Nothing here touches shared memory. `ipng_worker_slot` resolves a private table slot using a small per-worker hash keyed by -`(source_id, vip_id, code_idx)`. VIP lookup is cached on the connection so that keep-alive requests reuse the resolved ID. +`(source_id, vip_id, class_idx)`. VIP lookup is cached on the connection so that keep-alive requests reuse the resolved ID. #### Flush Timer @@ -459,9 +474,10 @@ fixed-size buffer per chain link and requests new links only when full. - **One nginx content handler**, `ipng_stats`, usable in any `location` block. Serves Prometheus text and JSON, filtered by optional query parameters. - **Two new `listen` parameters**, `device=` and `ipng_source_tag=`, usable anywhere a `listen` directive is used. -- **Five new `http`-level directives**: `ipng_stats_zone`, `ipng_stats_flush_interval`, `ipng_stats_default_source`, - `ipng_stats_buckets`, `ipng_stats` (on/off). -- **A Prometheus metric family** prefixed `nginx_ipng_*`, labelled `source_tag`, `vip`, and (for request counters) `code`. +- **Six new `http`-level directives**: `ipng_stats_zone`, `ipng_stats_flush_interval`, `ipng_stats_default_source`, + `ipng_stats_buckets`, `ipng_stats_byte_buckets`, `ipng_stats` (on/off). +- **A Prometheus metric family** prefixed `nginx_ipng_*`, labelled `source_tag`, `vip`, and (for counter metrics) a `code` class label + (`1xx`..`5xx`/`unknown`). **Consumes.** diff --git a/docs/user-guide.md b/docs/user-guide.md index 54a4128..460f43a 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -183,16 +183,20 @@ curl -s http://127.0.0.1:9113/.well-known/ipng/statsz Default output is Prometheus text format: ``` -# HELP nginx_ipng_requests_total Total HTTP requests, per (source_tag, vip, code). +# HELP nginx_ipng_requests_total Total HTTP requests. # TYPE nginx_ipng_requests_total counter -nginx_ipng_requests_total{source_tag="mg1",vip="192.0.2.10",code="200"} 12345 -nginx_ipng_requests_total{source_tag="mg1",vip="192.0.2.10",code="404"} 17 -nginx_ipng_requests_total{source_tag="mg2",vip="192.0.2.10",code="200"} 9876 -nginx_ipng_requests_total{source_tag="direct",vip="192.0.2.10",code="200"} 42 -# HELP nginx_ipng_bytes_in_total Request bytes received, per (source_tag, vip, code). +nginx_ipng_requests_total{source_tag="mg1",vip="192.0.2.10",code="2xx"} 12345 +nginx_ipng_requests_total{source_tag="mg1",vip="192.0.2.10",code="4xx"} 17 +nginx_ipng_requests_total{source_tag="mg2",vip="192.0.2.10",code="2xx"} 9876 +nginx_ipng_requests_total{source_tag="direct",vip="192.0.2.10",code="2xx"} 42 +# HELP nginx_ipng_bytes_in_total Request bytes received. # TYPE nginx_ipng_bytes_in_total counter -nginx_ipng_bytes_in_total{source_tag="mg1",vip="192.0.2.10",code="200"} 9876543 +nginx_ipng_bytes_in_total{source_tag="mg1",vip="192.0.2.10",code="2xx"} 9876543 # ... and so on + +# Histogram series (request_duration, upstream_response, bytes_in, bytes_out) +# do NOT carry a `code` label — they aggregate across classes per (source, vip). +nginx_ipng_request_duration_seconds_bucket{source_tag="mg1",vip="192.0.2.10",le="0.050"} 11200 ``` For JSON output instead, set the `Accept` header: @@ -237,7 +241,7 @@ Typical PromQL queries: sum by (source_tag, vip) (rate(nginx_ipng_requests_total[1m])) # 5xx error rate per VIP, aggregated across all sources: -sum by (vip) (rate(nginx_ipng_requests_total{code=~"5.."}[5m])) +sum by (vip) (rate(nginx_ipng_requests_total{code="5xx"}[5m])) / sum by (vip) (rate(nginx_ipng_requests_total[5m])) @@ -252,6 +256,11 @@ Operators who want a single unified access log covering all traffic — regardle have to repeat `access_log` in every `server {}` block or rely on a catch-all virtual host. The `ipng_stats_logtail` directive removes that requirement: one line at the `http` level registers a global log-phase writer that fires unconditionally for every request (FR-8.1). +The logtail is also the recommended escape hatch when you need richer cardinality than the stats zone exposes. The Prometheus counters +deliberately collapse HTTP status codes into six class lanes (`1xx`..`5xx`/`unknown`) to keep scrape size bounded. Operators who need +per-three-digit-code, per-path, per-user-agent, or any other high-cardinality breakdown should ship the logtail stream to an off-path +analytics receiver and compute those views there — that work happens in a different process and never touches the nginx hot path. + The logtail sends each buffer flush as a single UDP datagram to a `host:port`. Zero disk I/O, no backpressure, no blocking if the receiver is down. This makes it ideal for fire-and-forget analytics pipelines where delivery guarantees are unnecessary and disk writes would add unwanted I/O pressure. For file-based access logging, use nginx's built-in `access_log` directive. @@ -374,9 +383,10 @@ from any language. Once wired, a consumer can derive from the scrape data: - Live QPS per backend (from the EWMA gauges). -- Status-code mix per backend (from the counter families). -- p50/p95 latency per backend (from the duration histogram). -- Traffic volume per backend (from the bytes counters). +- Status-class mix per backend (the six-lane `1xx`..`5xx`/`unknown` counter families). Full three-digit codes are not exported by the + scrape endpoint; route the logtail stream off-host and aggregate there if you need per-code breakdowns. +- p50/p95 latency per backend (from the duration histogram, aggregated across classes). +- Traffic volume per backend (from the bytes counters and the new bytes histograms). For an example of this pattern in a GRE tunnel fleet, see [`vpp-maglev`](https://git.ipng.ch/ipng/vpp-maglev), whose frontend scrapes each nginx backend filtered by source tag to show per-backend traffic alongside health state. @@ -393,7 +403,8 @@ values in `listens.conf`, or the interfaces aren't up. Run `ip -br link` and con `nginx.conf` is stable across reloads — renaming the zone forces a new shared-memory segment. **`nginx_ipng_zone_full_events_total` is non-zero.** The shared-memory zone is too small for your VIP count. Increase the size in -`ipng_stats_zone ipng:` (default 4 MB is enough for ~hundreds of VIPs with the full status-code set). +`ipng_stats_zone ipng:` (default 4 MB is enough for ~hundreds of VIPs — the code dimension is bucketed to six classes, so +one 4 MB zone holds a very large deployment). **`curl http://127.0.0.1:9113/.well-known/ipng/statsz` returns "403 Forbidden".** The `allow`/`deny` ACL is blocking your source address. Either add yourself or scrape from a host already in the allow list. diff --git a/src/ngx_http_ipng_stats_module.c b/src/ngx_http_ipng_stats_module.c index 14893e3..7db82c4 100644 --- a/src/ngx_http_ipng_stats_module.c +++ b/src/ngx_http_ipng_stats_module.c @@ -67,25 +67,33 @@ extern ngx_module_t ngx_http_log_module; #define NGX_HTTP_IPNG_STATS_VERSION "0.1.0" -#define NGX_HTTP_IPNG_STATS_SCHEMA_VERSION 1 +#define NGX_HTTP_IPNG_STATS_SCHEMA_VERSION 2 /* Default histogram buckets in milliseconds (FR-2.3). */ #define NGX_HTTP_IPNG_STATS_DEFAULT_BUCKETS \ { 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000 } #define NGX_HTTP_IPNG_STATS_DEFAULT_BUCKET_COUNT 12 -/* Number of status code slots. We keep one lane per HTTP status code in - * [100, 599]; codes outside this range are clamped into a single - * "unknown" bucket (NFR-3.3). */ -#define NGX_HTTP_IPNG_STATS_CODE_MIN 100 -#define NGX_HTTP_IPNG_STATS_CODE_MAX 599 -#define NGX_HTTP_IPNG_STATS_CODE_UNKNOWN 0 +/* Default histogram buckets for request/response sizes, in bytes. */ +#define NGX_HTTP_IPNG_STATS_DEFAULT_BYTE_BUCKETS \ + { 100, 1000, 10000, 100000, 1000000, 10000000 } +#define NGX_HTTP_IPNG_STATS_DEFAULT_BYTE_BUCKET_COUNT 6 + +/* Status-code cardinality reduction: we bin every response into one of six + * classes — 1xx/2xx/3xx/4xx/5xx, plus a catch-all for codes outside + * [100, 599] (NFR-3.3). Operators who need full three-digit breakdown + * should use ipng_stats_logtail (FR-8); the stats zone is deliberately + * narrow. */ +#define NGX_HTTP_IPNG_STATS_CODE_MIN 100 +#define NGX_HTTP_IPNG_STATS_CODE_MAX 599 +#define NGX_HTTP_IPNG_STATS_CLASS_UNKNOWN 0 +#define NGX_HTTP_IPNG_STATS_NCLASSES 6 /* A single per-worker slot key. */ typedef struct { ngx_str_t source; /* points into shared-zone interning table */ ngx_str_t vip; /* ditto */ - ngx_uint_t code; /* 0 for unknown, else full 3-digit code */ + ngx_uint_t class; /* 0 unknown, 1..5 for 1xx..5xx */ } ngx_http_ipng_stats_key_t; @@ -101,15 +109,17 @@ typedef struct { ngx_queue_t lru; /* in-zone LRU for eviction on rename (NFR-4.4) */ ngx_uint_t source_id; ngx_uint_t vip_id; - ngx_uint_t code; + ngx_uint_t class; /* 0 unknown, 1..5 for 1xx..5xx */ ngx_atomic_uint_t requests; ngx_atomic_uint_t bytes_in; ngx_atomic_uint_t bytes_out; ngx_atomic_uint_t duration_sum_ms; ngx_atomic_uint_t upstream_sum_ms; - /* Followed by 2 * (nbuckets + 1) ngx_atomic_uint_t histogram lanes: - * lanes[0 .. nbuckets] -> request duration - * lanes[nbuckets+1 .. 2*nbuckets+1] -> upstream duration + /* Followed by histogram lanes, in order: + * [0 .. nb] -> request duration (nb = nbuckets) + * [nb+1 .. 2nb+1] -> upstream duration + * [2nb+2 .. 2nb+2+nbb] -> bytes_in (nbb = nbytebuckets) + * [2nb+3+nbb .. 2nb+3+2nbb] -> bytes_out */ } ngx_http_ipng_stats_node_t; @@ -122,7 +132,7 @@ typedef struct ngx_http_ipng_stats_slot_s { ngx_uint_t hash; ngx_uint_t source_id; ngx_uint_t vip_id; - ngx_uint_t code; + ngx_uint_t class; /* Deltas since last flush. */ uint64_t requests; @@ -130,8 +140,10 @@ typedef struct ngx_http_ipng_stats_slot_s { uint64_t bytes_out; uint64_t duration_sum_ms; uint64_t upstream_sum_ms; - uint64_t *dhist; /* nbuckets+1 lanes */ + uint64_t *dhist; /* nbuckets+1 lanes */ uint64_t *uhist; + uint64_t *bin_hist; /* nbytebuckets+1 lanes */ + uint64_t *bout_hist; /* Intrusive "dirty" linked list — dirty_next == NULL and * !is_dirty_head means "not on the list". */ @@ -186,6 +198,8 @@ typedef struct { ngx_str_t default_source; ngx_uint_t nbuckets; ngx_uint_t *bucket_bounds_ms; /* len = nbuckets */ + ngx_uint_t nbytebuckets; + ngx_uint_t *byte_bucket_bounds; /* len = nbytebuckets, bytes */ ngx_array_t *bindings; /* ngx_http_ipng_stats_binding_t */ ngx_flag_t enabled; @@ -214,6 +228,8 @@ typedef struct { ngx_log_t *log; uint64_t *dhist_arena; /* nslots * (nbuckets+1) u64 */ uint64_t *uhist_arena; + uint64_t *bin_arena; /* nslots * (nbytebuckets+1) u64 */ + uint64_t *bout_arena; /* Global logtail buffer. */ u_char *logtail_buf; @@ -239,6 +255,8 @@ static char *ngx_http_ipng_stats_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_ipng_stats_buckets(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_http_ipng_stats_byte_buckets(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); static char *ngx_http_ipng_stats_scrape(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_ipng_stats_logtail(ngx_conf_t *cf, ngx_command_t *cmd, @@ -333,6 +351,13 @@ static ngx_command_t ngx_http_ipng_stats_commands[] = { 0, NULL }, + { ngx_string("ipng_stats_byte_buckets"), + NGX_HTTP_MAIN_CONF|NGX_CONF_1MORE, + ngx_http_ipng_stats_byte_buckets, + NGX_HTTP_MAIN_CONF_OFFSET, + 0, + NULL }, + { ngx_string("ipng_stats"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF |NGX_CONF_NOARGS|NGX_CONF_TAKE1, @@ -576,6 +601,8 @@ ngx_http_ipng_stats_create_main_conf(ngx_conf_t *cf) imcf->flush_interval = NGX_CONF_UNSET_MSEC; imcf->nbuckets = 0; imcf->bucket_bounds_ms = NULL; + imcf->nbytebuckets = 0; + imcf->byte_bucket_bounds = NULL; imcf->enabled = NGX_CONF_UNSET; return imcf; @@ -612,6 +639,20 @@ ngx_http_ipng_stats_init_main_conf(ngx_conf_t *cf, void *conf) } } + if (imcf->nbytebuckets == 0) { + static const ngx_uint_t default_byte_bounds[] = + NGX_HTTP_IPNG_STATS_DEFAULT_BYTE_BUCKETS; + imcf->nbytebuckets = NGX_HTTP_IPNG_STATS_DEFAULT_BYTE_BUCKET_COUNT; + imcf->byte_bucket_bounds = ngx_palloc(cf->pool, + imcf->nbytebuckets * sizeof(ngx_uint_t)); + if (imcf->byte_bucket_bounds == NULL) { + return NGX_CONF_ERROR; + } + for (i = 0; i < imcf->nbytebuckets; i++) { + imcf->byte_bucket_bounds[i] = default_byte_bounds[i]; + } + } + if (imcf->enabled == NGX_CONF_UNSET) { imcf->enabled = 1; } @@ -742,6 +783,44 @@ ngx_http_ipng_stats_buckets(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } +/* ipng_stats_byte_buckets ... */ +static char * +ngx_http_ipng_stats_byte_buckets(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + ngx_http_ipng_stats_main_conf_t *imcf = conf; + ngx_str_t *value; + ngx_uint_t i, prev, n; + + value = cf->args->elts; + n = cf->args->nelts - 1; + + if (n < 1) { + return "requires at least one bucket boundary"; + } + + imcf->byte_bucket_bounds = ngx_palloc(cf->pool, n * sizeof(ngx_uint_t)); + if (imcf->byte_bucket_bounds == NULL) { + return NGX_CONF_ERROR; + } + imcf->nbytebuckets = n; + + prev = 0; + for (i = 0; i < n; i++) { + ssize_t v = ngx_parse_size(&value[i + 1]); + if (v == NGX_ERROR || v <= 0 || (ngx_uint_t) v <= prev) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "ipng_stats_byte_buckets values must be strictly increasing " + "positive sizes; got \"%V\"", &value[i + 1]); + return NGX_CONF_ERROR; + } + imcf->byte_bucket_bounds[i] = (ngx_uint_t) v; + prev = (ngx_uint_t) v; + } + return NGX_CONF_OK; +} + + static char * ngx_http_ipng_stats_scrape(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { @@ -1215,6 +1294,13 @@ ngx_http_ipng_stats_init_worker(ngx_cycle_t *cycle) return NGX_ERROR; } + arena_bytes = w->nalloc * (imcf->nbytebuckets + 1) * sizeof(uint64_t); + w->bin_arena = ngx_pcalloc(cycle->pool, arena_bytes); + w->bout_arena = ngx_pcalloc(cycle->pool, arena_bytes); + if (w->bin_arena == NULL || w->bout_arena == NULL) { + return NGX_ERROR; + } + /* Schedule first flush tick. */ ngx_memzero(&w->flush_ev, sizeof(w->flush_ev)); w->flush_ev.handler = ngx_http_ipng_stats_flush_timer; @@ -1312,7 +1398,7 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, ngx_http_ipng_stats_node_t *n; ngx_rbtree_node_t *rb; ngx_uint_t i; - ngx_uint_t nbuckets; + ngx_uint_t nbuckets, nbytebuckets; size_t node_size; ngx_atomic_uint_t *shared_lanes; @@ -1323,8 +1409,10 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, slab = (ngx_slab_pool_t *) imcf->shm_zone->shm.addr; sh = imcf->shm_zone->data; nbuckets = imcf->nbuckets; + nbytebuckets = imcf->nbytebuckets; node_size = sizeof(ngx_http_ipng_stats_node_t) - + 2 * (nbuckets + 1) * sizeof(ngx_atomic_uint_t); + + 2 * (nbuckets + 1) * sizeof(ngx_atomic_uint_t) + + 2 * (nbytebuckets + 1) * sizeof(ngx_atomic_uint_t); ngx_shmtx_lock(&slab->mutex); (void) ngx_atomic_fetch_add(&sh->flushes_total, 1); @@ -1350,7 +1438,7 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, n = (ngx_http_ipng_stats_node_t *) rb; if (n->source_id == slot->source_id && n->vip_id == slot->vip_id - && n->code == slot->code) + && n->class == slot->class) { break; } @@ -1374,12 +1462,16 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, slot->dhist[i] = 0; slot->uhist[i] = 0; } + for (i = 0; i <= nbytebuckets; i++) { + slot->bin_hist[i] = 0; + slot->bout_hist[i] = 0; + } continue; } n->rbnode.key = slot->hash; n->source_id = slot->source_id; n->vip_id = slot->vip_id; - n->code = slot->code; + n->class = slot->class; ngx_rbtree_insert(&sh->rbtree, &n->rbnode); ngx_queue_insert_tail(&sh->lru, &n->lru); } @@ -1400,6 +1492,20 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, &shared_lanes[nbuckets + 1 + i], slot->uhist[i]); } } + { + ngx_atomic_uint_t *bin_lanes = shared_lanes + 2 * (nbuckets + 1); + ngx_atomic_uint_t *bout_lanes = bin_lanes + (nbytebuckets + 1); + for (i = 0; i <= nbytebuckets; i++) { + if (slot->bin_hist[i]) { + (void) ngx_atomic_fetch_add(&bin_lanes[i], + slot->bin_hist[i]); + } + if (slot->bout_hist[i]) { + (void) ngx_atomic_fetch_add(&bout_lanes[i], + slot->bout_hist[i]); + } + } + } /* Clear local deltas. */ slot->requests = 0; @@ -1411,6 +1517,10 @@ ngx_http_ipng_stats_do_flush(ngx_http_ipng_stats_worker_t *w, slot->dhist[i] = 0; slot->uhist[i] = 0; } + for (i = 0; i <= nbytebuckets; i++) { + slot->bin_hist[i] = 0; + slot->bout_hist[i] = 0; + } ngx_queue_remove(&n->lru); ngx_queue_insert_tail(&sh->lru, &n->lru); @@ -1432,9 +1542,20 @@ ngx_http_ipng_stats_status_index(ngx_uint_t code) if (code < NGX_HTTP_IPNG_STATS_CODE_MIN || code > NGX_HTTP_IPNG_STATS_CODE_MAX) { - return NGX_HTTP_IPNG_STATS_CODE_UNKNOWN; + return NGX_HTTP_IPNG_STATS_CLASS_UNKNOWN; } - return code; + return code / 100; /* 1..5 */ +} + + +static const char * +ngx_http_ipng_stats_class_label(ngx_uint_t class) +{ + static const char *labels[NGX_HTTP_IPNG_STATS_NCLASSES] = { + "unknown", "1xx", "2xx", "3xx", "4xx", "5xx", + }; + if (class >= NGX_HTTP_IPNG_STATS_NCLASSES) return "unknown"; + return labels[class]; } @@ -1510,19 +1631,19 @@ ngx_http_ipng_stats_canonical_vip(ngx_http_request_t *r, u_char *buf, static ngx_http_ipng_stats_slot_t * ngx_http_ipng_stats_worker_slot(ngx_http_ipng_stats_worker_t *w, ngx_http_ipng_stats_main_conf_t *imcf, - ngx_uint_t source_id, ngx_uint_t vip_id, ngx_uint_t code) + ngx_uint_t source_id, ngx_uint_t vip_id, ngx_uint_t class) { ngx_http_ipng_stats_slot_t *s; ngx_uint_t i, hash; hash = (source_id * 2654435761u) ^ (vip_id * 40503u) - ^ (code * 131071u); + ^ (class * 131071u); for (i = 0; i < w->nslots; i++) { s = &w->slots[i]; if (s->hash == hash && s->source_id == source_id - && s->vip_id == vip_id && s->code == code) + && s->vip_id == vip_id && s->class == class) { return s; } @@ -1539,9 +1660,11 @@ ngx_http_ipng_stats_worker_slot(ngx_http_ipng_stats_worker_t *w, s->hash = hash; s->source_id = source_id; s->vip_id = vip_id; - s->code = code; + s->class = class; s->dhist = &w->dhist_arena[w->nslots * (imcf->nbuckets + 1)]; s->uhist = &w->uhist_arena[w->nslots * (imcf->nbuckets + 1)]; + s->bin_hist = &w->bin_arena [w->nslots * (imcf->nbytebuckets + 1)]; + s->bout_hist = &w->bout_arena[w->nslots * (imcf->nbytebuckets + 1)]; w->nslots++; return s; @@ -1572,8 +1695,9 @@ ngx_http_ipng_stats_log_handler(ngx_http_request_t *r) ngx_http_ipng_stats_slot_t *slot; ngx_str_t source, vip; u_char vipbuf[NGX_SOCKADDR_STRLEN]; - ngx_uint_t source_id, vip_id, code; + ngx_uint_t source_id, vip_id, class; ngx_uint_t bucket; + uint64_t bin_sz, bout_sz; ngx_msec_int_t elapsed_ms; ngx_time_t *tp; @@ -1604,7 +1728,7 @@ ngx_http_ipng_stats_log_handler(ngx_http_request_t *r) return NGX_OK; } - code = ngx_http_ipng_stats_status_index(r->headers_out.status); + class = ngx_http_ipng_stats_status_index(r->headers_out.status); /* Intern source and vip in the shared zone. This is the only * shared-zone write outside of flush — it runs rarely (once per @@ -1623,14 +1747,26 @@ ngx_http_ipng_stats_log_handler(ngx_http_request_t *r) } ngx_shmtx_unlock(&slab->mutex); - slot = ngx_http_ipng_stats_worker_slot(w, imcf, source_id, vip_id, code); + slot = ngx_http_ipng_stats_worker_slot(w, imcf, source_id, vip_id, class); if (slot == NULL) { return NGX_OK; } + bin_sz = r->request_length > 0 ? (uint64_t) r->request_length : 0; + bout_sz = (uint64_t) r->connection->sent; + slot->requests += 1; - slot->bytes_in += r->request_length > 0 ? (uint64_t) r->request_length : 0; - slot->bytes_out += (uint64_t) r->connection->sent; + slot->bytes_in += bin_sz; + slot->bytes_out += bout_sz; + + bucket = ngx_http_ipng_stats_bucket_index((ngx_uint_t) bin_sz, + imcf->byte_bucket_bounds, + imcf->nbytebuckets); + slot->bin_hist[bucket] += 1; + bucket = ngx_http_ipng_stats_bucket_index((ngx_uint_t) bout_sz, + imcf->byte_bucket_bounds, + imcf->nbytebuckets); + slot->bout_hist[bucket] += 1; /* Use the same formula nginx uses for $request_time: two-field * subtraction via ngx_timeofday(), which is the canonical way to @@ -1989,41 +2125,145 @@ ngx_http_ipng_stats_content_handler(ngx_http_request_t *r) } -/* Walk the shared-zone rbtree and invoke `emit` on each matching node. - * Caller holds the slab mutex. `emitted` is incremented for each node - * the emitter actually writes; renderers (e.g. JSON) use it to decide - * whether to prepend a separator. */ -typedef ngx_int_t (*ngx_http_ipng_stats_emit_pt)(ngx_http_request_t *r, - ngx_http_ipng_stats_main_conf_t *imcf, - ngx_http_ipng_stats_shctx_t *sh, - ngx_http_ipng_stats_node_t *n, - ngx_chain_t **cl, ngx_chain_t ***cl_last, ngx_uint_t *emitted); +/* Scrape rendering. + * + * Cardinality model (post-schema-2): counters are keyed by + * (source_tag, vip, code-class), where code-class is one of + * "1xx".."5xx" or "unknown" — six values rather than ~60 full codes. + * Histograms drop the code label entirely and aggregate across classes + * per (source_tag, vip). Operators who need a full per-code breakdown + * should enable ipng_stats_logtail (FR-8) and process the access-log + * stream off the hot path. + * + * The renderers below walk the shared-zone LRU once under the slab + * mutex, emit per-node counters inline, and accumulate histograms into + * a per-(source, vip) aggregation table allocated from r->pool. The + * aggregation table is then drained to produce one histogram group per + * (source, vip). */ + +typedef struct { + ngx_uint_t source_id; + ngx_uint_t vip_id; + ngx_uint_t used; + uint64_t duration_sum_ms; /* histogram _sum for request_duration */ + uint64_t upstream_sum_ms; + uint64_t bytes_in_sum; /* histogram _sum for bytes_in */ + uint64_t bytes_out_sum; + uint64_t req_total; /* total requests for this (source, vip) */ + uint64_t up_total; /* upstream observations */ + uint64_t *dhist; /* nbuckets+1 */ + uint64_t *uhist; + uint64_t *bin_hist; /* nbytebuckets+1 */ + uint64_t *bout_hist; +} ngx_http_ipng_stats_agg_t; + + +static ngx_chain_t * +ngx_http_ipng_stats_chain_buf(ngx_http_request_t *r, size_t size) +{ + ngx_buf_t *b; + ngx_chain_t *cl; + + b = ngx_create_temp_buf(r->pool, size); + if (b == NULL) return NULL; + cl = ngx_alloc_chain_link(r->pool); + if (cl == NULL) return NULL; + cl->buf = b; + cl->next = NULL; + return cl; +} static ngx_int_t -ngx_http_ipng_stats_walk(ngx_http_request_t *r, +ngx_http_ipng_stats_send(ngx_http_request_t *r, ngx_str_t *ctype, + ngx_chain_t *out) +{ + ngx_int_t rc; + + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.content_type = *ctype; + r->headers_out.content_type_len = ctype->len; + r->headers_out.content_length_n = -1; + + rc = ngx_http_send_header(r); + if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { + return rc; + } + if (out == NULL) { + return ngx_http_send_special(r, NGX_HTTP_LAST); + } + ngx_chain_t *last = out; + while (last->next) last = last->next; + last->buf->last_buf = 1; + last->buf->last_in_chain = 1; + return ngx_http_output_filter(r, out); +} + + +static ngx_int_t +ngx_http_ipng_stats_append(ngx_chain_t ***last, ngx_chain_t *cl) +{ + if (cl == NULL) return NGX_ERROR; + **last = cl; + *last = &cl->next; + return NGX_OK; +} + + +/* Find or create an aggregation entry for (source_id, vip_id). Linear + * scan — the table is small in practice (< 100 entries). */ +static ngx_http_ipng_stats_agg_t * +ngx_http_ipng_stats_agg_get(ngx_http_ipng_stats_agg_t *aggs, ngx_uint_t *naggs, + ngx_uint_t naggs_alloc, ngx_uint_t source_id, ngx_uint_t vip_id) +{ + ngx_uint_t i; + for (i = 0; i < *naggs; i++) { + if (aggs[i].source_id == source_id && aggs[i].vip_id == vip_id) { + return &aggs[i]; + } + } + if (*naggs >= naggs_alloc) return NULL; + aggs[*naggs].source_id = source_id; + aggs[*naggs].vip_id = vip_id; + aggs[*naggs].used = 1; + return &aggs[(*naggs)++]; +} + + +/* Walk matching nodes, emit per-node counters via `emit_counters`, + * accumulate histograms into `aggs`. Caller holds slab mutex. + * `ctx` is opaque — passed through to the emit callback. Returns number + * of distinct (source, vip) pairs observed, or -1 on error. */ +typedef ngx_int_t (*ngx_http_ipng_stats_counters_pt)(ngx_http_request_t *r, + ngx_http_ipng_stats_shctx_t *sh, ngx_http_ipng_stats_node_t *n, + ngx_chain_t ***last, ngx_uint_t *emitted); + +static ngx_int_t +ngx_http_ipng_stats_walk_aggregate(ngx_http_request_t *r, ngx_http_ipng_stats_main_conf_t *imcf, ngx_str_t *filter_src, ngx_str_t *filter_vip, - ngx_http_ipng_stats_emit_pt emit, - ngx_chain_t **cl, ngx_chain_t ***cl_last) + ngx_http_ipng_stats_counters_pt emit_counters, + ngx_chain_t ***last, ngx_uint_t *emitted, + ngx_http_ipng_stats_agg_t *aggs, ngx_uint_t naggs_alloc, + ngx_uint_t *naggs_out) { ngx_http_ipng_stats_shctx_t *sh = imcf->shm_zone->data; - ngx_slab_pool_t *slab; ngx_queue_t *q; ngx_http_ipng_stats_node_t *n; ngx_str_t *src_entry, *vip_entry; - ngx_int_t rc = NGX_OK; - ngx_uint_t emitted = 0; - - slab = (ngx_slab_pool_t *) imcf->shm_zone->shm.addr; - ngx_shmtx_lock(&slab->mutex); + ngx_atomic_uint_t *lanes, *blanes; + ngx_http_ipng_stats_agg_t *a; + ngx_uint_t i; + ngx_uint_t nb = imcf->nbuckets; + ngx_uint_t nbb = imcf->nbytebuckets; + ngx_uint_t naggs = 0; + ngx_int_t rc; for (q = ngx_queue_head(&sh->lru); q != ngx_queue_sentinel(&sh->lru); q = ngx_queue_next(q)) { n = ngx_queue_data(q, ngx_http_ipng_stats_node_t, lru); - if (n->source_id >= sh->sources.nelts || n->vip_id >= sh->vips.nelts) { @@ -2047,179 +2287,125 @@ ngx_http_ipng_stats_walk(ngx_http_request_t *r, continue; } - rc = emit(r, imcf, sh, n, cl, cl_last, &emitted); - if (rc != NGX_OK) break; + rc = emit_counters(r, sh, n, last, emitted); + if (rc != NGX_OK) return NGX_ERROR; + + a = ngx_http_ipng_stats_agg_get(aggs, &naggs, naggs_alloc, + n->source_id, n->vip_id); + if (a == NULL) continue; + + a->duration_sum_ms += n->duration_sum_ms; + a->upstream_sum_ms += n->upstream_sum_ms; + a->bytes_in_sum += n->bytes_in; + a->bytes_out_sum += n->bytes_out; + a->req_total += n->requests; + + lanes = (ngx_atomic_uint_t *) (n + 1); + blanes = lanes + 2 * (nb + 1); + + for (i = 0; i <= nb; i++) { + a->dhist[i] += lanes[i]; + a->uhist[i] += lanes[nb + 1 + i]; + a->up_total += lanes[nb + 1 + i]; + } + for (i = 0; i <= nbb; i++) { + a->bin_hist[i] += blanes[i]; + a->bout_hist[i] += blanes[nbb + 1 + i]; + } } - ngx_shmtx_unlock(&slab->mutex); - return rc; -} - - -static ngx_chain_t * -ngx_http_ipng_stats_chain_buf(ngx_http_request_t *r, size_t size) -{ - ngx_buf_t *b; - ngx_chain_t *cl; - - b = ngx_create_temp_buf(r->pool, size); - if (b == NULL) return NULL; - cl = ngx_alloc_chain_link(r->pool); - if (cl == NULL) return NULL; - cl->buf = b; - cl->next = NULL; - return cl; + *naggs_out = naggs; + return NGX_OK; } +/* Allocate the aggregation table plus its lane arena from r->pool. */ static ngx_int_t -ngx_http_ipng_stats_emit_prom(ngx_http_request_t *r, - ngx_http_ipng_stats_main_conf_t *imcf, - ngx_http_ipng_stats_shctx_t *sh, - ngx_http_ipng_stats_node_t *n, - ngx_chain_t **cl_head, ngx_chain_t ***cl_tail, ngx_uint_t *emitted) +ngx_http_ipng_stats_agg_alloc(ngx_http_request_t *r, + ngx_http_ipng_stats_main_conf_t *imcf, ngx_uint_t n, + ngx_http_ipng_stats_agg_t **aggs_out, ngx_uint_t *nalloc_out) { - ngx_chain_t *cl; - ngx_buf_t *b; - ngx_str_t *src = &sh->sources.entries[n->source_id]; - ngx_str_t *vip = &sh->vips.entries[n->vip_id]; + ngx_uint_t nb = imcf->nbuckets; + ngx_uint_t nbb = imcf->nbytebuckets; + size_t lanes_per = 2 * (nb + 1) + 2 * (nbb + 1); + uint64_t *arena; + ngx_http_ipng_stats_agg_t *aggs; ngx_uint_t i; - (void) cl_head; + if (n == 0) n = 1; + aggs = ngx_pcalloc(r->pool, n * sizeof(*aggs)); + arena = ngx_pcalloc(r->pool, n * lanes_per * sizeof(uint64_t)); + if (aggs == NULL || arena == NULL) return NGX_ERROR; - /* Reserve enough space for one rendered key: the worst-case single - * Prometheus line with a long label set plus the histogram tail. */ - cl = ngx_http_ipng_stats_chain_buf(r, 2048 + 64 * imcf->nbuckets); - if (cl == NULL) return NGX_ERROR; - b = cl->buf; - - b->last = ngx_sprintf(b->last, - "nginx_ipng_requests_total{source_tag=\"%V\",vip=\"%V\",code=\"%ui\"} %uA\n" - "nginx_ipng_bytes_in_total{source_tag=\"%V\",vip=\"%V\",code=\"%ui\"} %uA\n" - "nginx_ipng_bytes_out_total{source_tag=\"%V\",vip=\"%V\",code=\"%ui\"} %uA\n", - src, vip, n->code, n->requests, - src, vip, n->code, n->bytes_in, - src, vip, n->code, n->bytes_out); - - /* Histogram is per (source, vip); emit only for a canonical code - * slot (0 or the first seen) to avoid N-fold duplication. Keep it - * simple in v0.1: emit once per node, not once per (source, vip). - * Operators who need true histogram aggregation by (source, vip) - * should sum over the code dimension in PromQL. */ - ngx_atomic_uint_t *lanes = (ngx_atomic_uint_t *) (n + 1); - ngx_uint_t cumulative = 0; - - for (i = 0; i < imcf->nbuckets; i++) { - cumulative += lanes[i]; - b->last = ngx_sprintf(b->last, - "nginx_ipng_request_duration_seconds_bucket" - "{source_tag=\"%V\",vip=\"%V\",code=\"%ui\",le=\"%.3f\"} %ui\n", - src, vip, n->code, - (double) imcf->bucket_bounds_ms[i] / 1000.0, - cumulative); + for (i = 0; i < n; i++) { + uint64_t *base = &arena[i * lanes_per]; + aggs[i].dhist = base; + aggs[i].uhist = base + (nb + 1); + aggs[i].bin_hist = base + 2 * (nb + 1); + aggs[i].bout_hist = base + 2 * (nb + 1) + (nbb + 1); } - cumulative += lanes[imcf->nbuckets]; - b->last = ngx_sprintf(b->last, - "nginx_ipng_request_duration_seconds_bucket" - "{source_tag=\"%V\",vip=\"%V\",code=\"%ui\",le=\"+Inf\"} %ui\n" - "nginx_ipng_request_duration_seconds_sum" - "{source_tag=\"%V\",vip=\"%V\",code=\"%ui\"} %.3f\n" - "nginx_ipng_request_duration_seconds_count" - "{source_tag=\"%V\",vip=\"%V\",code=\"%ui\"} %ui\n", - src, vip, n->code, cumulative, - src, vip, n->code, (double) n->duration_sum_ms / 1000.0, - src, vip, n->code, cumulative); - - b->last_buf = 0; - b->last_in_chain = 0; - - **cl_tail = cl; - *cl_tail = &cl->next; - (*emitted)++; + *aggs_out = aggs; + *nalloc_out = n; return NGX_OK; } +/* -- Prometheus ---------------------------------------------------- */ + static ngx_int_t -ngx_http_ipng_stats_emit_json(ngx_http_request_t *r, - ngx_http_ipng_stats_main_conf_t *imcf, - ngx_http_ipng_stats_shctx_t *sh, - ngx_http_ipng_stats_node_t *n, - ngx_chain_t **cl_head, ngx_chain_t ***cl_tail, ngx_uint_t *emitted) +ngx_http_ipng_stats_prom_counters(ngx_http_request_t *r, + ngx_http_ipng_stats_shctx_t *sh, ngx_http_ipng_stats_node_t *n, + ngx_chain_t ***last, ngx_uint_t *emitted) { - /* v0.1 JSON output is a flat array of records, one per - * (source, vip, code) key, each carrying the counters and the - * request-duration histogram. The top-level `schema` field is - * versioned; consumers MUST check it before parsing. */ - ngx_chain_t *cl; - ngx_buf_t *b; - ngx_str_t *src = &sh->sources.entries[n->source_id]; - ngx_str_t *vip = &sh->vips.entries[n->vip_id]; - ngx_atomic_uint_t *lanes; - ngx_uint_t i; - size_t buf_size; - - (void) cl_head; - - /* Upper-bound: record header (~256B) + one bucket entry (~32B each) + - * closing braces. Round up generously. */ - buf_size = 512 + 48 * (imcf->nbuckets + 1); - - cl = ngx_http_ipng_stats_chain_buf(r, buf_size); + ngx_str_t *src = &sh->sources.entries[n->source_id]; + ngx_str_t *vip = &sh->vips.entries[n->vip_id]; + const char *cls = ngx_http_ipng_stats_class_label(n->class); + ngx_chain_t *cl = ngx_http_ipng_stats_chain_buf(r, 1024); if (cl == NULL) return NGX_ERROR; - b = cl->buf; - - b->last = ngx_sprintf(b->last, - "%s{\"source_tag\":\"%V\",\"vip\":\"%V\",\"code\":%ui," - "\"requests\":%uA,\"bytes_in\":%uA,\"bytes_out\":%uA," - "\"request_duration_ms\":{" - "\"sum\":%uA,\"count\":%uA,\"buckets\":{", - (*emitted == 0) ? "" : ",", - src, vip, n->code, - n->requests, n->bytes_in, n->bytes_out, - n->duration_sum_ms, n->requests); - - lanes = (ngx_atomic_uint_t *) (n + 1); - for (i = 0; i < imcf->nbuckets; i++) { - b->last = ngx_sprintf(b->last, "%s\"%ui\":%uA", - (i == 0) ? "" : ",", - imcf->bucket_bounds_ms[i], lanes[i]); - } - b->last = ngx_sprintf(b->last, ",\"+Inf\":%uA}}}", - lanes[imcf->nbuckets]); - - **cl_tail = cl; - *cl_tail = &cl->next; + cl->buf->last = ngx_sprintf(cl->buf->last, + "nginx_ipng_requests_total{source_tag=\"%V\",vip=\"%V\",code=\"%s\"} %uA\n" + "nginx_ipng_bytes_in_total{source_tag=\"%V\",vip=\"%V\",code=\"%s\"} %uA\n" + "nginx_ipng_bytes_out_total{source_tag=\"%V\",vip=\"%V\",code=\"%s\"} %uA\n" + "nginx_ipng_latency_total{source_tag=\"%V\",vip=\"%V\",code=\"%s\"} %.3f\n", + src, vip, cls, n->requests, + src, vip, cls, n->bytes_in, + src, vip, cls, n->bytes_out, + src, vip, cls, (double) n->duration_sum_ms / 1000.0); (*emitted)++; - return NGX_OK; + return ngx_http_ipng_stats_append(last, cl); } -static ngx_int_t -ngx_http_ipng_stats_send(ngx_http_request_t *r, ngx_str_t *ctype, - ngx_chain_t *out) +static u_char * +ngx_http_ipng_stats_render_hist(u_char *p, const char *metric, + ngx_str_t *src, ngx_str_t *vip, ngx_uint_t *bounds, ngx_uint_t nb, + uint64_t *lanes, double sum_units, int is_seconds) { - ngx_int_t rc; + ngx_uint_t i; + uint64_t cum = 0; - r->headers_out.status = NGX_HTTP_OK; - r->headers_out.content_type = *ctype; - r->headers_out.content_type_len = ctype->len; - r->headers_out.content_length_n = -1; /* chunked */ - - rc = ngx_http_send_header(r); - if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { - return rc; + for (i = 0; i < nb; i++) { + cum += lanes[i]; + if (is_seconds) { + p = ngx_sprintf(p, + "%s_bucket{source_tag=\"%V\",vip=\"%V\",le=\"%.3f\"} %uL\n", + metric, src, vip, + (double) bounds[i] / 1000.0, cum); + } else { + p = ngx_sprintf(p, + "%s_bucket{source_tag=\"%V\",vip=\"%V\",le=\"%ui\"} %uL\n", + metric, src, vip, bounds[i], cum); + } } - if (out == NULL) { - return ngx_http_send_special(r, NGX_HTTP_LAST); - } - /* Mark final chain link. */ - ngx_chain_t *last = out; - while (last->next) last = last->next; - last->buf->last_buf = 1; - last->buf->last_in_chain = 1; - - return ngx_http_output_filter(r, out); + cum += lanes[nb]; + p = ngx_sprintf(p, + "%s_bucket{source_tag=\"%V\",vip=\"%V\",le=\"+Inf\"} %uL\n" + "%s_sum{source_tag=\"%V\",vip=\"%V\"} %.3f\n" + "%s_count{source_tag=\"%V\",vip=\"%V\"} %uL\n", + metric, src, vip, cum, + metric, src, vip, sum_units, + metric, src, vip, cum); + return p; } @@ -2228,16 +2414,24 @@ ngx_http_ipng_stats_render_prom(ngx_http_request_t *r, ngx_http_ipng_stats_main_conf_t *imcf, ngx_str_t *filter_source, ngx_str_t *filter_vip) { - ngx_chain_t *out = NULL; - ngx_chain_t **last = &out; - ngx_str_t ctype = ngx_string("text/plain; version=0.0.4"); - ngx_chain_t *hdr_cl; - ngx_buf_t *hdr_b; + ngx_http_ipng_stats_shctx_t *sh = imcf->shm_zone->data; + ngx_slab_pool_t *slab; + ngx_chain_t *out = NULL, *cl; + ngx_chain_t **last = &out; + ngx_str_t ctype = ngx_string("text/plain; version=0.0.4"); + ngx_http_ipng_stats_agg_t *aggs; + ngx_uint_t naggs = 0, naggs_alloc; + ngx_uint_t emitted = 0, i; + ngx_uint_t nb = imcf->nbuckets; + ngx_uint_t nbb = imcf->nbytebuckets; + size_t hist_sz; + ngx_int_t rc; - hdr_cl = ngx_http_ipng_stats_chain_buf(r, 512); - if (hdr_cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; - hdr_b = hdr_cl->buf; - hdr_b->last = ngx_sprintf(hdr_b->last, + slab = (ngx_slab_pool_t *) imcf->shm_zone->shm.addr; + + cl = ngx_http_ipng_stats_chain_buf(r, 1536); + if (cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + cl->buf->last = ngx_sprintf(cl->buf->last, "# nginx-ipng-stats-plugin %s (schema=%d)\n" "# HELP nginx_ipng_requests_total Total HTTP requests.\n" "# TYPE nginx_ipng_requests_total counter\n" @@ -2245,56 +2439,320 @@ ngx_http_ipng_stats_render_prom(ngx_http_request_t *r, "# TYPE nginx_ipng_bytes_in_total counter\n" "# HELP nginx_ipng_bytes_out_total Response bytes sent.\n" "# TYPE nginx_ipng_bytes_out_total counter\n" - "# HELP nginx_ipng_request_duration_seconds Request duration.\n" - "# TYPE nginx_ipng_request_duration_seconds histogram\n", + "# HELP nginx_ipng_latency_total Sum of request durations in seconds.\n" + "# TYPE nginx_ipng_latency_total counter\n" + "# HELP nginx_ipng_request_duration_seconds Request duration histogram.\n" + "# TYPE nginx_ipng_request_duration_seconds histogram\n" + "# HELP nginx_ipng_upstream_response_seconds Upstream response-time histogram.\n" + "# TYPE nginx_ipng_upstream_response_seconds histogram\n" + "# HELP nginx_ipng_bytes_in Request size histogram in bytes.\n" + "# TYPE nginx_ipng_bytes_in histogram\n" + "# HELP nginx_ipng_bytes_out Response size histogram in bytes.\n" + "# TYPE nginx_ipng_bytes_out histogram\n", NGX_HTTP_IPNG_STATS_VERSION, NGX_HTTP_IPNG_STATS_SCHEMA_VERSION); - *last = hdr_cl; - last = &hdr_cl->next; - - if (ngx_http_ipng_stats_walk(r, imcf, filter_source, filter_vip, - ngx_http_ipng_stats_emit_prom, - &out, &last) != NGX_OK) - { + if (ngx_http_ipng_stats_append(&last, cl) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } + ngx_shmtx_lock(&slab->mutex); + + if (ngx_http_ipng_stats_agg_alloc(r, imcf, + sh->sources.nelts * sh->vips.nelts, &aggs, &naggs_alloc) != NGX_OK) + { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rc = ngx_http_ipng_stats_walk_aggregate(r, imcf, filter_source, filter_vip, + ngx_http_ipng_stats_prom_counters, &last, &emitted, + aggs, naggs_alloc, &naggs); + if (rc != NGX_OK) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + /* One chain link per (source, vip) for the four aggregated histograms. + * Size: per-bucket line ~96B, + sum/count/+Inf per metric ~96B each. */ + hist_sz = 256 + 96 * (2 * (nb + 1) + 2 * (nbb + 1)) + 4 * 200; + + for (i = 0; i < naggs; i++) { + ngx_http_ipng_stats_agg_t *a = &aggs[i]; + ngx_str_t *src = &sh->sources.entries[a->source_id]; + ngx_str_t *vip = &sh->vips.entries[a->vip_id]; + u_char *p; + + cl = ngx_http_ipng_stats_chain_buf(r, hist_sz); + if (cl == NULL) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + p = cl->buf->last; + p = ngx_http_ipng_stats_render_hist(p, + "nginx_ipng_request_duration_seconds", src, vip, + imcf->bucket_bounds_ms, nb, a->dhist, + (double) a->duration_sum_ms / 1000.0, 1); + p = ngx_http_ipng_stats_render_hist(p, + "nginx_ipng_upstream_response_seconds", src, vip, + imcf->bucket_bounds_ms, nb, a->uhist, + (double) a->upstream_sum_ms / 1000.0, 1); + p = ngx_http_ipng_stats_render_hist(p, + "nginx_ipng_bytes_in", src, vip, + imcf->byte_bucket_bounds, nbb, a->bin_hist, + (double) a->bytes_in_sum, 0); + p = ngx_http_ipng_stats_render_hist(p, + "nginx_ipng_bytes_out", src, vip, + imcf->byte_bucket_bounds, nbb, a->bout_hist, + (double) a->bytes_out_sum, 0); + cl->buf->last = p; + if (ngx_http_ipng_stats_append(&last, cl) != NGX_OK) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + } + + ngx_shmtx_unlock(&slab->mutex); return ngx_http_ipng_stats_send(r, &ctype, out); } +/* -- JSON ---------------------------------------------------------- */ + +/* Per-(source, vip, class) counter group. We render one JSON object per + * aggregated (source, vip) record, with the class breakdown stored in + * an interim table while the walk is in progress. */ +typedef struct { + ngx_uint_t source_id; + ngx_uint_t vip_id; + ngx_uint_t class; + uint64_t requests; + uint64_t bytes_in; + uint64_t bytes_out; + uint64_t duration_sum_ms; + uint64_t upstream_sum_ms; +} ngx_http_ipng_stats_jnode_t; + + static ngx_int_t ngx_http_ipng_stats_render_json(ngx_http_request_t *r, ngx_http_ipng_stats_main_conf_t *imcf, ngx_str_t *filter_source, ngx_str_t *filter_vip) { - ngx_chain_t *out = NULL; - ngx_chain_t **last = &out; - ngx_chain_t *hdr_cl, *tail_cl; - ngx_buf_t *hdr_b, *tail_b; - ngx_str_t ctype = ngx_string("application/json"); + ngx_http_ipng_stats_shctx_t *sh = imcf->shm_zone->data; + ngx_slab_pool_t *slab; + ngx_chain_t *out = NULL, *cl; + ngx_chain_t **last = &out; + ngx_str_t ctype = ngx_string("application/json"); + ngx_http_ipng_stats_agg_t *aggs; + ngx_http_ipng_stats_jnode_t *jnodes; + ngx_uint_t njnodes = 0, njnodes_alloc; + ngx_uint_t naggs = 0, naggs_alloc; + ngx_uint_t i, j, emitted = 0; + ngx_uint_t nb = imcf->nbuckets; + ngx_uint_t nbb = imcf->nbytebuckets; + ngx_queue_t *q; + ngx_http_ipng_stats_node_t *nd; + ngx_str_t *src_entry, *vip_entry; + ngx_atomic_uint_t *lanes, *blanes; + ngx_http_ipng_stats_agg_t *a; + size_t rec_sz; - hdr_cl = ngx_http_ipng_stats_chain_buf(r, 64); - if (hdr_cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; - hdr_b = hdr_cl->buf; - hdr_b->last = ngx_sprintf(hdr_b->last, + slab = (ngx_slab_pool_t *) imcf->shm_zone->shm.addr; + + cl = ngx_http_ipng_stats_chain_buf(r, 64); + if (cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + cl->buf->last = ngx_sprintf(cl->buf->last, "{\"schema\":%d,\"records\":[", NGX_HTTP_IPNG_STATS_SCHEMA_VERSION); - *last = hdr_cl; - last = &hdr_cl->next; - - if (ngx_http_ipng_stats_walk(r, imcf, filter_source, filter_vip, - ngx_http_ipng_stats_emit_json, - &out, &last) != NGX_OK) - { + if (ngx_http_ipng_stats_append(&last, cl) != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } - tail_cl = ngx_http_ipng_stats_chain_buf(r, 8); - if (tail_cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; - tail_b = tail_cl->buf; - tail_b->last = ngx_sprintf(tail_b->last, "]}\n"); - *last = tail_cl; - last = &tail_cl->next; + ngx_shmtx_lock(&slab->mutex); + + naggs_alloc = sh->sources.nelts * sh->vips.nelts; + if (ngx_http_ipng_stats_agg_alloc(r, imcf, naggs_alloc, + &aggs, &naggs_alloc) != NGX_OK) + { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + /* Upper bound on jnodes = naggs_alloc * NCLASSES. */ + njnodes_alloc = naggs_alloc * NGX_HTTP_IPNG_STATS_NCLASSES; + if (njnodes_alloc == 0) njnodes_alloc = 1; + jnodes = ngx_pcalloc(r->pool, njnodes_alloc * sizeof(*jnodes)); + if (jnodes == NULL) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + for (q = ngx_queue_head(&sh->lru); + q != ngx_queue_sentinel(&sh->lru); + q = ngx_queue_next(q)) + { + nd = ngx_queue_data(q, ngx_http_ipng_stats_node_t, lru); + if (nd->source_id >= sh->sources.nelts + || nd->vip_id >= sh->vips.nelts) + { + continue; + } + src_entry = &sh->sources.entries[nd->source_id]; + vip_entry = &sh->vips.entries[nd->vip_id]; + + if (filter_source->len > 0 + && (src_entry->len != filter_source->len + || ngx_memcmp(src_entry->data, filter_source->data, + filter_source->len) != 0)) + { + continue; + } + if (filter_vip->len > 0 + && (vip_entry->len != filter_vip->len + || ngx_memcmp(vip_entry->data, filter_vip->data, + filter_vip->len) != 0)) + { + continue; + } + + if (njnodes < njnodes_alloc) { + jnodes[njnodes].source_id = nd->source_id; + jnodes[njnodes].vip_id = nd->vip_id; + jnodes[njnodes].class = nd->class; + jnodes[njnodes].requests = nd->requests; + jnodes[njnodes].bytes_in = nd->bytes_in; + jnodes[njnodes].bytes_out = nd->bytes_out; + jnodes[njnodes].duration_sum_ms = nd->duration_sum_ms; + jnodes[njnodes].upstream_sum_ms = nd->upstream_sum_ms; + njnodes++; + } + + a = ngx_http_ipng_stats_agg_get(aggs, &naggs, naggs_alloc, + nd->source_id, nd->vip_id); + if (a == NULL) continue; + a->duration_sum_ms += nd->duration_sum_ms; + a->upstream_sum_ms += nd->upstream_sum_ms; + a->bytes_in_sum += nd->bytes_in; + a->bytes_out_sum += nd->bytes_out; + a->req_total += nd->requests; + + lanes = (ngx_atomic_uint_t *) (nd + 1); + blanes = lanes + 2 * (nb + 1); + for (i = 0; i <= nb; i++) { + a->dhist[i] += lanes[i]; + a->uhist[i] += lanes[nb + 1 + i]; + a->up_total += lanes[nb + 1 + i]; + } + for (i = 0; i <= nbb; i++) { + a->bin_hist[i] += blanes[i]; + a->bout_hist[i] += blanes[nbb + 1 + i]; + } + } + + /* One JSON record per aggregated (source, vip). Size upper-bound + * accounts for: fixed overhead, up to NCLASSES class entries, 4 + * histograms. */ + rec_sz = 512 + + 160 * NGX_HTTP_IPNG_STATS_NCLASSES + + 48 * (2 * (nb + 1) + 2 * (nbb + 1)) + + 4 * 128; + + for (i = 0; i < naggs; i++) { + a = &aggs[i]; + ngx_str_t *src = &sh->sources.entries[a->source_id]; + ngx_str_t *vip = &sh->vips.entries[a->vip_id]; + u_char *p; + ngx_uint_t first_class = 1; + + cl = ngx_http_ipng_stats_chain_buf(r, rec_sz); + if (cl == NULL) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + p = cl->buf->last; + p = ngx_sprintf(p, + "%s{\"source_tag\":\"%V\",\"vip\":\"%V\",\"classes\":{", + (emitted == 0) ? "" : ",", src, vip); + + for (j = 0; j < njnodes; j++) { + if (jnodes[j].source_id != a->source_id + || jnodes[j].vip_id != a->vip_id) continue; + p = ngx_sprintf(p, + "%s\"%s\":{\"requests\":%uL,\"bytes_in\":%uL," + "\"bytes_out\":%uL,\"latency_ms\":%uL," + "\"upstream_latency_ms\":%uL}", + first_class ? "" : ",", + ngx_http_ipng_stats_class_label(jnodes[j].class), + jnodes[j].requests, jnodes[j].bytes_in, + jnodes[j].bytes_out, jnodes[j].duration_sum_ms, + jnodes[j].upstream_sum_ms); + first_class = 0; + } + p = ngx_sprintf(p, "},\"request_duration_ms\":{\"sum\":%uL," + "\"count\":%uL,\"buckets\":{", + a->duration_sum_ms, a->req_total); + for (j = 0; j <= nb; j++) { + if (j == nb) { + p = ngx_sprintf(p, "%s\"+Inf\":%uL", j == 0 ? "" : ",", + a->dhist[j]); + } else { + p = ngx_sprintf(p, "%s\"%ui\":%uL", j == 0 ? "" : ",", + imcf->bucket_bounds_ms[j], a->dhist[j]); + } + } + p = ngx_sprintf(p, "}},\"upstream_response_ms\":{\"sum\":%uL," + "\"count\":%uL,\"buckets\":{", + a->upstream_sum_ms, a->up_total); + for (j = 0; j <= nb; j++) { + if (j == nb) { + p = ngx_sprintf(p, "%s\"+Inf\":%uL", j == 0 ? "" : ",", + a->uhist[j]); + } else { + p = ngx_sprintf(p, "%s\"%ui\":%uL", j == 0 ? "" : ",", + imcf->bucket_bounds_ms[j], a->uhist[j]); + } + } + p = ngx_sprintf(p, + "}},\"bytes_in\":{\"sum\":%uL,\"count\":%uL,\"buckets\":{", + a->bytes_in_sum, a->req_total); + for (j = 0; j <= nbb; j++) { + if (j == nbb) { + p = ngx_sprintf(p, "%s\"+Inf\":%uL", j == 0 ? "" : ",", + a->bin_hist[j]); + } else { + p = ngx_sprintf(p, "%s\"%ui\":%uL", j == 0 ? "" : ",", + imcf->byte_bucket_bounds[j], a->bin_hist[j]); + } + } + p = ngx_sprintf(p, + "}},\"bytes_out\":{\"sum\":%uL,\"count\":%uL,\"buckets\":{", + a->bytes_out_sum, a->req_total); + for (j = 0; j <= nbb; j++) { + if (j == nbb) { + p = ngx_sprintf(p, "%s\"+Inf\":%uL", j == 0 ? "" : ",", + a->bout_hist[j]); + } else { + p = ngx_sprintf(p, "%s\"%ui\":%uL", j == 0 ? "" : ",", + imcf->byte_bucket_bounds[j], a->bout_hist[j]); + } + } + p = ngx_sprintf(p, "}}}"); + + cl->buf->last = p; + if (ngx_http_ipng_stats_append(&last, cl) != NGX_OK) { + ngx_shmtx_unlock(&slab->mutex); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + emitted++; + } + + ngx_shmtx_unlock(&slab->mutex); + + cl = ngx_http_ipng_stats_chain_buf(r, 8); + if (cl == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + cl->buf->last = ngx_sprintf(cl->buf->last, "]}\n"); + if (ngx_http_ipng_stats_append(&last, cl) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } return ngx_http_ipng_stats_send(r, &ctype, out); } diff --git a/tests/01-module/01-e2e.robot b/tests/01-module/01-e2e.robot index 24b0e6e..12bc3ec 100644 --- a/tests/01-module/01-e2e.robot +++ b/tests/01-module/01-e2e.robot @@ -71,14 +71,14 @@ Direct traffic tagged # --- Status code tracking --- -Per-code counters - [Documentation] 404 and 200 appear as distinct code= labels. +Per-class code counters + [Documentation] 4xx and 2xx appear as class-bucketed code= labels. Docker Exec Ignore Rc ${CLIENT1} curl -s http://10.0.1.1:8080/notfound Docker Exec Ignore Rc ${CLIENT1} curl -s http://10.0.1.1:8080/notfound Wait For Flush ${output} = Scrape With Filter source_tag=cl1 - Should Contain ${output} code="404" - Should Contain ${output} code="200" + Should Contain ${output} code="4xx" + Should Contain ${output} code="2xx" # --- Duration histogram ---