Harden scrape rendering and add AddressSanitizer test suite
Move all heap allocation out of the slab-mutex critical section in render_prom/render_json: snapshot cardinality under a brief lock, allocate aggs/snaps/string tables outside the lock, then re-acquire only to deep-copy strings and walk the LRU into the pre-allocated buffers. A worker crash during output buffer allocation can no longer leave the shared-memory zone locked, and a corrupt cardinality count is caught by a 10k sanity cap rather than causing a runaway ngx_pcalloc. Add build-asan and tests/02-asan/: a full sanitizer-instrumented nginx + module built via apt-source, and a 2-node containerlab Robot suite that drives reload storms, concurrent scrape-during-reload, and intern-table growth, failing if AddressSanitizer or UBSan reports anything on stderr. The two Robot suites now check for their required build artifacts up front so `make robot-test` no longer rebuilds them on every invocation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
84
Makefile
84
Makefile
@@ -27,7 +27,7 @@ VERSION := 0.2.0
|
||||
|
||||
NGINX_SRC ?=
|
||||
|
||||
.PHONY: help build pkg-deb robot-test install-deps clean fetch-nginx-src version-header
|
||||
.PHONY: help build build-asan pkg-deb robot-test install-deps clean fetch-nginx-src version-header
|
||||
|
||||
TEST ?= tests/
|
||||
|
||||
@@ -35,8 +35,10 @@ help:
|
||||
@echo "nginx-ipng-stats-plugin — make targets"
|
||||
@echo ""
|
||||
@echo " make build Build $(MODULE_NAME).so out-of-tree."
|
||||
@echo " make build-asan Build a full nginx + module with AddressSanitizer"
|
||||
@echo " into build/nginx-asan/ for local crash-hunting."
|
||||
@echo " make pkg-deb Build a Debian package via dpkg-buildpackage."
|
||||
@echo " make robot-test Build .deb, then run Robot Framework e2e tests."
|
||||
@echo " make robot-test Run Robot Framework e2e tests (all suites)."
|
||||
@echo " make install-deps Install build and test dependencies (apt)."
|
||||
@echo " make clean Remove build artifacts."
|
||||
@echo ""
|
||||
@@ -114,6 +116,80 @@ fetch-nginx-src:
|
||||
mv "$$NGX_DIR" $(BUILD_DIR)/nginx-src; \
|
||||
rm -rf $(BUILD_DIR)/apt-src
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# build-asan: full nginx + module built with AddressSanitizer. ASan
|
||||
# needs to be present in the main binary to instrument dynamic modules;
|
||||
# running a stock nginx with LD_PRELOAD=libasan.so does NOT work with
|
||||
# a statically-linked module's allocations. So we build nginx itself
|
||||
# here, into an isolated prefix under build/nginx-asan/, which you can
|
||||
# run by hand against any config. Normal `make build` is unaffected.
|
||||
#
|
||||
# Typical workflow for the reload-crash hunt:
|
||||
# make build-asan
|
||||
# cp your-test.conf build/nginx-asan/conf/nginx.conf
|
||||
# ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:halt_on_error=1:detect_leaks=0 \
|
||||
# build/nginx-asan/sbin/nginx -p build/nginx-asan
|
||||
# # in another shell:
|
||||
# build/nginx-asan/sbin/nginx -p build/nginx-asan -s reload
|
||||
# ASan writes any findings to stderr of the master before aborting.
|
||||
#
|
||||
# detect_odr_violation=0 suppresses false positives for symbols nginx
|
||||
# intentionally duplicates between its main binary and dynamic modules
|
||||
# (e.g. ngx_module_names in objs/ngx_http_ipng_stats_module_modules.c).
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# -fno-sanitize=nonnull-attribute suppresses a UBSan finding in nginx's
|
||||
# ngx_cpymem (src/core/ngx_string.c:84): the macro expands to memcpy(...)
|
||||
# which glibc declares _Nonnull, but nginx legitimately calls it with
|
||||
# (dst, NULL, 0) in many places. Filtering just that check keeps real
|
||||
# undefined behaviour visible.
|
||||
ASAN_CFLAGS := -fsanitize=address -fsanitize=undefined -fno-sanitize=nonnull-attribute -fno-omit-frame-pointer -fno-common -g -O1
|
||||
ASAN_LDFLAGS := -fsanitize=address -fsanitize=undefined
|
||||
|
||||
# The ASan build rebuilds nginx from source rather than only the module,
|
||||
# because ASan must instrument the main binary to catch heap issues in
|
||||
# modules it loads. The `nginx-dev` shim source tree under /usr/share
|
||||
# only carries headers, so we always fall through to `apt source nginx`
|
||||
# here and keep the full source tree isolated in build/nginx-asan-src/.
|
||||
build-asan: version-header
|
||||
@set -e; \
|
||||
NGX_SRC="$(BUILD_DIR)/nginx-asan-src"; \
|
||||
if [ ! -d "$$NGX_SRC" ] || [ ! -f "$$NGX_SRC/src/core/nginx.c" ]; then \
|
||||
echo "Fetching full nginx source via apt-source for ASan build"; \
|
||||
mkdir -p $(BUILD_DIR)/apt-src-asan; \
|
||||
rm -rf $$NGX_SRC; \
|
||||
cd $(BUILD_DIR)/apt-src-asan && apt source nginx; \
|
||||
NGX_DIR=$$(find $(BUILD_DIR)/apt-src-asan -maxdepth 1 -type d -name 'nginx-*' | head -n1); \
|
||||
if [ -z "$$NGX_DIR" ]; then \
|
||||
echo "error: could not find unpacked nginx source tree" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
mv "$$NGX_DIR" $$NGX_SRC; \
|
||||
rm -rf $(BUILD_DIR)/apt-src-asan; \
|
||||
fi; \
|
||||
PREFIX="$(BUILD_DIR)/nginx-asan"; \
|
||||
echo "Configuring ASan nginx in $$NGX_SRC (prefix=$$PREFIX)"; \
|
||||
cd "$$NGX_SRC" && ./configure \
|
||||
--prefix="$$PREFIX" \
|
||||
--with-cc-opt="$(ASAN_CFLAGS)" \
|
||||
--with-ld-opt="$(ASAN_LDFLAGS)" \
|
||||
--with-compat \
|
||||
--with-debug \
|
||||
--add-dynamic-module=$(MODULE_DIR); \
|
||||
echo "Building ASan nginx + module (this takes a minute)"; \
|
||||
$(MAKE) -C "$$NGX_SRC" -f objs/Makefile build; \
|
||||
$(MAKE) -C "$$NGX_SRC" -f objs/Makefile install; \
|
||||
install -d $$PREFIX/modules; \
|
||||
install -m 0644 "$$NGX_SRC/objs/$(MODULE_NAME).so" $$PREFIX/modules/; \
|
||||
echo ""; \
|
||||
echo "ASan build ready:"; \
|
||||
echo " nginx: $$PREFIX/sbin/nginx"; \
|
||||
echo " module: $$PREFIX/modules/$(MODULE_NAME).so"; \
|
||||
echo ""; \
|
||||
echo "Run with:"; \
|
||||
echo " ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:halt_on_error=1:detect_leaks=0 \\"; \
|
||||
echo " $$PREFIX/sbin/nginx -p $$PREFIX"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# pkg-deb: build a .deb
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -146,10 +222,6 @@ tests/.venv: tests/requirements.txt
|
||||
tests/.venv/bin/pip install -q -r tests/requirements.txt
|
||||
|
||||
robot-test: tests/.venv
|
||||
@if [ ! -f $(BUILD_DIR)/libnginx-mod-http-ipng-stats_*.deb ]; then \
|
||||
echo "error: no .deb found in $(BUILD_DIR)/. Run 'make pkg-deb' first." >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
tests/rf-run.sh docker $(TEST)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user