Pim van Pelt
450391af6b
SO_BINDTODEVICE pins both ingress *and* egress to the bound interface — the kernel uses the listening socket's device binding when choosing the output interface for the SYN-ACK, which is sent before accept() returns and therefore can't be fixed up in userspace. That's fatal for maglev / DSR deployments where the SYN arrives through a GRE tunnel but the return path has to leave via the default route; the SYN-ACK goes out the GRE and is dropped by the uplink, so every new connection times out. Rework the listen plumbing so the module never touches SO_BINDTODEVICE. init_module now enables IP_PKTINFO and IPV6_RECVPKTINFO on every HTTP listening socket and resolves each configured `device=` name to an ifindex. At request time resolve_source calls getsockopt(IP_PKTOPTIONS) on the accepted fd to read the per-connection in(6)_pktinfo cmsg the kernel stashed during the handshake, then matches (ifindex, family) against the bindings table. The listening sockets remain plain wildcards, so the return path follows the normal routing table and DSR works. The wrapper also no longer clones or rebinds sockets: it still dedups per (cscf, sockaddr) so multiple device-tagged listens in a single server block coexist, and dedups bindings on (device, family) so the same device can carry different tags for v4 and v6 (e.g. tag2-v4 / tag2-v6) but not pointlessly duplicate when a listen include is shared across server blocks. Drive-by fixes to unblock `make pkg-deb` after a prior `make build-asan`: - debian/rules overrides dh_clean to exclude build/, since nginx-asan's install creates nobody:0700 temp dirs dh_clean can't traverse. - Makefile's build-asan removes those unused runtime temp dirs so the tree is clean afterwards. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
253 lines
10 KiB
Makefile
253 lines
10 KiB
Makefile
# SPDX-License-Identifier: Apache-2.0
|
|
# Makefile for nginx-ipng-stats-plugin
|
|
#
|
|
# Targets:
|
|
# build - build ngx_http_ipng_stats_module.so out-of-tree.
|
|
# pkg-deb - build a .deb via dpkg-buildpackage for the current release.
|
|
# robot-test - build .deb, then run Robot Framework end-to-end tests
|
|
# in containerlab (requires docker + containerlab).
|
|
# install-deps - install build and test dependencies via apt.
|
|
# clean - remove build artifacts and the fetched nginx source tree.
|
|
# help - print this help.
|
|
#
|
|
# Overridable variables:
|
|
# NGINX_SRC - path to an unpacked nginx source tree. If unset, the
|
|
# `build` target will apt-source one into ./build/nginx-src.
|
|
|
|
MODULE_NAME := ngx_http_ipng_stats_module
|
|
MODULE_DIR := $(CURDIR)
|
|
BUILD_DIR := $(CURDIR)/build
|
|
|
|
# Single source of truth for the module version. When cutting a release,
|
|
# bump this AND add a matching top entry to debian/changelog — dpkg reads
|
|
# the package version from there directly. The C code picks up VERSION
|
|
# via the generated src/version.h (written by the version-header target
|
|
# below and depended on by the module build).
|
|
VERSION := 0.4.0
|
|
|
|
NGINX_SRC ?=
|
|
|
|
.PHONY: help build build-asan pkg-deb robot-test install-deps clean fetch-nginx-src version-header
|
|
|
|
TEST ?= tests/
|
|
|
|
help:
|
|
@echo "nginx-ipng-stats-plugin — make targets"
|
|
@echo ""
|
|
@echo " make build Build $(MODULE_NAME).so out-of-tree."
|
|
@echo " make build-asan Build a full nginx + module with AddressSanitizer"
|
|
@echo " into build/nginx-asan/ for local crash-hunting."
|
|
@echo " make pkg-deb Build a Debian package via dpkg-buildpackage."
|
|
@echo " make robot-test Run Robot Framework e2e tests (all suites)."
|
|
@echo " make install-deps Install build and test dependencies (apt)."
|
|
@echo " make clean Remove build artifacts."
|
|
@echo ""
|
|
@echo "Overridable:"
|
|
@echo " NGINX_SRC=<path> Use an existing nginx source tree."
|
|
@echo " TEST=<path> Run a specific .robot file (default: tests/)."
|
|
|
|
# ----------------------------------------------------------------------
|
|
# build: out-of-tree dynamic module build
|
|
# ----------------------------------------------------------------------
|
|
|
|
build: $(BUILD_DIR)/$(MODULE_NAME).so
|
|
@echo ""
|
|
@echo "Built: $(BUILD_DIR)/$(MODULE_NAME).so"
|
|
@echo ""
|
|
@echo "To try it locally without installing a .deb:"
|
|
@echo " sudo install -m 0644 $(BUILD_DIR)/$(MODULE_NAME).so /usr/lib/nginx/modules/"
|
|
@echo " echo 'load_module modules/$(MODULE_NAME).so;' | sudo tee /etc/nginx/modules-enabled/50-mod-http-ipng-stats.conf"
|
|
@echo " sudo nginx -t && sudo nginx -s reload"
|
|
|
|
# version-header: write src/version.h iff its contents would change. The
|
|
# target is .PHONY so it's re-evaluated every build, but the file itself
|
|
# is only touched on VERSION bumps — keeps the .so from rebuilding when
|
|
# nothing has actually changed.
|
|
version-header:
|
|
@NEW='#define NGX_HTTP_IPNG_STATS_VERSION "$(VERSION)"'; \
|
|
if [ ! -f $(MODULE_DIR)/src/version.h ] \
|
|
|| [ "$$NEW" != "$$(cat $(MODULE_DIR)/src/version.h)" ]; then \
|
|
echo "Generating src/version.h (VERSION=$(VERSION))"; \
|
|
echo "$$NEW" > $(MODULE_DIR)/src/version.h; \
|
|
fi
|
|
|
|
$(BUILD_DIR)/$(MODULE_NAME).so: version-header fetch-nginx-src
|
|
@set -e; \
|
|
if [ -z "$(NGINX_SRC)" ]; then \
|
|
NGX_SRC="$(BUILD_DIR)/nginx-src"; \
|
|
else \
|
|
NGX_SRC="$(NGINX_SRC)"; \
|
|
fi; \
|
|
echo "Configuring nginx in $$NGX_SRC against module at $(MODULE_DIR)"; \
|
|
cd "$$NGX_SRC" && ./configure --with-compat --add-dynamic-module=$(MODULE_DIR); \
|
|
echo "Building module"; \
|
|
$(MAKE) -C "$$NGX_SRC" -f objs/Makefile modules; \
|
|
mkdir -p $(BUILD_DIR); \
|
|
cp "$$NGX_SRC/objs/$(MODULE_NAME).so" $(BUILD_DIR)/$(MODULE_NAME).so
|
|
|
|
fetch-nginx-src:
|
|
@set -e; \
|
|
if [ -n "$(NGINX_SRC)" ]; then \
|
|
echo "Using NGINX_SRC=$(NGINX_SRC)"; \
|
|
exit 0; \
|
|
fi; \
|
|
if [ -d "$(BUILD_DIR)/nginx-src" ] && [ -f "$(BUILD_DIR)/nginx-src/configure" ]; then \
|
|
echo "Reusing $(BUILD_DIR)/nginx-src"; \
|
|
exit 0; \
|
|
fi; \
|
|
mkdir -p $(BUILD_DIR); \
|
|
if [ -d /usr/share/nginx/src ] && [ -f /usr/share/nginx/src/configure ]; then \
|
|
echo "Copying /usr/share/nginx/src (from nginx-dev) to $(BUILD_DIR)/nginx-src"; \
|
|
rm -rf $(BUILD_DIR)/nginx-src; \
|
|
cp -a /usr/share/nginx/src $(BUILD_DIR)/nginx-src; \
|
|
chmod -R u+w $(BUILD_DIR)/nginx-src; \
|
|
exit 0; \
|
|
fi; \
|
|
rm -rf $(BUILD_DIR)/apt-src; \
|
|
mkdir -p $(BUILD_DIR)/apt-src; \
|
|
echo "Fetching nginx source via \`apt source nginx\` in $(BUILD_DIR)/apt-src"; \
|
|
cd $(BUILD_DIR)/apt-src && apt source nginx; \
|
|
NGX_DIR=$$(find $(BUILD_DIR)/apt-src -maxdepth 1 -type d -name 'nginx-*' | head -n1); \
|
|
if [ -z "$$NGX_DIR" ]; then \
|
|
echo "error: could not find unpacked nginx source tree under $(BUILD_DIR)/apt-src" >&2; \
|
|
exit 1; \
|
|
fi; \
|
|
rm -rf $(BUILD_DIR)/nginx-src; \
|
|
mv "$$NGX_DIR" $(BUILD_DIR)/nginx-src; \
|
|
rm -rf $(BUILD_DIR)/apt-src
|
|
|
|
# ----------------------------------------------------------------------
|
|
# build-asan: full nginx + module built with AddressSanitizer. ASan
|
|
# needs to be present in the main binary to instrument dynamic modules;
|
|
# running a stock nginx with LD_PRELOAD=libasan.so does NOT work with
|
|
# a statically-linked module's allocations. So we build nginx itself
|
|
# here, into an isolated prefix under build/nginx-asan/, which you can
|
|
# run by hand against any config. Normal `make build` is unaffected.
|
|
#
|
|
# Typical workflow for the reload-crash hunt:
|
|
# make build-asan
|
|
# cp your-test.conf build/nginx-asan/conf/nginx.conf
|
|
# ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:halt_on_error=1:detect_leaks=0 \
|
|
# build/nginx-asan/sbin/nginx -p build/nginx-asan
|
|
# # in another shell:
|
|
# build/nginx-asan/sbin/nginx -p build/nginx-asan -s reload
|
|
# ASan writes any findings to stderr of the master before aborting.
|
|
#
|
|
# detect_odr_violation=0 suppresses false positives for symbols nginx
|
|
# intentionally duplicates between its main binary and dynamic modules
|
|
# (e.g. ngx_module_names in objs/ngx_http_ipng_stats_module_modules.c).
|
|
# ----------------------------------------------------------------------
|
|
|
|
# -fno-sanitize=nonnull-attribute suppresses a UBSan finding in nginx's
|
|
# ngx_cpymem (src/core/ngx_string.c:84): the macro expands to memcpy(...)
|
|
# which glibc declares _Nonnull, but nginx legitimately calls it with
|
|
# (dst, NULL, 0) in many places. Filtering just that check keeps real
|
|
# undefined behaviour visible.
|
|
ASAN_CFLAGS := -fsanitize=address -fsanitize=undefined -fno-sanitize=nonnull-attribute -fno-omit-frame-pointer -fno-common -g -O1
|
|
ASAN_LDFLAGS := -fsanitize=address -fsanitize=undefined
|
|
|
|
# The ASan build rebuilds nginx from source rather than only the module,
|
|
# because ASan must instrument the main binary to catch heap issues in
|
|
# modules it loads. The `nginx-dev` shim source tree under /usr/share
|
|
# only carries headers, so we always fall through to `apt source nginx`
|
|
# here and keep the full source tree isolated in build/nginx-asan-src/.
|
|
build-asan: version-header
|
|
@set -e; \
|
|
NGX_SRC="$(BUILD_DIR)/nginx-asan-src"; \
|
|
if [ ! -d "$$NGX_SRC" ] || [ ! -f "$$NGX_SRC/src/core/nginx.c" ]; then \
|
|
echo "Fetching full nginx source via apt-source for ASan build"; \
|
|
mkdir -p $(BUILD_DIR)/apt-src-asan; \
|
|
rm -rf $$NGX_SRC; \
|
|
cd $(BUILD_DIR)/apt-src-asan && apt source nginx; \
|
|
NGX_DIR=$$(find $(BUILD_DIR)/apt-src-asan -maxdepth 1 -type d -name 'nginx-*' | head -n1); \
|
|
if [ -z "$$NGX_DIR" ]; then \
|
|
echo "error: could not find unpacked nginx source tree" >&2; \
|
|
exit 1; \
|
|
fi; \
|
|
mv "$$NGX_DIR" $$NGX_SRC; \
|
|
rm -rf $(BUILD_DIR)/apt-src-asan; \
|
|
fi; \
|
|
PREFIX="$(BUILD_DIR)/nginx-asan"; \
|
|
echo "Configuring ASan nginx in $$NGX_SRC (prefix=$$PREFIX)"; \
|
|
cd "$$NGX_SRC" && ./configure \
|
|
--prefix="$$PREFIX" \
|
|
--with-cc-opt="$(ASAN_CFLAGS)" \
|
|
--with-ld-opt="$(ASAN_LDFLAGS)" \
|
|
--with-compat \
|
|
--with-debug \
|
|
--add-dynamic-module=$(MODULE_DIR); \
|
|
echo "Building ASan nginx + module (this takes a minute)"; \
|
|
$(MAKE) -C "$$NGX_SRC" -f objs/Makefile build; \
|
|
$(MAKE) -C "$$NGX_SRC" -f objs/Makefile install; \
|
|
install -d $$PREFIX/modules; \
|
|
install -m 0644 "$$NGX_SRC/objs/$(MODULE_NAME).so" $$PREFIX/modules/; \
|
|
for d in client_body_temp fastcgi_temp proxy_temp scgi_temp uwsgi_temp; do \
|
|
rm -rf "$$PREFIX/$$d"; \
|
|
done; \
|
|
echo ""; \
|
|
echo "ASan build ready:"; \
|
|
echo " nginx: $$PREFIX/sbin/nginx"; \
|
|
echo " module: $$PREFIX/modules/$(MODULE_NAME).so"; \
|
|
echo ""; \
|
|
echo "Run with:"; \
|
|
echo " ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:halt_on_error=1:detect_leaks=0 \\"; \
|
|
echo " $$PREFIX/sbin/nginx -p $$PREFIX"
|
|
|
|
# ----------------------------------------------------------------------
|
|
# pkg-deb: build a .deb
|
|
# ----------------------------------------------------------------------
|
|
|
|
pkg-deb:
|
|
dpkg-buildpackage -us -uc -b
|
|
@mkdir -p $(BUILD_DIR)
|
|
@# dpkg-buildpackage writes artifacts to ../ — relocate them into
|
|
@# $(BUILD_DIR) so everything ephemeral lives under build/.
|
|
@for f in ../libnginx-mod-http-ipng-stats*.deb \
|
|
../libnginx-mod-http-ipng-stats*.ddeb \
|
|
../nginx-ipng-stats-plugin_*.buildinfo \
|
|
../nginx-ipng-stats-plugin_*.changes; do \
|
|
if [ -f "$$f" ]; then mv -f "$$f" $(BUILD_DIR)/; fi; \
|
|
done
|
|
@echo ""
|
|
@echo "Resulting .deb(s):"
|
|
@ls -1 $(BUILD_DIR)/*.deb 2>/dev/null || true
|
|
|
|
# ----------------------------------------------------------------------
|
|
# clean
|
|
# ----------------------------------------------------------------------
|
|
|
|
# ----------------------------------------------------------------------
|
|
# robot-test: containerlab + Robot Framework end-to-end tests
|
|
# ----------------------------------------------------------------------
|
|
|
|
tests/.venv: tests/requirements.txt
|
|
python3 -m venv tests/.venv
|
|
tests/.venv/bin/pip install -q -r tests/requirements.txt
|
|
|
|
robot-test: tests/.venv
|
|
tests/rf-run.sh docker $(TEST)
|
|
|
|
# ----------------------------------------------------------------------
|
|
# install-deps: install build and test dependencies
|
|
# ----------------------------------------------------------------------
|
|
|
|
install-deps:
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y \
|
|
nginx-dev dpkg-dev debhelper \
|
|
python3 python3-venv \
|
|
curl
|
|
@echo ""
|
|
@echo "Build dependencies installed. For 'make robot-test' you also need:"
|
|
@echo " - docker: https://docs.docker.com/engine/install/debian/"
|
|
@echo " - containerlab: https://containerlab.dev/install/"
|
|
|
|
# ----------------------------------------------------------------------
|
|
# clean
|
|
# ----------------------------------------------------------------------
|
|
|
|
clean:
|
|
rm -rf $(BUILD_DIR) tests/.venv tests/out
|
|
rm -f $(MODULE_DIR)/src/version.h
|
|
-dh_clean 2>/dev/null || true
|