Initial revisin of healthchecker, inspired by HAProxy

2026-04-10 17:30:44 +02:00
commit b84b3274b1
24 changed files with 4400 additions and 0 deletions
--- a/internal/prober/http.go
+++ b/internal/prober/http.go
@@ -0,0 +1,156 @@
+package prober
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"git.ipng.ch/ipng/vpp-maglev/internal/health"
+)
+
+// HTTPProbe sends a plain HTTP GET to cfg.Target inside the healthcheck netns.
+func HTTPProbe(ctx context.Context, cfg ProbeConfig) health.ProbeResult {
+	return doHTTPProbe(ctx, cfg, false)
+}
+
+// HTTPSProbe sends an HTTP GET over TLS to cfg.Target inside the healthcheck netns.
+func HTTPSProbe(ctx context.Context, cfg ProbeConfig) health.ProbeResult {
+	return doHTTPProbe(ctx, cfg, true)
+}
+
+func doHTTPProbe(ctx context.Context, cfg ProbeConfig, useTLS bool) health.ProbeResult {
+	if cfg.HTTP == nil {
+		return health.ProbeResult{OK: false, Layer: health.LayerUnknown, Code: "UNKNOWN", Detail: "missing HTTP params"}
+	}
+	p := cfg.HTTP
+
+	port := cfg.Port
+	if port == 0 {
+		if useTLS {
+			port = 443
+		} else {
+			port = 80
+		}
+	}
+
+	scheme := "http"
+	if useTLS {
+		scheme = "https"
+	}
+	target := fmt.Sprintf("%s://%s%s", scheme, net.JoinHostPort(cfg.Target.String(), strconv.Itoa(int(port))), p.Path)
+
+	hostHeader := p.Host
+	if hostHeader == "" {
+		hostHeader = cfg.Target.String()
+	}
+
+	// Dial (and optionally handshake) inside the healthcheck netns.
+	// The socket retains its netns after creation, so HTTP can be done outside.
+	var conn net.Conn
+	dialErr := inNetns(cfg.HealthCheckNetns, func() error {
+		dialer := &net.Dialer{Timeout: cfg.Timeout}
+		if cfg.ProbeSrc != nil {
+			dialer.LocalAddr = &net.TCPAddr{IP: cfg.ProbeSrc}
+		}
+		c, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(cfg.Target.String(), strconv.Itoa(int(port))))
+		if err != nil {
+			return err
+		}
+		if useTLS {
+			tlsConn := tls.Client(c, tlsConfig(p.ServerName, p.InsecureSkipVerify))
+			if err := tlsConn.HandshakeContext(ctx); err != nil {
+				c.Close()
+				return err
+			}
+			conn = tlsConn
+		} else {
+			conn = c
+		}
+		return nil
+	})
+	if dialErr != nil {
+		if isTimeout(dialErr) {
+			return health.ProbeResult{OK: false, Layer: health.LayerL4, Code: "L4TOUT", Detail: dialErr.Error()}
+		}
+		// Distinguish TLS handshake failures (L6) from TCP connect failures (L4).
+		// conn is non-nil only when TCP succeeded but TLS handshake failed.
+		if useTLS && conn == nil && isTLSError(dialErr) {
+			if isTimeout(dialErr) {
+				return health.ProbeResult{OK: false, Layer: health.LayerL6, Code: "L6TOUT", Detail: dialErr.Error()}
+			}
+			return health.ProbeResult{OK: false, Layer: health.LayerL6, Code: "L6RSP", Detail: dialErr.Error()}
+		}
+		return health.ProbeResult{OK: false, Layer: health.LayerL4, Code: "L4CON", Detail: dialErr.Error()}
+	}
+	defer conn.Close()
+
+	transport := &http.Transport{
+		DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+			return conn, nil
+		},
+		DisableKeepAlives: true,
+	}
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   cfg.Timeout,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse // never follow redirects
+		},
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+	if err != nil {
+		return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7RSP", Detail: err.Error()}
+	}
+	req.Host = hostHeader
+	req.Header.Set("User-Agent", "maglev-healthchecker/1.0")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		if isTimeout(err) {
+			return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7TOUT", Detail: err.Error()}
+		}
+		return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7RSP", Detail: err.Error()}
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < p.ResponseCodeMin || resp.StatusCode > p.ResponseCodeMax {
+		return health.ProbeResult{
+			OK:     false,
+			Layer:  health.LayerL7,
+			Code:   "L7STS",
+			Detail: fmt.Sprintf("HTTP %d (want %d-%d)", resp.StatusCode, p.ResponseCodeMin, p.ResponseCodeMax),
+		}
+	}
+
+	if p.ResponseRegexp != nil {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7TOUT", Detail: err.Error()}
+		}
+		if !p.ResponseRegexp.Match(body) {
+			return health.ProbeResult{
+				OK:     false,
+				Layer:  health.LayerL7,
+				Code:   "L7RSP",
+				Detail: fmt.Sprintf("body did not match regexp %q", p.ResponseRegexp),
+			}
+		}
+	}
+
+	return health.ProbeResult{OK: true, Layer: health.LayerL7, Code: "L7OK"}
+}
+
+// isTLSError returns true if err originated from the TLS layer.
+func isTLSError(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, ok := err.(tls.AlertError)
+	return ok || strings.Contains(err.Error(), "tls:")
+}
--- a/internal/prober/http_test.go
+++ b/internal/prober/http_test.go
@@ -0,0 +1,193 @@
+package prober
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+	"time"
+
+	"git.ipng.ch/ipng/vpp-maglev/internal/config"
+)
+
+// dialAndProbe dials addr directly (bypassing netns/interface binding) and
+// exercises the HTTP probe response-checking logic.
+func dialAndProbe(ctx context.Context, addr string, cfg ProbeConfig) (bool, error) {
+	if cfg.HTTP == nil {
+		return false, fmt.Errorf("dialAndProbe requires HTTP params")
+	}
+	p := cfg.HTTP
+
+	path := p.Path
+	if path == "" {
+		path = "/"
+	}
+
+	conn, err := net.DialTimeout("tcp", addr, cfg.Timeout)
+	if err != nil {
+		return false, err
+	}
+
+	transport := &http.Transport{
+		DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+			return conn, nil
+		},
+		DisableKeepAlives: true,
+	}
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   cfg.Timeout,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	target := "http://" + addr + path
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+	if err != nil {
+		return false, err
+	}
+	if p.Host != "" {
+		req.Host = p.Host
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < p.ResponseCodeMin || resp.StatusCode > p.ResponseCodeMax {
+		return false, nil
+	}
+	if p.ResponseRegexp != nil {
+		body, _ := io.ReadAll(resp.Body)
+		if !p.ResponseRegexp.Match(body) {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func TestHTTPProbeStatusCode(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "healthy")
+	}))
+	defer srv.Close()
+
+	cfg := ProbeConfig{
+		Timeout: 2 * time.Second,
+		HTTP: &config.HTTPParams{
+			Path:            "/healthz",
+			ResponseCodeMin: 200,
+			ResponseCodeMax: 200,
+		},
+	}
+	ok, err := dialAndProbe(context.Background(), srv.Listener.Addr().String(), cfg)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !ok {
+		t.Error("expected probe success")
+	}
+}
+
+func TestHTTPProbeWrongStatusCode(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}))
+	defer srv.Close()
+
+	cfg := ProbeConfig{
+		Timeout: 2 * time.Second,
+		HTTP: &config.HTTPParams{
+			Path:            "/",
+			ResponseCodeMin: 200,
+			ResponseCodeMax: 200,
+		},
+	}
+	ok, err := dialAndProbe(context.Background(), srv.Listener.Addr().String(), cfg)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if ok {
+		t.Error("expected probe failure on wrong status code")
+	}
+}
+
+func TestHTTPProbeRegexpMatch(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		fmt.Fprint(w, `{"status":"ok"}`)
+	}))
+	defer srv.Close()
+
+	cfg := ProbeConfig{
+		Timeout: 2 * time.Second,
+		HTTP: &config.HTTPParams{
+			Path:            "/",
+			ResponseCodeMin: 200,
+			ResponseCodeMax: 200,
+			ResponseRegexp:  regexp.MustCompile(`"status":"ok"`),
+		},
+	}
+	ok, err := dialAndProbe(context.Background(), srv.Listener.Addr().String(), cfg)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !ok {
+		t.Error("expected probe success with matching regexp")
+	}
+}
+
+func TestHTTPProbeRegexpNoMatch(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		fmt.Fprint(w, `{"status":"degraded"}`)
+	}))
+	defer srv.Close()
+
+	cfg := ProbeConfig{
+		Timeout: 2 * time.Second,
+		HTTP: &config.HTTPParams{
+			Path:            "/",
+			ResponseCodeMin: 200,
+			ResponseCodeMax: 200,
+			ResponseRegexp:  regexp.MustCompile(`"status":"ok"`),
+		},
+	}
+	ok, err := dialAndProbe(context.Background(), srv.Listener.Addr().String(), cfg)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if ok {
+		t.Error("expected probe failure when regexp does not match")
+	}
+}
+
+func TestHTTPProbeNoRedirect(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/other", http.StatusFound)
+	}))
+	defer srv.Close()
+
+	// Probe expects 302 — redirect is not followed.
+	cfg := ProbeConfig{
+		Timeout: 2 * time.Second,
+		HTTP: &config.HTTPParams{
+			Path:            "/",
+			ResponseCodeMin: 302,
+			ResponseCodeMax: 302,
+		},
+	}
+	ok, err := dialAndProbe(context.Background(), srv.Listener.Addr().String(), cfg)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !ok {
+		t.Error("expected success when probe expects 302 and server returns 302")
+	}
+}
--- a/internal/prober/icmp.go
+++ b/internal/prober/icmp.go
@@ -0,0 +1,118 @@
+package prober
+
+import (
+	"context"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"time"
+
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"git.ipng.ch/ipng/vpp-maglev/internal/health"
+)
+
+// ICMPProbe sends an ICMP echo request to cfg.Target inside the healthcheck
+// netns and waits for a matching reply.
+func ICMPProbe(ctx context.Context, cfg ProbeConfig) health.ProbeResult {
+	isV6 := cfg.Target.To4() == nil
+
+	var ok bool
+	err := inNetns(cfg.HealthCheckNetns, func() error {
+		var network string
+		var proto int
+		var msgType icmp.Type
+		if isV6 {
+			network = "ip6:ipv6-icmp"
+			proto = 58
+			msgType = ipv6.ICMPTypeEchoRequest
+		} else {
+			network = "ip4:icmp"
+			proto = 1
+			msgType = ipv4.ICMPTypeEcho
+		}
+
+		listenAddr := ""
+		if cfg.ProbeSrc != nil {
+			listenAddr = cfg.ProbeSrc.String()
+		}
+		pc, err := net.ListenPacket(network, listenAddr)
+		if err != nil {
+			return fmt.Errorf("listen icmp (%s): %w", network, err)
+		}
+		defer pc.Close()
+
+		id := rand.IntN(0xffff) + 1
+		seq := rand.IntN(0xffff) + 1
+
+		msg := icmp.Message{
+			Type: msgType,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   id,
+				Seq:  seq,
+				Data: []byte("maglev-hc"),
+			},
+		}
+		b, err := msg.Marshal(nil)
+		if err != nil {
+			return fmt.Errorf("marshal icmp: %w", err)
+		}
+
+		dst := &net.IPAddr{IP: cfg.Target}
+		if _, err := pc.WriteTo(b, dst); err != nil {
+			return fmt.Errorf("write icmp: %w", err)
+		}
+
+		deadline := time.Now().Add(cfg.Timeout)
+		pc.SetDeadline(deadline) //nolint:errcheck
+
+		buf := make([]byte, 1500)
+		for time.Now().Before(deadline) {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			default:
+			}
+
+			n, _, err := pc.ReadFrom(buf)
+			if err != nil {
+				if isTimeout(err) {
+					return nil
+				}
+				return fmt.Errorf("read icmp: %w", err)
+			}
+
+			reply, err := icmp.ParseMessage(proto, buf[:n])
+			if err != nil {
+				continue
+			}
+			echo, ok2 := reply.Body.(*icmp.Echo)
+			if !ok2 {
+				continue
+			}
+			if echo.ID == id && echo.Seq == seq {
+				ok = true
+				return nil
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7TOUT", Detail: err.Error()}
+	}
+	if ok {
+		return health.ProbeResult{OK: true, Layer: health.LayerL7, Code: "L7OK"}
+	}
+	return health.ProbeResult{OK: false, Layer: health.LayerL7, Code: "L7TOUT", Detail: "no reply received"}
+}
+
+func isTimeout(err error) bool {
+	if netErr, ok := err.(net.Error); ok {
+		return netErr.Timeout()
+	}
+	return false
+}
--- a/internal/prober/netns.go
+++ b/internal/prober/netns.go
@@ -0,0 +1,40 @@
+package prober
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/vishvananda/netns"
+)
+
+// inNetns runs fn while the current OS thread is switched into the named
+// network namespace. The thread is locked for the duration so the switch is safe.
+// The original netns is restored before returning.
+// If nsName is empty, fn is run in the current namespace without any switching.
+func inNetns(nsName string, fn func() error) error {
+	if nsName == "" {
+		return fn()
+	}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	origNs, err := netns.Get()
+	if err != nil {
+		return fmt.Errorf("get current netns: %w", err)
+	}
+	defer origNs.Close()
+	defer netns.Set(origNs) //nolint:errcheck
+
+	targetNs, err := netns.GetFromName(nsName)
+	if err != nil {
+		return fmt.Errorf("get netns %q: %w", nsName, err)
+	}
+	defer targetNs.Close()
+
+	if err := netns.Set(targetNs); err != nil {
+		return fmt.Errorf("enter netns %q: %w", nsName, err)
+	}
+
+	return fn()
+}
--- a/internal/prober/prober.go
+++ b/internal/prober/prober.go
@@ -0,0 +1,49 @@
+package prober
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"git.ipng.ch/ipng/vpp-maglev/internal/config"
+	"git.ipng.ch/ipng/vpp-maglev/internal/health"
+)
+
+// ProbeFunc is the signature for a health check probe.
+type ProbeFunc func(ctx context.Context, cfg ProbeConfig) health.ProbeResult
+
+// ProbeConfig holds all parameters needed to execute a single probe.
+type ProbeConfig struct {
+	Target           net.IP        // backend address to probe
+	Port             uint16        // destination port (used by TCP and HTTP probers)
+	ProbeSrc         net.IP        // source address to bind; nil lets the OS choose
+	HealthCheckNetns string        // network namespace name; sockets are created inside it
+	Timeout          time.Duration
+	HTTP             *config.HTTPParams // non-nil for type http/https
+	TCP              *config.TCPParams  // non-nil for type tcp
+}
+
+// ForType returns the ProbeFunc registered for the given healthcheck type.
+// Returns a failing stub for unknown types.
+func ForType(t string) ProbeFunc {
+	switch t {
+	case "icmp":
+		return ICMPProbe
+	case "tcp":
+		return TCPProbe
+	case "http":
+		return HTTPProbe
+	case "https":
+		return HTTPSProbe
+	default:
+		return func(_ context.Context, _ ProbeConfig) health.ProbeResult {
+			return health.ProbeResult{
+				OK:     false,
+				Layer:  health.LayerUnknown,
+				Code:   "UNKNOWN",
+				Detail: fmt.Sprintf("unknown probe type %q", t),
+			}
+		}
+	}
+}
--- a/internal/prober/tcp.go
+++ b/internal/prober/tcp.go
@@ -0,0 +1,76 @@
+package prober
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"strconv"
+	"time"
+
+	"git.ipng.ch/ipng/vpp-maglev/internal/health"
+)
+
+// TCPProbe performs a TCP connect to cfg.Target:cfg.Port inside the healthcheck
+// netns. If cfg.TCP.SSL is true a TLS handshake is performed after connect,
+// making this an L6 check (useful for smtps, imaps, etc.).
+// Plain connect returns L4OK/L4TOUT/L4CON.
+// TLS handshake returns L6OK/L6TOUT/L6RSP (on top of an L4OK connect).
+func TCPProbe(ctx context.Context, cfg ProbeConfig) health.ProbeResult {
+	port := cfg.Port
+	if port == 0 {
+		port = 80
+	}
+	addr := net.JoinHostPort(cfg.Target.String(), strconv.Itoa(int(port)))
+
+	doTLS := cfg.TCP != nil && cfg.TCP.SSL
+	var serverName string
+	var insecureSkipVerify bool
+	if cfg.TCP != nil {
+		serverName = cfg.TCP.ServerName
+		insecureSkipVerify = cfg.TCP.InsecureSkipVerify
+	}
+
+	var result health.ProbeResult
+	err := inNetns(cfg.HealthCheckNetns, func() error {
+		dialer := &net.Dialer{Timeout: cfg.Timeout}
+		if cfg.ProbeSrc != nil {
+			dialer.LocalAddr = &net.TCPAddr{IP: cfg.ProbeSrc}
+		}
+		conn, err := dialer.DialContext(ctx, "tcp", addr)
+		if err != nil {
+			if isTimeout(err) {
+				result = health.ProbeResult{OK: false, Layer: health.LayerL4, Code: "L4TOUT", Detail: err.Error()}
+			} else {
+				result = health.ProbeResult{OK: false, Layer: health.LayerL4, Code: "L4CON", Detail: err.Error()}
+			}
+			return nil
+		}
+
+		if !doTLS {
+			conn.Close()
+			result = health.ProbeResult{OK: true, Layer: health.LayerL4, Code: "L4OK"}
+			return nil
+		}
+
+		// TLS handshake.
+		tlsConn := tls.Client(conn, tlsConfig(serverName, insecureSkipVerify))
+		tlsConn.SetDeadline(time.Now().Add(cfg.Timeout)) //nolint:errcheck
+		if err := tlsConn.HandshakeContext(ctx); err != nil {
+			tlsConn.Close()
+			if isTimeout(err) {
+				result = health.ProbeResult{OK: false, Layer: health.LayerL6, Code: "L6TOUT", Detail: err.Error()}
+			} else {
+				result = health.ProbeResult{OK: false, Layer: health.LayerL6, Code: "L6RSP", Detail: err.Error()}
+			}
+			return nil
+		}
+		tlsConn.Close()
+		result = health.ProbeResult{OK: true, Layer: health.LayerL6, Code: "L6OK"}
+		return nil
+	})
+
+	if err != nil {
+		return health.ProbeResult{OK: false, Layer: health.LayerL4, Code: "L4CON", Detail: err.Error()}
+	}
+	return result
+}
--- a/internal/prober/tls.go
+++ b/internal/prober/tls.go
@@ -0,0 +1,13 @@
+package prober
+
+import (
+	"crypto/tls"
+)
+
+// tlsConfig builds a *tls.Config from explicit TLS parameters.
+func tlsConfig(serverName string, insecureSkipVerify bool) *tls.Config {
+	return &tls.Config{
+		ServerName:         serverName,
+		InsecureSkipVerify: insecureSkipVerify, //nolint:gosec
+	}
+}