Three reliability fixes bundled with docs updates. Restart-neutral VPP LB sync via a startup warmup window (internal/vpp/warmup.go). Before this, a maglevd restart would immediately issue SyncLBStateAll with every backend still in StateUnknown — mapped through BackendEffectiveWeight to weight 0 — and VPP would black-hole all new flows until the checker's rise counters caught up, several seconds later. The new warmup tracker owns a process-wide state machine gated by two config knobs: vpp.lb.startup-min-delay (default 5s) is an absolute hands-off window during which neither the periodic sync loop nor the per-transition reconciler touches VPP; vpp.lb. startup-max-delay (default 30s) is the watchdog for a per-VIP release phase that runs between the two, releasing each frontend as soon as every backend it references reaches a non-Unknown state. At max-delay a final SyncLBStateAll runs for any stragglers still in Unknown. Config reload does not reset the clock. Both delays can be set to 0 to disable the warmup entirely. The reconciler's suppressed-during-warmup events log at DEBUG so operators can still see them with --log-level debug. Unit tests cover the tracker state machine, allBackendsKnown precondition, and the zero-delay escape hatch. Deterministic AS iteration in VPP LB sync. reconcileVIP and recreateVIP now issue their lb_as_add_del / lb_as_set_weight calls in numeric IP order (IPv4 before IPv6, ascending within each family) via a new sortedIPKeys helper, instead of Go map iteration order. VPP's LB plugin breaks per-bucket ties in the Maglev lookup table by insertion position in its internal AS vec, so without a stable call order two maglevd instances on the same config could push identical AS sets into VPP in different orders and produce divergent new-flow tables. Numeric sort is used in preference to lexicographic so the sync log stays human-readable: string order would place 10.0.0.10 before 10.0.0.2, and the same problem in v6. Unit tests cover empty, single, v4/v6 numeric vs lexicographic, v4-before-v6 grouping, a 1000-iteration stability loop against Go's randomised map iteration, insertion-order invariance, and the desiredAS call-site type. maglevt interval fix. runProbeLoop used to sleep the full jittered interval after every probe, so a 100ms --interval with a 30ms probe actually produced a 130ms period. The sleep now subtracts result.Duration so cadence matches the flag. Probes that overrun clamp sleep to zero and fire the next probe immediately without trying to catch up on missed cycles — a slow backend doesn't get flooded with back-to-back probes at the moment it's already struggling. Docs. config-guide now documents flush-on-down and the new startup-min-delay / startup-max-delay knobs; user-guide's maglevd section explains the restart-neutrality property, the three warmup phases, and the relevant slog lines operators should watch for during a bounce. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
159 lines
5.1 KiB
Go
159 lines
5.1 KiB
Go
// Copyright (c) 2026, Pim van Pelt <pim@ipng.ch>
|
|
|
|
package vpp
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
|
|
"git.ipng.ch/ipng/vpp-maglev/internal/checker"
|
|
)
|
|
|
|
// EventSource is the subset of checker.Checker that Reconciler needs.
|
|
// Decoupling via an interface keeps the dependency direction
|
|
// vpp → checker (checker never imports vpp).
|
|
type EventSource interface {
|
|
Subscribe() (<-chan checker.Event, func())
|
|
}
|
|
|
|
// Reconciler bridges checker state transitions to VPP dataplane changes.
|
|
// It subscribes to the checker's event channel and, for every transition,
|
|
// runs SyncLBStateVIP for the frontend the backend belongs to. This is
|
|
// the ONLY place in the codebase where backend state transitions cause
|
|
// VPP calls — every LB change flows through Client.SyncLBStateVIP.
|
|
//
|
|
// The reconciler carries no state of its own. Idempotency is guaranteed
|
|
// by SyncLBStateVIP itself (diff-based, driven by the pure asFromBackend
|
|
// mapping in lbsync.go).
|
|
type Reconciler struct {
|
|
client *Client
|
|
events EventSource
|
|
stateSrc StateSource
|
|
}
|
|
|
|
// NewReconciler creates a Reconciler. client is the VPP client, events is
|
|
// the checker (or anything that implements Subscribe), and stateSrc provides
|
|
// the live config for SyncLBStateVIP calls. All three are normally the
|
|
// checker/vpp client pair constructed at daemon startup.
|
|
func NewReconciler(client *Client, events EventSource, stateSrc StateSource) *Reconciler {
|
|
return &Reconciler{client: client, events: events, stateSrc: stateSrc}
|
|
}
|
|
|
|
// Run subscribes to the checker and loops until ctx is cancelled. Each
|
|
// received event fires a single-VIP sync for the frontend the transitioned
|
|
// backend belongs to.
|
|
func (r *Reconciler) Run(ctx context.Context) {
|
|
ch, unsub := r.events.Subscribe()
|
|
defer unsub()
|
|
|
|
slog.Info("vpp-reconciler-start")
|
|
defer slog.Info("vpp-reconciler-stop")
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case ev, ok := <-ch:
|
|
if !ok {
|
|
return
|
|
}
|
|
r.handle(ev)
|
|
}
|
|
}
|
|
}
|
|
|
|
// handle reconciles one event. Operates only on backend-transition events
|
|
// that carry a frontend name (the checker emits one event per frontend that
|
|
// references the backend, so a backend shared across multiple frontends
|
|
// produces multiple events and all relevant VIPs are reconciled).
|
|
// Frontend-transition events are observational only — the dataplane work
|
|
// they would imply has already been done by the backend-transition event
|
|
// that triggered them.
|
|
//
|
|
// The handler consults the VPP client's warmup tracker before doing any
|
|
// dataplane work. During the startup warmup window the reconciler is
|
|
// either fully suppressed (inside min-delay) or per-VIP gated (the
|
|
// frontend must have been released before events for it pass through).
|
|
// When a transition fires for a VIP that isn't yet released but whose
|
|
// backends have now all settled, the handler opportunistically releases
|
|
// it here so the per-VIP release fires on the event rather than waiting
|
|
// for the next warmup poll tick.
|
|
func (r *Reconciler) handle(ev checker.Event) {
|
|
if ev.FrontendTransition != nil {
|
|
return // frontend-only event; no dataplane work
|
|
}
|
|
if ev.FrontendName == "" {
|
|
return
|
|
}
|
|
cfg := r.stateSrc.Config()
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
|
|
w := r.client.warmup
|
|
feName := ev.FrontendName
|
|
if !w.isReleased(feName) {
|
|
// Warmup is still gating this frontend. Decide whether to
|
|
// release it now, or defer until a later event / the warmup
|
|
// poll / the final max-delay SyncLBStateAll.
|
|
if w.inMinDelay() {
|
|
slog.Debug("vpp-reconciler-suppressed-min-delay",
|
|
"frontend", feName,
|
|
"backend", ev.BackendName,
|
|
"from", ev.Transition.From.String(),
|
|
"to", ev.Transition.To.String(),
|
|
"elapsed", w.elapsed(),
|
|
"reason", "inside vpp.lb.startup-min-delay window")
|
|
return
|
|
}
|
|
fe, ok := cfg.Frontends[feName]
|
|
if !ok {
|
|
return
|
|
}
|
|
if !allBackendsKnown(fe, r.stateSrc) {
|
|
slog.Debug("vpp-reconciler-suppressed-warmup",
|
|
"frontend", feName,
|
|
"backend", ev.BackendName,
|
|
"from", ev.Transition.From.String(),
|
|
"to", ev.Transition.To.String(),
|
|
"elapsed", w.elapsed(),
|
|
"reason", "frontend has backends still in StateUnknown; "+
|
|
"waiting for all to settle or for max-delay watchdog")
|
|
return
|
|
}
|
|
if !w.tryRelease(feName) {
|
|
// Lost a race with finishAll or another release caller.
|
|
// Either way the next isReleased call will return true, but
|
|
// for this event we've already done the right thing by
|
|
// letting the next few lines re-check and proceed.
|
|
return
|
|
}
|
|
slog.Info("vpp-lb-warmup-release",
|
|
"frontend", feName,
|
|
"trigger", "reconciler-event",
|
|
"backend", ev.BackendName,
|
|
"elapsed", w.elapsed())
|
|
}
|
|
|
|
slog.Debug("vpp-reconciler-event",
|
|
"frontend", feName,
|
|
"backend", ev.BackendName,
|
|
"from", ev.Transition.From.String(),
|
|
"to", ev.Transition.To.String())
|
|
|
|
if err := r.client.SyncLBStateVIP(cfg, feName, ""); err != nil {
|
|
if errors.Is(err, ErrFrontendNotFound) {
|
|
// Frontend was removed between the event being emitted and
|
|
// us handling it; a periodic SyncLBStateAll will clean it up.
|
|
return
|
|
}
|
|
slog.Warn("vpp-reconciler-error",
|
|
"frontend", feName,
|
|
"backend", ev.BackendName,
|
|
"from", ev.Transition.From.String(),
|
|
"to", ev.Transition.To.String(),
|
|
"err", err)
|
|
}
|
|
}
|