vpp-maglev/internal/vpp/client.go

// SPDX-License-Identifier: Apache-2.0

// Package vpp manages the connection to a local VPP instance over its
// binary API and stats sockets. The Client reconnects automatically when
// VPP restarts.
package vpp

import (
	"context"
	"log/slog"
	"sync"
	"sync/atomic"
	"time"

	"go.fd.io/govpp/adapter"
	"go.fd.io/govpp/adapter/socketclient"
	"go.fd.io/govpp/adapter/statsclient"
	"go.fd.io/govpp/binapi/vpe"
	"go.fd.io/govpp/core"

	"git.ipng.ch/ipng/vpp-maglev/internal/config"
	"git.ipng.ch/ipng/vpp-maglev/internal/health"
	"git.ipng.ch/ipng/vpp-maglev/internal/metrics"
	lb "git.ipng.ch/ipng/vpp-maglev/internal/vpp/binapi/lb"
)

// StateSource provides a live view of the running config and the current
// health state of each backend. checker.Checker satisfies this interface via
// its Config() and BackendState() methods. Decoupling via an interface avoids
// an import cycle with the checker package.
type StateSource interface {
	Config() *config.Config
	BackendState(name string) (health.State, bool)
}

const retryInterval = 5 * time.Second
const pingInterval = 10 * time.Second
const defaultLBSyncInterval = 30 * time.Second

// Info holds VPP version and connection metadata, populated on connect.
type Info struct {
	Version        string
	BuildDate      string
	BuildDirectory string
	PID            uint32
	BootTime       time.Time // when VPP started (from /sys/boottime stats counter)
	ConnectedSince time.Time // when maglevd connected to VPP
}

// Client manages connections to both the VPP API and stats sockets.
// Both connections are treated as a unit: if either drops, both are
// torn down and re-established together.
type Client struct {
	apiAddr   string
	statsAddr string

	mu          sync.Mutex
	apiConn     *core.Connection
	statsConn   *core.StatsConnection
	statsClient adapter.StatsAPI // raw adapter for DumpStats
	info        Info             // populated on successful connect
	stateSrc    StateSource      // optional; enables periodic LB sync
	lastLBConf  *lb.LbConf       // cached last-pushed lb_conf (dedup)

	// lbStatsSnap is the most recent per-VIP stats snapshot captured by
	// lbStatsLoop. Published as an immutable slice via atomic.Pointer so
	// Prometheus scrapes (metrics.Collector.Collect) don't take any lock.
	lbStatsSnap atomic.Pointer[[]metrics.VIPStatEntry]

	// warmup gates every VPP LB sync call (both periodic and event-
	// driven) during the first StartupMaxDelay seconds after Client
	// construction. See warmup.go for the state machine. Process-wide,
	// not per-connection: reconnects do not re-enter warmup.
	warmup *warmupTracker
}

// SetStateSource attaches a live config + health state source. When set, the
// VPP client runs a periodic SyncLBStateAll loop (at the interval from
// cfg.VPP.LB.SyncInterval) for as long as the VPP connection is up, and
// state-aware weights are used throughout the sync path. Must be called
// before Run.
func (c *Client) SetStateSource(src StateSource) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.stateSrc = src
}

// getStateSource returns the registered state source under the mutex.
func (c *Client) getStateSource() StateSource {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.stateSrc
}

// New creates a Client for the given socket paths. The warmup tracker's
// clock starts here — the restart-neutrality window is measured from the
// moment the Client is constructed, which in practice is a few tens of
// milliseconds after process start (see cmd/server/main.go startup
// sequence). If main.go ever grows a long-running initialisation step
// before vpp.New(), the warmup clock should be moved accordingly.
func New(apiAddr, statsAddr string) *Client {
	return &Client{
		apiAddr:   apiAddr,
		statsAddr: statsAddr,
		warmup:    newWarmupTracker(),
	}
}

// Run connects to VPP and maintains the connection until ctx is cancelled.
// If VPP is unavailable or restarts, Run reconnects automatically.
func (c *Client) Run(ctx context.Context) {
	for {
		if err := c.connect(); err != nil {
			slog.Debug("vpp-connect-failed", "err", err)
			select {
			case <-ctx.Done():
				return
			case <-time.After(retryInterval):
				continue
			}
		}

		// Fetch version info and record connect time.
		// fetchInfo uses NewAPIChannel and statsClient which both take c.mu,
		// so we must not hold c.mu here.
		info := c.fetchInfo()
		c.mu.Lock()
		c.info = info
		c.mu.Unlock()
		slog.Info("vpp-connect", "version", c.info.Version,
			"build-date", c.info.BuildDate,
			"pid", c.info.PID,
			"api", c.apiAddr, "stats", c.statsAddr)

		// Read the current LB plugin state so we can log what's programmed.
		if state, err := c.GetLBStateAll(); err != nil {
			slog.Warn("vpp-lb-read-failed", "err", err)
		} else {
			totalAS := 0
			for _, v := range state.VIPs {
				totalAS += len(v.ASes)
			}
			slog.Info("vpp-lb-state",
				"vips", len(state.VIPs),
				"application-servers", totalAS,
				"sticky-buckets-per-core", state.Conf.StickyBucketsPerCore,
				"flow-timeout", state.Conf.FlowTimeout)
		}

		// Push global LB conf (src addresses, buckets, timeout) from the
		// running config. On startup this is the initial set; on reconnect
		// (VPP restart) VPP has forgotten everything, so we set it again.
		c.mu.Lock()
		src := c.stateSrc
		c.mu.Unlock()
		if src != nil {
			if cfg := src.Config(); cfg != nil {
				if err := c.SetLBConf(cfg); err != nil {
					slog.Warn("vpp-lb-conf-set-failed", "err", err)
				}
			}
		}

		// Start the LB sync and stats loops for as long as the connection
		// is up. Both exit when connCtx is cancelled.
		connCtx, connCancel := context.WithCancel(ctx)
		go c.lbSyncLoop(connCtx)
		go c.lbStatsLoop(connCtx)

		// Hold the connection, pinging periodically to detect VPP restarts.
		c.monitor(ctx)
		connCancel()

		// If ctx is done we're shutting down; otherwise VPP dropped and we retry.
		c.disconnect()
		if ctx.Err() != nil {
			return
		}
		slog.Warn("vpp-disconnect", "msg", "connection lost, reconnecting")
	}
}

// lbSyncLoop drives the periodic VPP LB sync. On first entry (after the
// first successful VPP connect) it runs the warmup phase via runWarmup,
// which enforces the restart-neutrality window and handles the first full
// sync itself. Subsequent reconnect entries find warmup.allDone == true
// and skip straight to the periodic ticker. Exits when ctx is cancelled.
//
// The warmup phase is intentionally run from inside this loop rather
// than from Run: it needs the state source registered (which happens
// only after SetStateSource) and it wants to be cancelled by the same
// connCtx that cancels the stats loop on disconnect, so a VPP drop
// during warmup doesn't leak a goroutine.
func (c *Client) lbSyncLoop(ctx context.Context) {
	src := c.getStateSource()
	if src == nil {
		return // no state source registered; nothing to sync
	}

	// Warmup phase: runs once per process. On the first successful
	// VPP connect, runWarmup handles the entire window (min-delay
	// hands-off, per-VIP release phase, final SyncLBStateAll at
	// max-delay) and calls finishAll before returning. On any
	// reconnect after that, the gate is already open and we skip
	// straight to the periodic ticker. A VPP drop mid-warmup
	// returns from runWarmup via ctx.Done without closing the gate;
	// the next reconnect re-enters runWarmup, which re-reads the
	// process-relative clock and picks up wherever it left off.
	if !c.warmup.isAllDone() {
		c.runWarmup(ctx)
		if ctx.Err() != nil {
			return
		}
	}

	cfg := src.Config()
	if cfg == nil {
		return
	}
	interval := cfg.VPP.LB.SyncInterval
	if interval <= 0 {
		interval = defaultLBSyncInterval
	}
	next := time.Now().Add(interval)
	for {
		wait := time.Until(next)
		if wait < 0 {
			wait = 0
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(wait):
		}

		cfg = src.Config()
		if cfg == nil {
			next = time.Now().Add(defaultLBSyncInterval)
			continue
		}
		interval = cfg.VPP.LB.SyncInterval
		if interval <= 0 {
			interval = defaultLBSyncInterval
		}

		if err := c.SyncLBStateAll(cfg); err != nil {
			slog.Warn("vpp-lb-sync-error", "err", err)
		}
		next = time.Now().Add(interval)
	}
}

// IsConnected returns true if both API and stats connections are active.
func (c *Client) IsConnected() bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.apiConn != nil && c.statsConn != nil
}

// GetInfo returns the VPP version and connection metadata, or an error
// if VPP is not connected.
func (c *Client) GetInfo() (Info, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.apiConn == nil {
		return Info{}, errNotConnected
	}
	return c.info, nil
}

// VIPStats satisfies metrics.VPPSource. It returns the latest snapshot of
// per-VIP LB stats-segment counters captured by lbStatsLoop. Returns nil
// until the first scrape completes, or after a disconnect (the pointer is
// cleared when the connection drops).
func (c *Client) VIPStats() []metrics.VIPStatEntry {
	p := c.lbStatsSnap.Load()
	if p == nil {
		return nil
	}
	return *p
}

// VPPInfo satisfies metrics.VPPSource. It returns a copy of the cached
// connection info as a metrics-local struct so the metrics package doesn't
// need to import internal/vpp. Second return is false when VPP is not
// connected (the collector skips the vpp_* gauges in that case).
func (c *Client) VPPInfo() (metrics.VPPInfo, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.apiConn == nil {
		return metrics.VPPInfo{}, false
	}
	return metrics.VPPInfo{
		Version:        c.info.Version,
		BuildDate:      c.info.BuildDate,
		PID:            c.info.PID,
		BootTime:       c.info.BootTime,
		ConnectedSince: c.info.ConnectedSince,
	}, true
}

// connect establishes both API and stats connections. If either fails,
// both are torn down.
func (c *Client) connect() error {
	sc := socketclient.NewVppClient(c.apiAddr)
	sc.SetClientName("vpp-maglev")
	apiConn, err := core.Connect(sc)
	if err != nil {
		return err
	}

	stc := statsclient.NewStatsClient(c.statsAddr)
	statsConn, err := core.ConnectStats(stc)
	if err != nil {
		safeDisconnectAPI(apiConn)
		return err
	}

	c.mu.Lock()
	c.apiConn = apiConn
	c.statsConn = statsConn
	c.statsClient = stc
	c.mu.Unlock()
	return nil
}

// disconnect tears down both connections.
func (c *Client) disconnect() {
	c.mu.Lock()
	apiConn := c.apiConn
	statsConn := c.statsConn
	c.apiConn = nil
	c.statsConn = nil
	c.statsClient = nil
	c.info = Info{}
	c.lastLBConf = nil // force re-push of lb_conf on reconnect
	c.mu.Unlock()
	c.lbStatsSnap.Store(nil)

	safeDisconnectAPI(apiConn)
	safeDisconnectStats(statsConn)
}

// monitor blocks until the context is cancelled or a liveness ping fails.
func (c *Client) monitor(ctx context.Context) {
	ticker := time.NewTicker(pingInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			if !c.ping() {
				return
			}
		}
	}
}

// ping sends a control_ping to VPP and returns true if it succeeds.
func (c *Client) ping() bool {
	ch, err := c.apiChannel()
	if err != nil {
		return false
	}
	defer ch.Close()

	req := &core.ControlPing{}
	reply := &core.ControlPingReply{}
	if err := ch.SendRequest(req).ReceiveReply(reply); err != nil {
		slog.Debug("vpp-ping-failed", "err", err)
		return false
	}
	return true
}

// fetchInfo queries VPP for version info, PID, and boot time.
// Must be called after connect succeeds (apiConn and statsClient are set).
func (c *Client) fetchInfo() Info {
	info := Info{ConnectedSince: time.Now()}

	ch, err := c.apiChannel()
	if err != nil {
		return info
	}
	defer ch.Close()

	ver := &vpe.ShowVersionReply{}
	if err := ch.SendRequest(&vpe.ShowVersion{}).ReceiveReply(ver); err == nil {
		info.Version = ver.Version
		info.BuildDate = ver.BuildDate
		info.BuildDirectory = ver.BuildDirectory
	}

	ping := &core.ControlPingReply{}
	if err := ch.SendRequest(&core.ControlPing{}).ReceiveReply(ping); err == nil {
		info.PID = ping.VpePID
	}

	// Read VPP boot time from the stats segment.
	c.mu.Lock()
	sc := c.statsClient
	c.mu.Unlock()
	if sc != nil {
		if entries, err := sc.DumpStats("/sys/boottime"); err == nil {
			for _, e := range entries {
				if s, ok := e.Data.(adapter.ScalarStat); ok && s != 0 {
					info.BootTime = time.Unix(int64(s), 0)
				}
			}
		}
	}

	return info
}

// safeDisconnectAPI disconnects an API connection, recovering from any panic
// that GoVPP may raise on a stale connection.
func safeDisconnectAPI(conn *core.Connection) {
	if conn == nil {
		return
	}
	defer func() { recover() }() //nolint:errcheck
	conn.Disconnect()
}

// safeDisconnectStats disconnects a stats connection, recovering from panics.
func safeDisconnectStats(conn *core.StatsConnection) {
	if conn == nil {
		return
	}
	defer func() { recover() }() //nolint:errcheck
	conn.Disconnect()
}

type vppError struct{ msg string }

func (e *vppError) Error() string { return e.msg }

var errNotConnected = &vppError{msg: "VPP API connection not established"}