Guard pause/resume against disabled backends; clean up CLI errors
- PauseBackend and ResumeBackend return an error (not bool) when the
backend is disabled, preventing an inconsistent state where the
health state says "paused" but enabled=false.
- DisableBackend and EnableBackend now log uniform backend-transition
lines with from/to instead of separate backend-disable/backend-enable
messages.
- CLI errors strip gRPC boilerplate ("rpc error: code = ... desc = ")
and display the server message in red (when color is enabled). Both
the interactive shell and one-shot mode use the same formatError path.
This commit is contained in:
@@ -288,12 +288,16 @@ func (c *Checker) GetBackendInfo(name string) (metrics.BackendInfo, bool) {
|
||||
// goroutine is cancelled so no further traffic is sent to the backend. The
|
||||
// backend's state is set to paused and remains frozen until ResumeBackend is
|
||||
// called (which starts a fresh probe goroutine).
|
||||
func (c *Checker) PauseBackend(name string) (BackendSnapshot, bool) {
|
||||
// Returns an error if the backend is not found or is disabled.
|
||||
func (c *Checker) PauseBackend(name string) (BackendSnapshot, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
w, ok := c.workers[name]
|
||||
if !ok {
|
||||
return BackendSnapshot{}, false
|
||||
return BackendSnapshot{}, fmt.Errorf("backend %q not found", name)
|
||||
}
|
||||
if !w.entry.Enabled {
|
||||
return BackendSnapshot{}, fmt.Errorf("backend %q is disabled; enable it first", name)
|
||||
}
|
||||
maxHistory := c.cfg.HealthChecker.TransitionHistory
|
||||
if w.backend.Pause(maxHistory) {
|
||||
@@ -305,18 +309,22 @@ func (c *Checker) PauseBackend(name string) (BackendSnapshot, bool) {
|
||||
c.emitForBackend(name, w.backend.Address, t, c.cfg.Frontends)
|
||||
}
|
||||
w.cancel()
|
||||
return BackendSnapshot{Health: w.backend, Config: w.entry}, true
|
||||
return BackendSnapshot{Health: w.backend, Config: w.entry}, nil
|
||||
}
|
||||
|
||||
// ResumeBackend resumes health checking for a backend by name. A fresh probe
|
||||
// goroutine is started and the backend re-enters StateUnknown. The existing
|
||||
// transition history is preserved.
|
||||
func (c *Checker) ResumeBackend(name string) (BackendSnapshot, bool) {
|
||||
// Returns an error if the backend is not found or is disabled.
|
||||
func (c *Checker) ResumeBackend(name string) (BackendSnapshot, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
w, ok := c.workers[name]
|
||||
if !ok {
|
||||
return BackendSnapshot{}, false
|
||||
return BackendSnapshot{}, fmt.Errorf("backend %q not found", name)
|
||||
}
|
||||
if !w.entry.Enabled {
|
||||
return BackendSnapshot{}, fmt.Errorf("backend %q is disabled; enable it first", name)
|
||||
}
|
||||
maxHistory := c.cfg.HealthChecker.TransitionHistory
|
||||
if w.backend.Resume(maxHistory) {
|
||||
@@ -333,7 +341,7 @@ func (c *Checker) ResumeBackend(name string) (BackendSnapshot, bool) {
|
||||
w.cancel = cancel
|
||||
w.wakeCh = make(chan struct{}, 1)
|
||||
go c.runProbe(wCtx, name, 0, 1)
|
||||
return BackendSnapshot{Health: w.backend, Config: w.entry}, true
|
||||
return BackendSnapshot{Health: w.backend, Config: w.entry}, nil
|
||||
}
|
||||
|
||||
// DisableBackend stops health checking for a backend and removes it from active
|
||||
@@ -351,7 +359,10 @@ func (c *Checker) DisableBackend(name string) (BackendSnapshot, bool) {
|
||||
}
|
||||
maxHistory := c.cfg.HealthChecker.TransitionHistory
|
||||
t := w.backend.Disable(maxHistory)
|
||||
slog.Info("backend-disable", "backend", name)
|
||||
slog.Info("backend-transition", "backend", name,
|
||||
"from", t.From.String(),
|
||||
"to", t.To.String(),
|
||||
)
|
||||
c.emitForBackend(name, w.backend.Address, t, c.cfg.Frontends)
|
||||
w.cancel()
|
||||
w.entry.Enabled = false
|
||||
@@ -382,10 +393,14 @@ func (c *Checker) EnableBackend(name string) (BackendSnapshot, bool) {
|
||||
}
|
||||
maxHistory := c.cfg.HealthChecker.TransitionHistory
|
||||
hc := c.cfg.HealthChecks[entry.HealthCheck]
|
||||
slog.Info("backend-enable", "backend", name)
|
||||
c.startWorker(c.runCtx, name, entry, hc, 0, 1, maxHistory)
|
||||
nw := c.workers[name]
|
||||
c.emitForBackend(name, nw.backend.Address, nw.backend.Transitions[0], c.cfg.Frontends)
|
||||
t := nw.backend.Transitions[0]
|
||||
slog.Info("backend-transition", "backend", name,
|
||||
"from", t.From.String(),
|
||||
"to", t.To.String(),
|
||||
)
|
||||
c.emitForBackend(name, nw.backend.Address, t, c.cfg.Frontends)
|
||||
return BackendSnapshot{Health: nw.backend, Config: nw.entry}, true
|
||||
}
|
||||
|
||||
|
||||
@@ -358,19 +358,30 @@ func TestPauseResume(t *testing.T) {
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
b, ok := c.PauseBackend("be0")
|
||||
if !ok {
|
||||
t.Fatal("PauseBackend: not found")
|
||||
b, err := c.PauseBackend("be0")
|
||||
if err != nil {
|
||||
t.Fatalf("PauseBackend: %v", err)
|
||||
}
|
||||
if b.Health.State != health.StatePaused {
|
||||
t.Errorf("after pause: %s", b.Health.State)
|
||||
}
|
||||
|
||||
b, ok = c.ResumeBackend("be0")
|
||||
if !ok {
|
||||
t.Fatal("ResumeBackend: not found")
|
||||
b, err = c.ResumeBackend("be0")
|
||||
if err != nil {
|
||||
t.Fatalf("ResumeBackend: %v", err)
|
||||
}
|
||||
if b.Health.State != health.StateUnknown {
|
||||
t.Errorf("after resume: %s", b.Health.State)
|
||||
}
|
||||
|
||||
// Pause/resume on a disabled backend must return an error.
|
||||
c.mu.Lock()
|
||||
c.workers["be0"].entry.Enabled = false
|
||||
c.mu.Unlock()
|
||||
if _, err := c.PauseBackend("be0"); err == nil {
|
||||
t.Error("PauseBackend on disabled backend: expected error")
|
||||
}
|
||||
if _, err := c.ResumeBackend("be0"); err == nil {
|
||||
t.Error("ResumeBackend on disabled backend: expected error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,18 +64,18 @@ func (s *Server) GetBackend(_ context.Context, req *GetBackendRequest) (*Backend
|
||||
|
||||
// PauseBackend pauses health checking for a backend by name.
|
||||
func (s *Server) PauseBackend(_ context.Context, req *BackendRequest) (*BackendInfo, error) {
|
||||
b, ok := s.checker.PauseBackend(req.Name)
|
||||
if !ok {
|
||||
return nil, status.Errorf(codes.NotFound, "backend %q not found", req.Name)
|
||||
b, err := s.checker.PauseBackend(req.Name)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.FailedPrecondition, "%v", err)
|
||||
}
|
||||
return backendToProto(b), nil
|
||||
}
|
||||
|
||||
// ResumeBackend resumes health checking for a backend by name.
|
||||
func (s *Server) ResumeBackend(_ context.Context, req *BackendRequest) (*BackendInfo, error) {
|
||||
b, ok := s.checker.ResumeBackend(req.Name)
|
||||
if !ok {
|
||||
return nil, status.Errorf(codes.NotFound, "backend %q not found", req.Name)
|
||||
b, err := s.checker.ResumeBackend(req.Name)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.FailedPrecondition, "%v", err)
|
||||
}
|
||||
return backendToProto(b), nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user