feat(agent): add an LLM provider circuit breaker with a deterministic fallback
docs/AGENT-DESIGN.md states plainly: "there is no confirmed... no circuit breaker: a slow/erroring LLM provider stalls that tick's decision loop." Add llm.CircuitBreaker (new internal/llm/breaker.go): trips after N consecutive failures within a rolling window (RATION_AGENT_LLM_BREAKER_* env vars), short-circuiting every call with ErrCircuitOpen until a cooldown elapses, then allows one half-open trial before fully closing again. internal/loop.Runner now treats ErrCircuitOpen as a signal to execute the already-computed, deterministic hard-rule envelope directly (same tool-handler code path the LLM itself would call, canned reason/message instead of LLM-authored text) rather than skipping the tenant's decision entirely — the breaker's only job is making sure a stuck provider never blocks or delays the money movement the hard rules already decided was safe, matching the existing "LLM decides phrasing only" design. Verified live: with no ANTHROPIC_API_KEY configured, the breaker trips after 3 failures, then correctly executes real bridge-funding grants (HOLD entries landing on ledger-core, held balance increasing) via the fallback path with zero LLM calls, and the velocity cap correctly blocks further grants afterward.
This commit is contained in:
parent
0f0de9b0ca
commit
9c859defa3
13
.env.example
13
.env.example
@ -38,6 +38,19 @@ RATION_LLM_GATEWAY_MODEL=
|
||||
# (e.g. the quickstart config above) — any real deployment needs this.
|
||||
RATION_LLM_GATEWAY_API_KEY=
|
||||
|
||||
# --- Agent LLM circuit breaker (see services/agent/internal/llm/breaker.go,
|
||||
# docs/AGENT-DESIGN.md) ---
|
||||
# Trips after this many consecutive provider failures/timeouts within
|
||||
# the failure window below, short-circuiting every subsequent call
|
||||
# with llm.ErrCircuitOpen (no provider call made at all) until the
|
||||
# cooldown elapses — never blocking or delaying the already-safe
|
||||
# deterministic hard-rule envelope internal/loop.Runner executes
|
||||
# directly once tripped. All three are optional; unset/unparseable
|
||||
# falls back to the documented default in breaker.go.
|
||||
RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD=3
|
||||
RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS=300
|
||||
RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS=60
|
||||
|
||||
# --- Services ---
|
||||
LEDGER_CORE_ADDR=localhost:7001
|
||||
LEDGER_CORE_METRICS_ADDR=localhost:9099
|
||||
|
||||
@ -167,7 +167,16 @@ func run(logger *slog.Logger) error {
|
||||
provider := llm.NewAnthropicProviderFromEnv()
|
||||
policyCfg := policy.LoadConfigFromEnv()
|
||||
|
||||
runner := loop.NewRunner(ledgerClient, gatewayClient, provider, policyCfg, logger)
|
||||
// Wrap the real provider in a circuit breaker (see internal/llm/
|
||||
// breaker.go, docs/AGENT-DESIGN.md: "there is no confirmed... no
|
||||
// circuit breaker: a slow/erroring LLM provider stalls that tick's
|
||||
// decision loop") so a stuck Anthropic/Relay call can never block
|
||||
// a tick beyond RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS — once
|
||||
// tripped, internal/loop.Runner executes the already-safe
|
||||
// deterministic hard-rule envelope directly instead of waiting.
|
||||
llmProvider := llm.NewCircuitBreakerFromEnv(provider, logger)
|
||||
|
||||
runner := loop.NewRunner(ledgerClient, gatewayClient, llmProvider, policyCfg, logger)
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
281
services/agent/internal/llm/breaker.go
Normal file
281
services/agent/internal/llm/breaker.go
Normal file
@ -0,0 +1,281 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrCircuitOpen is returned by CircuitBreaker.RunConversation instead of
|
||||
// calling the wrapped Provider at all, once the breaker has tripped.
|
||||
//
|
||||
// docs/AGENT-DESIGN.md states plainly: "there is no confirmed... cap...
|
||||
// no circuit breaker: a slow/erroring LLM provider stalls that tick's
|
||||
// decision loop." internal/loop.Runner's own design already guarantees
|
||||
// "LLM decides phrasing only" — policy.ComputeBridgeEnvelope's
|
||||
// deterministic hard-rule result is computed and recorded
|
||||
// (RecordAgentDecision) BEFORE the LLM is ever called, so the LLM's
|
||||
// only real job per tick is choosing how much of that already-safe
|
||||
// ceiling to actually execute and how to phrase it. callers that see
|
||||
// ErrCircuitOpen should execute that same ceiling directly (see
|
||||
// internal/loop.Runner.runFallbackDecision) rather than treating a
|
||||
// tripped breaker as a reason to skip the tenant's decision entirely —
|
||||
// a stuck LLM provider must never block or delay the money movement
|
||||
// the hard rules already decided was safe.
|
||||
var ErrCircuitOpen = errors.New("llm: circuit breaker open, provider calls short-circuited")
|
||||
|
||||
// BreakerState is one of CircuitBreaker's three states, exported for
|
||||
// logging/tests.
|
||||
type BreakerState string
|
||||
|
||||
const (
|
||||
// BreakerClosed is the normal state: every call is forwarded to the
|
||||
// wrapped Provider.
|
||||
BreakerClosed BreakerState = "closed"
|
||||
// BreakerOpen short-circuits every call with ErrCircuitOpen, calling
|
||||
// the wrapped Provider zero times, until Cooldown has elapsed since
|
||||
// the trip.
|
||||
BreakerOpen BreakerState = "open"
|
||||
// BreakerHalfOpen allows exactly the next call through as a trial:
|
||||
// success closes the breaker again, failure reopens it immediately
|
||||
// (not waiting for FailureThreshold more failures — one failed
|
||||
// recovery attempt is enough evidence the provider is still down).
|
||||
BreakerHalfOpen BreakerState = "half_open"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultFailureThreshold is how many consecutive failures (within
|
||||
// defaultFailureWindow of each other) trip the breaker.
|
||||
defaultFailureThreshold = 3
|
||||
// defaultFailureWindow bounds how long a string of failures can be
|
||||
// spread out and still count as "consecutive" for tripping purposes
|
||||
// — a handful of failures spread across days is noise, not an
|
||||
// outage; a handful within minutes is an outage.
|
||||
defaultFailureWindow = 5 * time.Minute
|
||||
// defaultCooldown is how long the breaker stays fully Open (never
|
||||
// calling the provider at all) before allowing one HalfOpen trial
|
||||
// call through.
|
||||
defaultCooldown = 60 * time.Second
|
||||
)
|
||||
|
||||
// CircuitBreaker wraps a Provider so a slow/erroring LLM provider can
|
||||
// never stall the agent's decision loop beyond this breaker's own
|
||||
// cooldown. Safe for concurrent use (guarded by mu), though
|
||||
// internal/loop.Runner's own Tick processes deposits sequentially
|
||||
// today — the lock exists for correctness regardless of caller
|
||||
// concurrency, not because of a known concurrent caller.
|
||||
//
|
||||
// Tripping and recovery are both logged loudly (slog, at Warn/Info
|
||||
// level respectively) per docs/AGENT-DESIGN.md's explicit ask — an
|
||||
// operator watching logs must be able to see this happening, not just
|
||||
// infer it from a stream of ErrCircuitOpen-caused fallback decisions.
|
||||
type CircuitBreaker struct {
|
||||
provider Provider
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
failureThreshold int
|
||||
failureWindow time.Duration
|
||||
cooldown time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
state BreakerState
|
||||
consecutiveFailures int
|
||||
lastFailureAt time.Time
|
||||
openedAt time.Time
|
||||
}
|
||||
|
||||
// BreakerOption configures a CircuitBreaker at construction time.
|
||||
type BreakerOption func(*CircuitBreaker)
|
||||
|
||||
// WithFailureThreshold overrides defaultFailureThreshold.
|
||||
func WithFailureThreshold(n int) BreakerOption {
|
||||
return func(b *CircuitBreaker) {
|
||||
if n > 0 {
|
||||
b.failureThreshold = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithFailureWindow overrides defaultFailureWindow.
|
||||
func WithFailureWindow(d time.Duration) BreakerOption {
|
||||
return func(b *CircuitBreaker) {
|
||||
if d > 0 {
|
||||
b.failureWindow = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithCooldown overrides defaultCooldown.
|
||||
func WithCooldown(d time.Duration) BreakerOption {
|
||||
return func(b *CircuitBreaker) {
|
||||
if d > 0 {
|
||||
b.cooldown = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger overrides the default (slog.Default()) logger trip/
|
||||
// recovery events are reported to.
|
||||
func WithLogger(logger *slog.Logger) BreakerOption {
|
||||
return func(b *CircuitBreaker) {
|
||||
if logger != nil {
|
||||
b.logger = logger
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withNow overrides the clock. Test-only (unexported); production
|
||||
// callers always get the real time.Now.
|
||||
func withNow(now func() time.Time) BreakerOption {
|
||||
return func(b *CircuitBreaker) { b.now = now }
|
||||
}
|
||||
|
||||
// NewCircuitBreaker wraps provider with a breaker using
|
||||
// default{FailureThreshold,FailureWindow,Cooldown} unless overridden by
|
||||
// opts.
|
||||
func NewCircuitBreaker(provider Provider, opts ...BreakerOption) *CircuitBreaker {
|
||||
b := &CircuitBreaker{
|
||||
provider: provider,
|
||||
logger: slog.Default(),
|
||||
now: time.Now,
|
||||
failureThreshold: defaultFailureThreshold,
|
||||
failureWindow: defaultFailureWindow,
|
||||
cooldown: defaultCooldown,
|
||||
state: BreakerClosed,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(b)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// NewCircuitBreakerFromEnv builds a CircuitBreaker wrapping provider,
|
||||
// reading RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD,
|
||||
// RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS and
|
||||
// RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS (all optional, falling back
|
||||
// to the same defaults NewCircuitBreaker uses when unset or
|
||||
// unparseable) — the same env-driven construction convention
|
||||
// internal/policy.LoadConfigFromEnv already uses for its own tunables.
|
||||
func NewCircuitBreakerFromEnv(provider Provider, logger *slog.Logger) *CircuitBreaker {
|
||||
opts := []BreakerOption{WithLogger(logger)}
|
||||
|
||||
if v := os.Getenv("RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
opts = append(opts, WithFailureThreshold(n))
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
opts = append(opts, WithFailureWindow(time.Duration(n)*time.Second))
|
||||
}
|
||||
}
|
||||
if v := os.Getenv("RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
opts = append(opts, WithCooldown(time.Duration(n)*time.Second))
|
||||
}
|
||||
}
|
||||
|
||||
return NewCircuitBreaker(provider, opts...)
|
||||
}
|
||||
|
||||
// State reports the breaker's current state, resolving an Open state
|
||||
// whose cooldown has already elapsed into HalfOpen first — the same
|
||||
// resolution RunConversation itself applies before deciding whether to
|
||||
// call through. Exposed for logging/metrics/tests.
|
||||
func (b *CircuitBreaker) State() BreakerState {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.resolveStateLocked()
|
||||
}
|
||||
|
||||
// resolveStateLocked must be called with mu held. It transitions Open
|
||||
// to HalfOpen once cooldown has elapsed, mutating b.state, and returns
|
||||
// the resolved state.
|
||||
func (b *CircuitBreaker) resolveStateLocked() BreakerState {
|
||||
if b.state == BreakerOpen && b.now().Sub(b.openedAt) >= b.cooldown {
|
||||
b.state = BreakerHalfOpen
|
||||
}
|
||||
return b.state
|
||||
}
|
||||
|
||||
// RunConversation implements Provider. When the breaker is Open, this
|
||||
// returns ErrCircuitOpen immediately, calling the wrapped Provider
|
||||
// zero times — see the package doc comment on ErrCircuitOpen for why a
|
||||
// caller should treat that as "execute the deterministic fallback",
|
||||
// not as a generic failure to log and move on from.
|
||||
func (b *CircuitBreaker) RunConversation(ctx context.Context, systemPrompt, userMessage string, tools []Tool, handler ToolHandler) (string, error) {
|
||||
b.mu.Lock()
|
||||
state := b.resolveStateLocked()
|
||||
if state == BreakerOpen {
|
||||
b.mu.Unlock()
|
||||
return "", ErrCircuitOpen
|
||||
}
|
||||
wasHalfOpen := state == BreakerHalfOpen
|
||||
b.mu.Unlock()
|
||||
|
||||
result, err := b.provider.RunConversation(ctx, systemPrompt, userMessage, tools, handler)
|
||||
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if err != nil {
|
||||
b.recordFailureLocked(wasHalfOpen, err)
|
||||
} else {
|
||||
b.recordSuccessLocked(wasHalfOpen)
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
// recordFailureLocked must be called with mu held, immediately after a
|
||||
// call to the wrapped Provider failed.
|
||||
func (b *CircuitBreaker) recordFailureLocked(wasHalfOpen bool, callErr error) {
|
||||
now := b.now()
|
||||
|
||||
if wasHalfOpen {
|
||||
// One failed recovery attempt is enough evidence the provider
|
||||
// is still down — reopen immediately rather than waiting to
|
||||
// re-accumulate failureThreshold more failures.
|
||||
b.state = BreakerOpen
|
||||
b.openedAt = now
|
||||
b.lastFailureAt = now
|
||||
b.logger.Warn("llm: circuit breaker recovery attempt failed, reopening",
|
||||
"cooldown", b.cooldown.String(), "error", callErr)
|
||||
return
|
||||
}
|
||||
|
||||
if b.lastFailureAt.IsZero() || now.Sub(b.lastFailureAt) > b.failureWindow {
|
||||
// Either the first failure ever, or the previous failure is
|
||||
// old enough that this one doesn't continue the same streak —
|
||||
// start counting over.
|
||||
b.consecutiveFailures = 1
|
||||
} else {
|
||||
b.consecutiveFailures++
|
||||
}
|
||||
b.lastFailureAt = now
|
||||
|
||||
if b.consecutiveFailures >= b.failureThreshold && b.state == BreakerClosed {
|
||||
b.state = BreakerOpen
|
||||
b.openedAt = now
|
||||
b.logger.Warn("llm: circuit breaker TRIPPED — short-circuiting to the hard-rule envelope until cooldown elapses",
|
||||
"consecutive_failures", b.consecutiveFailures,
|
||||
"failure_threshold", b.failureThreshold,
|
||||
"failure_window", b.failureWindow.String(),
|
||||
"cooldown", b.cooldown.String(),
|
||||
"error", callErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// recordSuccessLocked must be called with mu held, immediately after a
|
||||
// call to the wrapped Provider succeeded.
|
||||
func (b *CircuitBreaker) recordSuccessLocked(wasHalfOpen bool) {
|
||||
b.consecutiveFailures = 0
|
||||
if wasHalfOpen {
|
||||
b.state = BreakerClosed
|
||||
b.logger.Info("llm: circuit breaker recovered — provider call succeeded, resuming normal operation")
|
||||
}
|
||||
}
|
||||
@ -8,6 +8,7 @@ package loop
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
@ -174,6 +175,17 @@ func (r *Runner) processLateDeposit(ctx context.Context, dep *ledger.Deposit, no
|
||||
llm.NewHandler(toolCtx),
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, llm.ErrCircuitOpen) {
|
||||
// A tripped breaker (services/agent/internal/llm/breaker.go)
|
||||
// must never block or delay the decision the deterministic
|
||||
// hard-rule envelope already made — see
|
||||
// runFallbackDecision's doc comment. This is the intended,
|
||||
// safe behavior, not a failure to report up as a tick
|
||||
// error.
|
||||
logger.Warn("llm circuit breaker open, executing hard-rule envelope directly", "decision", envelope.Decision)
|
||||
r.runFallbackDecision(ctx, toolCtx, envelope, logger)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("LLM conversation: %w", err)
|
||||
}
|
||||
|
||||
@ -181,6 +193,48 @@ func (r *Runner) processLateDeposit(ctx context.Context, dep *ledger.Deposit, no
|
||||
return nil
|
||||
}
|
||||
|
||||
// runFallbackDecision executes the already-computed, deterministic
|
||||
// hard-rule envelope directly — bypassing the LLM conversation
|
||||
// entirely — by calling the exact same tool handler
|
||||
// (llm.NewHandler(toolCtx)) the LLM itself would call, with a canned
|
||||
// reason/message instead of an LLM-authored one.
|
||||
//
|
||||
// This is what makes short-circuiting on llm.ErrCircuitOpen safe
|
||||
// rather than merely fast: per docs/AGENT-DESIGN.md's "LLM decides
|
||||
// phrasing only" guarantee, envelope.MaxGrantMinorUnits is already the
|
||||
// safe ceiling (policy.ComputeBridgeEnvelope, computed and recorded
|
||||
// via RecordAgentDecision before the LLM was ever consulted — see
|
||||
// above) — the LLM's only real job this tick was choosing how much of
|
||||
// that ceiling to execute and how to phrase it to the tenant. A tripped
|
||||
// breaker means that choice can't be made by the LLM right now, so
|
||||
// this executes the full ceiling deterministically instead of leaving
|
||||
// safe, already-decided money movement blocked on a stuck provider.
|
||||
// Every call here goes through the identical tool-handler code path
|
||||
// (and therefore the identical gateway/ledger-core re-validation and
|
||||
// RecordAgentDecision audit trail) a real LLM tool call would use.
|
||||
func (r *Runner) runFallbackDecision(ctx context.Context, toolCtx llm.ToolContext, envelope policy.Envelope, logger *slog.Logger) {
|
||||
handler := llm.NewHandler(toolCtx)
|
||||
|
||||
if envelope.Decision != policy.DecisionDeclined && envelope.MaxGrantMinorUnits > 0 {
|
||||
input, _ := json.Marshal(map[string]any{
|
||||
"amount_minor_units": envelope.MaxGrantMinorUnits,
|
||||
"reason": "circuit breaker fallback: LLM provider unavailable, granting the full hard-rule envelope ceiling deterministically",
|
||||
})
|
||||
result, isErr := handler(ctx, llm.ToolRationBridgeFunding, input)
|
||||
if isErr {
|
||||
logger.Error("circuit breaker fallback: ration_bridge_funding call failed", "result", result)
|
||||
} else {
|
||||
logger.Info("circuit breaker fallback: ration_bridge_funding executed", "result", result)
|
||||
}
|
||||
}
|
||||
|
||||
message := fmt.Sprintf("Automated decision (LLM temporarily unavailable): %s — %s", envelope.Decision, envelope.Reason)
|
||||
notifyInput, _ := json.Marshal(map[string]any{"message": message})
|
||||
if result, isErr := handler(ctx, llm.ToolNotifyTenant, notifyInput); isErr {
|
||||
logger.Error("circuit breaker fallback: notify_tenant call failed", "result", result)
|
||||
}
|
||||
}
|
||||
|
||||
// hasOpenGrantWithinWindow implements the velocity-cap data gathering
|
||||
// described in docs/AGENT-DESIGN.md: a prior RATION_BRIDGE grant inside
|
||||
// the rolling window blocks a new grant unless its target deposit has
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user