From 9c859defa314fee8dafd7c7ae83aee3edd2c813a Mon Sep 17 00:00:00 2001 From: caelum0x Date: Fri, 23 Oct 2026 19:01:51 +0300 Subject: [PATCH] feat(agent): add an LLM provider circuit breaker with a deterministic fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/AGENT-DESIGN.md states plainly: "there is no confirmed... no circuit breaker: a slow/erroring LLM provider stalls that tick's decision loop." Add llm.CircuitBreaker (new internal/llm/breaker.go): trips after N consecutive failures within a rolling window (RATION_AGENT_LLM_BREAKER_* env vars), short-circuiting every call with ErrCircuitOpen until a cooldown elapses, then allows one half-open trial before fully closing again. internal/loop.Runner now treats ErrCircuitOpen as a signal to execute the already-computed, deterministic hard-rule envelope directly (same tool-handler code path the LLM itself would call, canned reason/message instead of LLM-authored text) rather than skipping the tenant's decision entirely — the breaker's only job is making sure a stuck provider never blocks or delays the money movement the hard rules already decided was safe, matching the existing "LLM decides phrasing only" design. Verified live: with no ANTHROPIC_API_KEY configured, the breaker trips after 3 failures, then correctly executes real bridge-funding grants (HOLD entries landing on ledger-core, held balance increasing) via the fallback path with zero LLM calls, and the velocity cap correctly blocks further grants afterward. --- .env.example | 13 ++ services/agent/cmd/agent/main.go | 11 +- services/agent/internal/llm/breaker.go | 281 +++++++++++++++++++++++++ services/agent/internal/loop/runner.go | 54 +++++ 4 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 services/agent/internal/llm/breaker.go diff --git a/.env.example b/.env.example index de294ac..1fcb241 100644 --- a/.env.example +++ b/.env.example @@ -38,6 +38,19 @@ RATION_LLM_GATEWAY_MODEL= # (e.g. the quickstart config above) — any real deployment needs this. RATION_LLM_GATEWAY_API_KEY= +# --- Agent LLM circuit breaker (see services/agent/internal/llm/breaker.go, +# docs/AGENT-DESIGN.md) --- +# Trips after this many consecutive provider failures/timeouts within +# the failure window below, short-circuiting every subsequent call +# with llm.ErrCircuitOpen (no provider call made at all) until the +# cooldown elapses — never blocking or delaying the already-safe +# deterministic hard-rule envelope internal/loop.Runner executes +# directly once tripped. All three are optional; unset/unparseable +# falls back to the documented default in breaker.go. +RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD=3 +RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS=300 +RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS=60 + # --- Services --- LEDGER_CORE_ADDR=localhost:7001 LEDGER_CORE_METRICS_ADDR=localhost:9099 diff --git a/services/agent/cmd/agent/main.go b/services/agent/cmd/agent/main.go index a3744ec..f05126a 100644 --- a/services/agent/cmd/agent/main.go +++ b/services/agent/cmd/agent/main.go @@ -167,7 +167,16 @@ func run(logger *slog.Logger) error { provider := llm.NewAnthropicProviderFromEnv() policyCfg := policy.LoadConfigFromEnv() - runner := loop.NewRunner(ledgerClient, gatewayClient, provider, policyCfg, logger) + // Wrap the real provider in a circuit breaker (see internal/llm/ + // breaker.go, docs/AGENT-DESIGN.md: "there is no confirmed... no + // circuit breaker: a slow/erroring LLM provider stalls that tick's + // decision loop") so a stuck Anthropic/Relay call can never block + // a tick beyond RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS — once + // tripped, internal/loop.Runner executes the already-safe + // deterministic hard-rule envelope directly instead of waiting. + llmProvider := llm.NewCircuitBreakerFromEnv(provider, logger) + + runner := loop.NewRunner(ledgerClient, gatewayClient, llmProvider, policyCfg, logger) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() diff --git a/services/agent/internal/llm/breaker.go b/services/agent/internal/llm/breaker.go new file mode 100644 index 0000000..a629057 --- /dev/null +++ b/services/agent/internal/llm/breaker.go @@ -0,0 +1,281 @@ +package llm + +import ( + "context" + "errors" + "log/slog" + "os" + "strconv" + "sync" + "time" +) + +// ErrCircuitOpen is returned by CircuitBreaker.RunConversation instead of +// calling the wrapped Provider at all, once the breaker has tripped. +// +// docs/AGENT-DESIGN.md states plainly: "there is no confirmed... cap... +// no circuit breaker: a slow/erroring LLM provider stalls that tick's +// decision loop." internal/loop.Runner's own design already guarantees +// "LLM decides phrasing only" — policy.ComputeBridgeEnvelope's +// deterministic hard-rule result is computed and recorded +// (RecordAgentDecision) BEFORE the LLM is ever called, so the LLM's +// only real job per tick is choosing how much of that already-safe +// ceiling to actually execute and how to phrase it. callers that see +// ErrCircuitOpen should execute that same ceiling directly (see +// internal/loop.Runner.runFallbackDecision) rather than treating a +// tripped breaker as a reason to skip the tenant's decision entirely — +// a stuck LLM provider must never block or delay the money movement +// the hard rules already decided was safe. +var ErrCircuitOpen = errors.New("llm: circuit breaker open, provider calls short-circuited") + +// BreakerState is one of CircuitBreaker's three states, exported for +// logging/tests. +type BreakerState string + +const ( + // BreakerClosed is the normal state: every call is forwarded to the + // wrapped Provider. + BreakerClosed BreakerState = "closed" + // BreakerOpen short-circuits every call with ErrCircuitOpen, calling + // the wrapped Provider zero times, until Cooldown has elapsed since + // the trip. + BreakerOpen BreakerState = "open" + // BreakerHalfOpen allows exactly the next call through as a trial: + // success closes the breaker again, failure reopens it immediately + // (not waiting for FailureThreshold more failures — one failed + // recovery attempt is enough evidence the provider is still down). + BreakerHalfOpen BreakerState = "half_open" +) + +const ( + // defaultFailureThreshold is how many consecutive failures (within + // defaultFailureWindow of each other) trip the breaker. + defaultFailureThreshold = 3 + // defaultFailureWindow bounds how long a string of failures can be + // spread out and still count as "consecutive" for tripping purposes + // — a handful of failures spread across days is noise, not an + // outage; a handful within minutes is an outage. + defaultFailureWindow = 5 * time.Minute + // defaultCooldown is how long the breaker stays fully Open (never + // calling the provider at all) before allowing one HalfOpen trial + // call through. + defaultCooldown = 60 * time.Second +) + +// CircuitBreaker wraps a Provider so a slow/erroring LLM provider can +// never stall the agent's decision loop beyond this breaker's own +// cooldown. Safe for concurrent use (guarded by mu), though +// internal/loop.Runner's own Tick processes deposits sequentially +// today — the lock exists for correctness regardless of caller +// concurrency, not because of a known concurrent caller. +// +// Tripping and recovery are both logged loudly (slog, at Warn/Info +// level respectively) per docs/AGENT-DESIGN.md's explicit ask — an +// operator watching logs must be able to see this happening, not just +// infer it from a stream of ErrCircuitOpen-caused fallback decisions. +type CircuitBreaker struct { + provider Provider + logger *slog.Logger + now func() time.Time + + failureThreshold int + failureWindow time.Duration + cooldown time.Duration + + mu sync.Mutex + state BreakerState + consecutiveFailures int + lastFailureAt time.Time + openedAt time.Time +} + +// BreakerOption configures a CircuitBreaker at construction time. +type BreakerOption func(*CircuitBreaker) + +// WithFailureThreshold overrides defaultFailureThreshold. +func WithFailureThreshold(n int) BreakerOption { + return func(b *CircuitBreaker) { + if n > 0 { + b.failureThreshold = n + } + } +} + +// WithFailureWindow overrides defaultFailureWindow. +func WithFailureWindow(d time.Duration) BreakerOption { + return func(b *CircuitBreaker) { + if d > 0 { + b.failureWindow = d + } + } +} + +// WithCooldown overrides defaultCooldown. +func WithCooldown(d time.Duration) BreakerOption { + return func(b *CircuitBreaker) { + if d > 0 { + b.cooldown = d + } + } +} + +// WithLogger overrides the default (slog.Default()) logger trip/ +// recovery events are reported to. +func WithLogger(logger *slog.Logger) BreakerOption { + return func(b *CircuitBreaker) { + if logger != nil { + b.logger = logger + } + } +} + +// withNow overrides the clock. Test-only (unexported); production +// callers always get the real time.Now. +func withNow(now func() time.Time) BreakerOption { + return func(b *CircuitBreaker) { b.now = now } +} + +// NewCircuitBreaker wraps provider with a breaker using +// default{FailureThreshold,FailureWindow,Cooldown} unless overridden by +// opts. +func NewCircuitBreaker(provider Provider, opts ...BreakerOption) *CircuitBreaker { + b := &CircuitBreaker{ + provider: provider, + logger: slog.Default(), + now: time.Now, + failureThreshold: defaultFailureThreshold, + failureWindow: defaultFailureWindow, + cooldown: defaultCooldown, + state: BreakerClosed, + } + for _, opt := range opts { + opt(b) + } + return b +} + +// NewCircuitBreakerFromEnv builds a CircuitBreaker wrapping provider, +// reading RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD, +// RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS and +// RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS (all optional, falling back +// to the same defaults NewCircuitBreaker uses when unset or +// unparseable) — the same env-driven construction convention +// internal/policy.LoadConfigFromEnv already uses for its own tunables. +func NewCircuitBreakerFromEnv(provider Provider, logger *slog.Logger) *CircuitBreaker { + opts := []BreakerOption{WithLogger(logger)} + + if v := os.Getenv("RATION_AGENT_LLM_BREAKER_FAILURE_THRESHOLD"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts = append(opts, WithFailureThreshold(n)) + } + } + if v := os.Getenv("RATION_AGENT_LLM_BREAKER_FAILURE_WINDOW_SECONDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts = append(opts, WithFailureWindow(time.Duration(n)*time.Second)) + } + } + if v := os.Getenv("RATION_AGENT_LLM_BREAKER_COOLDOWN_SECONDS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts = append(opts, WithCooldown(time.Duration(n)*time.Second)) + } + } + + return NewCircuitBreaker(provider, opts...) +} + +// State reports the breaker's current state, resolving an Open state +// whose cooldown has already elapsed into HalfOpen first — the same +// resolution RunConversation itself applies before deciding whether to +// call through. Exposed for logging/metrics/tests. +func (b *CircuitBreaker) State() BreakerState { + b.mu.Lock() + defer b.mu.Unlock() + return b.resolveStateLocked() +} + +// resolveStateLocked must be called with mu held. It transitions Open +// to HalfOpen once cooldown has elapsed, mutating b.state, and returns +// the resolved state. +func (b *CircuitBreaker) resolveStateLocked() BreakerState { + if b.state == BreakerOpen && b.now().Sub(b.openedAt) >= b.cooldown { + b.state = BreakerHalfOpen + } + return b.state +} + +// RunConversation implements Provider. When the breaker is Open, this +// returns ErrCircuitOpen immediately, calling the wrapped Provider +// zero times — see the package doc comment on ErrCircuitOpen for why a +// caller should treat that as "execute the deterministic fallback", +// not as a generic failure to log and move on from. +func (b *CircuitBreaker) RunConversation(ctx context.Context, systemPrompt, userMessage string, tools []Tool, handler ToolHandler) (string, error) { + b.mu.Lock() + state := b.resolveStateLocked() + if state == BreakerOpen { + b.mu.Unlock() + return "", ErrCircuitOpen + } + wasHalfOpen := state == BreakerHalfOpen + b.mu.Unlock() + + result, err := b.provider.RunConversation(ctx, systemPrompt, userMessage, tools, handler) + + b.mu.Lock() + defer b.mu.Unlock() + if err != nil { + b.recordFailureLocked(wasHalfOpen, err) + } else { + b.recordSuccessLocked(wasHalfOpen) + } + return result, err +} + +// recordFailureLocked must be called with mu held, immediately after a +// call to the wrapped Provider failed. +func (b *CircuitBreaker) recordFailureLocked(wasHalfOpen bool, callErr error) { + now := b.now() + + if wasHalfOpen { + // One failed recovery attempt is enough evidence the provider + // is still down — reopen immediately rather than waiting to + // re-accumulate failureThreshold more failures. + b.state = BreakerOpen + b.openedAt = now + b.lastFailureAt = now + b.logger.Warn("llm: circuit breaker recovery attempt failed, reopening", + "cooldown", b.cooldown.String(), "error", callErr) + return + } + + if b.lastFailureAt.IsZero() || now.Sub(b.lastFailureAt) > b.failureWindow { + // Either the first failure ever, or the previous failure is + // old enough that this one doesn't continue the same streak — + // start counting over. + b.consecutiveFailures = 1 + } else { + b.consecutiveFailures++ + } + b.lastFailureAt = now + + if b.consecutiveFailures >= b.failureThreshold && b.state == BreakerClosed { + b.state = BreakerOpen + b.openedAt = now + b.logger.Warn("llm: circuit breaker TRIPPED — short-circuiting to the hard-rule envelope until cooldown elapses", + "consecutive_failures", b.consecutiveFailures, + "failure_threshold", b.failureThreshold, + "failure_window", b.failureWindow.String(), + "cooldown", b.cooldown.String(), + "error", callErr, + ) + } +} + +// recordSuccessLocked must be called with mu held, immediately after a +// call to the wrapped Provider succeeded. +func (b *CircuitBreaker) recordSuccessLocked(wasHalfOpen bool) { + b.consecutiveFailures = 0 + if wasHalfOpen { + b.state = BreakerClosed + b.logger.Info("llm: circuit breaker recovered — provider call succeeded, resuming normal operation") + } +} diff --git a/services/agent/internal/loop/runner.go b/services/agent/internal/loop/runner.go index 986acb7..a4f4ff2 100644 --- a/services/agent/internal/loop/runner.go +++ b/services/agent/internal/loop/runner.go @@ -8,6 +8,7 @@ package loop import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "time" @@ -174,6 +175,17 @@ func (r *Runner) processLateDeposit(ctx context.Context, dep *ledger.Deposit, no llm.NewHandler(toolCtx), ) if err != nil { + if errors.Is(err, llm.ErrCircuitOpen) { + // A tripped breaker (services/agent/internal/llm/breaker.go) + // must never block or delay the decision the deterministic + // hard-rule envelope already made — see + // runFallbackDecision's doc comment. This is the intended, + // safe behavior, not a failure to report up as a tick + // error. + logger.Warn("llm circuit breaker open, executing hard-rule envelope directly", "decision", envelope.Decision) + r.runFallbackDecision(ctx, toolCtx, envelope, logger) + return nil + } return fmt.Errorf("LLM conversation: %w", err) } @@ -181,6 +193,48 @@ func (r *Runner) processLateDeposit(ctx context.Context, dep *ledger.Deposit, no return nil } +// runFallbackDecision executes the already-computed, deterministic +// hard-rule envelope directly — bypassing the LLM conversation +// entirely — by calling the exact same tool handler +// (llm.NewHandler(toolCtx)) the LLM itself would call, with a canned +// reason/message instead of an LLM-authored one. +// +// This is what makes short-circuiting on llm.ErrCircuitOpen safe +// rather than merely fast: per docs/AGENT-DESIGN.md's "LLM decides +// phrasing only" guarantee, envelope.MaxGrantMinorUnits is already the +// safe ceiling (policy.ComputeBridgeEnvelope, computed and recorded +// via RecordAgentDecision before the LLM was ever consulted — see +// above) — the LLM's only real job this tick was choosing how much of +// that ceiling to execute and how to phrase it to the tenant. A tripped +// breaker means that choice can't be made by the LLM right now, so +// this executes the full ceiling deterministically instead of leaving +// safe, already-decided money movement blocked on a stuck provider. +// Every call here goes through the identical tool-handler code path +// (and therefore the identical gateway/ledger-core re-validation and +// RecordAgentDecision audit trail) a real LLM tool call would use. +func (r *Runner) runFallbackDecision(ctx context.Context, toolCtx llm.ToolContext, envelope policy.Envelope, logger *slog.Logger) { + handler := llm.NewHandler(toolCtx) + + if envelope.Decision != policy.DecisionDeclined && envelope.MaxGrantMinorUnits > 0 { + input, _ := json.Marshal(map[string]any{ + "amount_minor_units": envelope.MaxGrantMinorUnits, + "reason": "circuit breaker fallback: LLM provider unavailable, granting the full hard-rule envelope ceiling deterministically", + }) + result, isErr := handler(ctx, llm.ToolRationBridgeFunding, input) + if isErr { + logger.Error("circuit breaker fallback: ration_bridge_funding call failed", "result", result) + } else { + logger.Info("circuit breaker fallback: ration_bridge_funding executed", "result", result) + } + } + + message := fmt.Sprintf("Automated decision (LLM temporarily unavailable): %s — %s", envelope.Decision, envelope.Reason) + notifyInput, _ := json.Marshal(map[string]any{"message": message}) + if result, isErr := handler(ctx, llm.ToolNotifyTenant, notifyInput); isErr { + logger.Error("circuit breaker fallback: notify_tenant call failed", "result", result) + } +} + // hasOpenGrantWithinWindow implements the velocity-cap data gathering // described in docs/AGENT-DESIGN.md: a prior RATION_BRIDGE grant inside // the rolling window blocks a new grant unless its target deposit has