ration/pkg/telemetry/interceptors.go
caelum0x 13abc31645 feat(ration): observability, infra-as-code, security hardening, and test depth
Phase 2 sections 2-6 and 8 of docs/SCALE-EXPANSION.md:

- pkg/telemetry + pkg/metrics: OpenTelemetry trace propagation and
  Prometheus metrics across every service hop (CLI -> agent -> gateway ->
  card-issuance/tenant-registry/auth -> ledger-core), tracing added to
  ledger-core itself via services/ledger-core/src/telemetry.rs.
- infra/: otel-collector + Prometheus + Grafana docker-compose stack, a
  Terraform ECS Fargate module + dev/staging/prod envs (plan-only, no
  apply — real AWS spend is an owner action), and a dev-only mTLS CA
  (infra/dev-ca) wired through pkg/mtls + services/ledger-core/src/mtls.rs.
- services/audit-log (from the prior commit) gets its async mirror writer
  in services/ledger-core/src/audit_mirror.rs, and services/gateway gets
  a per-tenant token-bucket rate limiter (internal/ratelimit).
- Test depth: a Rust proptest suite proving concurrent ApplyEntry never
  breaches the reserve floor (tests/apply_entry_proptest.rs, passing), a Go
  fuzz seed corpus for the policy envelope (rules_fuzz_test.go), a libFuzzer
  harness for the C WAL frame parser (services/wal/fuzz, deliberately not
  wired into `make test` per its own Makefile comment - needs a
  libFuzzer-bundled clang), and a real chaos test
  (scripts/chaos/kill-ledger-mid-write.sh) that SIGKILLs ledger-core
  mid-write under concurrent load and verifies via ListEntries that
  nothing was lost or double-applied.
- docs/formal/ledger.tla: a TLA+ spec of the ledger state machine (balance
  >= reserve_floor safety, HOLD-eventually-resolves liveness) plus a TLC
  model config - not run through TLC in this pass, that's a precise
  documented TODO, not faked.
- docs/adr/0001-0004, docs/RUNBOOK.md, docs/SLO.md, docs/THREAT-MODEL.md.

Verified: `make build && make test` clean (C WAL, Rust incl. the new
proptest suite, all 10 Go modules) from a cold target.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-19 12:25:58 +03:00

125 lines
4.7 KiB
Go

package telemetry
import (
"context"
"log/slog"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
// UnaryServerInterceptor returns a grpc.UnaryServerInterceptor that:
// 1. extracts an incoming x-ration-correlation-id (from a caller that
// already has one, e.g. gateway forwarding a cli-issued ID to
// ledger-core) or mints a fresh one if this RPC is the root of a new
// unit of work;
// 2. stores it on the request context (retrievable via
// CorrelationIDFromContext for use in RecordAgentDecision audit
// trails and application logs);
// 3. logs one structured JSON line per RPC (method, duration, error,
// correlation_id, trace_id/span_id) via logger.
//
// Combine with ServerOption (grpc.StatsHandler) for tracing — this
// interceptor handles correlation IDs and logging, the stats handler
// handles the W3C trace context; both run on every RPC.
func UnaryServerInterceptor(logger *slog.Logger) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
ctx = ensureCorrelationID(ctx)
start := time.Now()
resp, err := handler(ctx, req)
logRPC(logger, ctx, info.FullMethod, time.Since(start), err)
return resp, err
}
}
// StreamServerInterceptor is StreamServerInterceptor's streaming
// counterpart, used for gateway.proto's StreamActivity RPC. The
// correlation ID and per-RPC log line cover the stream's full lifetime
// (open to close), not each message.
func StreamServerInterceptor(logger *slog.Logger) grpc.StreamServerInterceptor {
return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
ctx := ensureCorrelationID(ss.Context())
wrapped := &correlationServerStream{ServerStream: ss, ctx: ctx}
start := time.Now()
err := handler(srv, wrapped)
logRPC(logger, ctx, info.FullMethod, time.Since(start), err)
return err
}
}
// correlationServerStream overrides grpc.ServerStream.Context() so
// handler code (and anything it calls) observes the correlation-ID-
// bearing context rather than the raw stream context.
type correlationServerStream struct {
grpc.ServerStream
ctx context.Context
}
func (s *correlationServerStream) Context() context.Context { return s.ctx }
// UnaryClientInterceptor returns a grpc.UnaryClientInterceptor that
// ensures the outgoing context carries a correlation ID (minting one if
// the caller — typically a CLI command's root context — never set one)
// and propagates it to the callee via gRPC metadata.
func UnaryClientInterceptor() grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
ctx = injectCorrelationID(ctx)
return invoker(ctx, method, req, reply, cc, opts...)
}
}
// StreamClientInterceptor is UnaryClientInterceptor's streaming
// counterpart (used when dialing gateway's StreamActivity from the TUI).
func StreamClientInterceptor() grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
ctx = injectCorrelationID(ctx)
return streamer(ctx, desc, cc, method, opts...)
}
}
// ensureCorrelationID reads x-ration-correlation-id out of incoming gRPC
// metadata if present, otherwise mints a new one, and returns a context
// carrying it (for CorrelationIDFromContext).
func ensureCorrelationID(ctx context.Context) context.Context {
if md, ok := metadata.FromIncomingContext(ctx); ok {
if vals := md.Get(CorrelationIDHeader); len(vals) > 0 && vals[0] != "" {
return ContextWithCorrelationID(ctx, vals[0])
}
}
return ContextWithCorrelationID(ctx, NewCorrelationID())
}
// injectCorrelationID ensures ctx carries a correlation ID (minting one
// if absent) and attaches it to outgoing gRPC metadata so the next hop's
// server interceptor picks it up.
func injectCorrelationID(ctx context.Context) context.Context {
id := CorrelationIDFromContext(ctx)
if id == "" {
id = NewCorrelationID()
ctx = ContextWithCorrelationID(ctx, id)
}
return metadata.AppendToOutgoingContext(ctx, CorrelationIDHeader, id)
}
func logRPC(logger *slog.Logger, ctx context.Context, method string, dur time.Duration, err error) {
if logger == nil {
return
}
attrs := append(LogAttrs(ctx),
slog.String("grpc_method", method),
slog.Duration("duration", dur),
)
if err != nil {
attrs = append(attrs, slog.String("error", err.Error()), slog.String("grpc_code", status.Code(err).String()))
logger.LogAttrs(ctx, slog.LevelWarn, "rpc failed", attrs...)
return
}
logger.LogAttrs(ctx, slog.LevelInfo, "rpc completed", attrs...)
}