predikt/asp/test/resolution.test.ts
caelum0x 3cc4fd4caf fix: adversarial review of expansion — SSRF guard, webhook scoping, sweeper concurrency (294 tests)
Second adversarial review pass, this time over the expansion modules
(comments, webhooks, search, resolution, SDK). 9 findings, all verified real,
zero false positives:

- CRITICAL SSRF: the webhook dispatcher POSTed to any user-supplied URL,
  letting an authed caller hit cloud-metadata (169.254.169.254) and internal
  hosts, and use delivery outcome as a port-scan side channel. Added
  src/webhooks/ssrf.ts (blocks loopback/link-local/RFC-1918/metadata for IPv4,
  IPv6, and mapped forms) wired into webhook creation; residual DNS-rebinding
  risk documented.
- HIGH: new webhooks received events that predated their subscription (global
  high-water mark). Added per-subscription start_event_id with migration.
- HIGH: ResolutionSweeper had no concurrent-tick guard → duplicate AI calls
  when the provider is slow. Added the running-flag guard.
- MEDIUM: webhook-create rate limit (10/hr), atomic dispatcher writes,
  resolution force-flag body parsing without Content-Type; LOW: x402 signer
  browser base64 via TextEncoder.

tsc clean; 288 -> 294 tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-15 01:23:10 +03:00

377 lines
12 KiB
TypeScript

// Auto-resolution pipeline: the sweeper closes overdue markets and stores
// validated AI suggestions (BINARY + MULTI, including answer-text matching and
// the unmatched -> UNCLEAR path), tracks failures with a retry cap, and the
// routes apply decisive suggestions under a confidence gate. The completion
// function is injected and deterministic — no network.
import { Hono } from 'hono'
import { beforeEach, describe, expect, it } from 'vitest'
import { openDb, type Db } from '../src/engine/store'
import { MarketService, type Account } from '../src/engine/service'
import type { ChatCompletionFn } from '../src/ai/openrouter'
import {
getAttemptCount,
getSuggestion,
MAX_ATTEMPTS,
ResolutionSweeper,
} from '../src/resolution/sweeper'
import { createResolutionRoutes } from '../src/routes/resolution'
let db: Db
let svc: MarketService
let alice: Account
let aliceKey: string
let bob: Account
let bobKey: string
const FUTURE = () => Date.now() + 7 * 24 * 60 * 60 * 1000
beforeEach(() => {
db = openDb(':memory:')
svc = new MarketService(db)
const a = svc.createAccount('alice-agent')
alice = a.account
aliceKey = a.apiKey
const b = svc.createAccount('bob-agent')
bob = b.account
bobKey = b.apiKey
})
function makeMarket(overrides: Record<string, unknown> = {}) {
return svc.createMarket(alice.id, {
question: 'Will BTC close above $150k on Dec 31, 2026?',
criteria: 'Resolves YES on a CoinGecko daily close above $150,000.',
closeTime: FUTURE(),
subsidy: 100,
...overrides,
})
}
// Backdate a market's close time so the sweeper treats it as overdue.
function overdue(marketId: string): void {
db.prepare('UPDATE markets SET close_time = ? WHERE id = ?').run(
Date.now() - 60_000,
marketId
)
}
// A completion fn that answers based on the market question in the prompt.
function responder(fn: (userText: string) => string): ChatCompletionFn {
return async ({ messages }) => {
const user = messages.find((m) => m.role === 'user')?.content ?? ''
return fn(user)
}
}
const YES_HIGH = JSON.stringify({
verdict: 'YES',
confidence: 0.92,
rationale: 'The close was above the threshold on the resolution date.',
citations: ['coingecko.com'],
})
const NO_LOW = JSON.stringify({
verdict: 'NO',
confidence: 0.4,
rationale: 'Evidence is thin and mixed.',
citations: [],
})
function sweeper(complete: ChatCompletionFn): ResolutionSweeper {
return new ResolutionSweeper({ db, service: svc, complete })
}
describe('ResolutionSweeper.tick', () => {
it('closes overdue OPEN markets and stores a validated BINARY suggestion', async () => {
const market = makeMarket()
overdue(market.id)
const result = await sweeper(responder(() => YES_HIGH)).tick()
expect(result.closed).toBe(1)
expect(result.suggested).toBe(1)
expect(svc.getMarket(market.id).status).toBe('CLOSED')
const stored = getSuggestion(db, market.id)!
expect(stored.verdict).toBe('YES')
expect(stored.confidence).toBeCloseTo(0.92, 6)
expect(stored.answerId).toBeNull()
expect(stored.citations).toEqual(['coingecko.com'])
})
it('matches a MULTI ANSWER verdict to an answer id by text', async () => {
const market = makeMarket({
question: 'Which team wins the final?',
outcomeType: 'MULTI',
answers: ['Red Team', 'Blue Team', 'Draw'],
})
overdue(market.id)
const winner = market.answers!.find((a) => a.text === 'Blue Team')!
await sweeper(
responder(() =>
JSON.stringify({
verdict: 'ANSWER',
answer: 'blue team', // case-insensitive match
confidence: 0.81,
rationale: 'Blue Team won 3-1.',
citations: [],
})
)
).tick()
const stored = getSuggestion(db, market.id)!
expect(stored.verdict).toBe('ANSWER')
expect(stored.answerId).toBe(winner.id)
})
it('maps an unmatched MULTI answer to UNCLEAR with a note', async () => {
const market = makeMarket({
question: 'Which team wins the final?',
outcomeType: 'MULTI',
answers: ['Red Team', 'Blue Team'],
})
overdue(market.id)
await sweeper(
responder(() =>
JSON.stringify({
verdict: 'ANSWER',
answer: 'Green Team', // no such answer
confidence: 0.7,
rationale: 'Ambiguous source.',
citations: [],
})
)
).tick()
const stored = getSuggestion(db, market.id)!
expect(stored.verdict).toBe('UNCLEAR')
expect(stored.answerId).toBeNull()
expect(stored.rationale).toContain('did not match')
})
it('does not touch markets that are not yet overdue', async () => {
const market = makeMarket() // closes in the future
const result = await sweeper(responder(() => YES_HIGH)).tick()
expect(result.closed).toBe(0)
expect(result.suggested).toBe(0)
expect(getSuggestion(db, market.id)).toBeNull()
expect(svc.getMarket(market.id).status).toBe('OPEN')
})
it('retries on malformed AI output and stops after MAX_ATTEMPTS', async () => {
const market = makeMarket()
overdue(market.id)
const bad = sweeper(responder(() => 'not json at all'))
for (let i = 1; i <= MAX_ATTEMPTS; i += 1) {
const result = await bad.tick()
expect(result.failed).toBe(1)
expect(getAttemptCount(db, market.id)).toBe(i)
expect(getSuggestion(db, market.id)).toBeNull()
}
// Cap reached: the market is no longer a candidate.
const afterCap = await bad.tick()
expect(afterCap.failed).toBe(0)
expect(afterCap.suggested).toBe(0)
})
it('guards against overlapping ticks (no duplicate AI calls)', async () => {
const market = makeMarket()
overdue(market.id)
let calls = 0
let release!: () => void
const gate = new Promise<void>((resolve) => {
release = resolve
})
// A slow AI call: it blocks until we release the gate, letting a second
// tick start while the first is still in flight.
const complete: ChatCompletionFn = async () => {
calls += 1
await gate
return YES_HIGH
}
const s = sweeper(complete)
const first = s.tick() // begins, running = true, then awaits the AI call
const second = await s.tick() // must short-circuit on the running guard
expect(second).toEqual({ closed: 0, suggested: 0, failed: 0 })
expect(calls).toBe(1) // the overlapping tick did NOT issue a second AI call
release()
const firstResult = await first
expect(firstResult.suggested).toBe(1)
expect(calls).toBe(1)
expect(getSuggestion(db, market.id)!.verdict).toBe('YES')
})
it('is idempotent: a second tick re-suggests nothing', async () => {
const market = makeMarket()
overdue(market.id)
const s = sweeper(responder(() => YES_HIGH))
await s.tick()
const second = await s.tick()
expect(second.suggested).toBe(0)
expect(second.closed).toBe(0)
expect(getSuggestion(db, market.id)!.verdict).toBe('YES')
})
})
describe('resolution routes', () => {
type Api = Hono
let api: Api
beforeEach(() => {
api = new Hono()
api.route('/', createResolutionRoutes({ service: svc, db }))
})
function req(
path: string,
options: { method?: string; body?: unknown; key?: string } = {}
) {
return api.request(path, {
method: options.method ?? (options.body === undefined ? 'GET' : 'POST'),
headers: {
'Content-Type': 'application/json',
...(options.key ? { Authorization: `Bearer ${options.key}` } : {}),
},
...(options.body === undefined
? {}
: { body: JSON.stringify(options.body) }),
})
}
async function json(res: Response) {
return (await res.json()) as { success: boolean; data?: any; error?: string }
}
async function sweepWith(complete: ChatCompletionFn, marketId: string) {
overdue(marketId)
await sweeper(complete).tick()
}
it('404s the suggestion endpoint until a suggestion exists', async () => {
const market = makeMarket()
expect((await req(`/markets/${market.id}/resolution-suggestion`)).status).toBe(404)
expect((await req('/markets/mkt_missing/resolution-suggestion')).status).toBe(404)
await sweepWith(responder(() => YES_HIGH), market.id)
const res = await req(`/markets/${market.id}/resolution-suggestion`)
expect(res.status).toBe(200)
expect((await json(res)).data.suggestion.verdict).toBe('YES')
})
it('applies a confident suggestion and pays out the winner', async () => {
const market = makeMarket()
const trade = svc.buy(bob.id, market.id, 'YES', 50)
await sweepWith(responder(() => YES_HIGH), market.id)
const res = await req(`/markets/${market.id}/resolve-suggested`, {
key: aliceKey,
body: {},
})
expect(res.status).toBe(200)
const body = await json(res)
expect(body.data.market.status).toBe('RESOLVED')
expect(body.data.market.outcome).toBe('YES')
// Bob's YES shares each pay 1 credit.
expect(svc.getAccount(bob.id).balance).toBeCloseTo(1000 - 50 + trade.shares, 4)
})
it('rejects a low-confidence suggestion with 409 unless forced', async () => {
const market = makeMarket()
await sweepWith(responder(() => NO_LOW), market.id)
const gated = await req(`/markets/${market.id}/resolve-suggested`, {
key: aliceKey,
body: {},
})
expect(gated.status).toBe(409)
const gatedBody = await json(gated)
expect(gatedBody.data.suggestion.confidence).toBeCloseTo(0.4, 6)
expect(svc.getMarket(market.id).status).toBe('CLOSED') // not resolved
const forced = await req(`/markets/${market.id}/resolve-suggested`, {
key: aliceKey,
body: { force: true },
})
expect(forced.status).toBe(200)
expect((await json(forced)).data.market.outcome).toBe('NO')
})
it('honors force even when the body is sent without a JSON content-type', async () => {
const market = makeMarket()
await sweepWith(responder(() => NO_LOW), market.id) // confidence 0.4, below gate
// A client sends { force: true } but omits Content-Type: application/json.
// The previous `c.req.json().catch(() => ({}))` swallowed this into {},
// silently ignoring force and returning 409. It must now be honored.
const res = await api.request(`/markets/${market.id}/resolve-suggested`, {
method: 'POST',
headers: {
'Content-Type': 'text/plain',
Authorization: `Bearer ${aliceKey}`,
},
body: JSON.stringify({ force: true }),
})
expect(res.status).toBe(200)
expect((await json(res)).data.market.outcome).toBe('NO')
})
it('rejects a genuinely malformed body with 400', async () => {
const market = makeMarket()
await sweepWith(responder(() => YES_HIGH), market.id)
const res = await api.request(`/markets/${market.id}/resolve-suggested`, {
method: 'POST',
headers: { Authorization: `Bearer ${aliceKey}` },
body: '{ not json',
})
expect(res.status).toBe(400)
expect(svc.getMarket(market.id).status).toBe('CLOSED') // not resolved
})
it('never auto-applies an UNCLEAR suggestion, even forced', async () => {
const market = makeMarket()
await sweepWith(
responder(() =>
JSON.stringify({
verdict: 'UNCLEAR',
confidence: 0.99,
rationale: 'Cannot determine from available evidence.',
citations: [],
})
),
market.id
)
for (const body of [{}, { force: true }]) {
const res = await req(`/markets/${market.id}/resolve-suggested`, {
key: aliceKey,
body,
})
expect(res.status).toBe(409)
}
expect(svc.getMarket(market.id).status).toBe('CLOSED')
})
it('only the creator can apply, and auth is required', async () => {
const market = makeMarket()
await sweepWith(responder(() => YES_HIGH), market.id)
expect(
(await req(`/markets/${market.id}/resolve-suggested`, { body: {} })).status
).toBe(401)
const foreign = await req(`/markets/${market.id}/resolve-suggested`, {
key: bobKey,
body: {},
})
expect(foreign.status).toBe(403)
expect(svc.getMarket(market.id).status).toBe('CLOSED')
})
})