Compare commits

...

17 Commits

Author SHA1 Message Date
miguel
50bd61574f Improve verifier trajectory evidence capture 2026-05-22 16:04:33 -07:00
miguel
25fadb1426 refactor(verifier): centralize trajectory evidence handling 2026-05-22 13:54:24 -07:00
miguel
2780db25f2 fix(verifier): redact inline screenshot payloads 2026-05-22 13:37:55 -07:00
miguel
2ba6c1f2bb test(agent): drop stale bus mocks 2026-05-22 13:09:18 -07:00
miguel
8596d2aedd fix(evals): avoid useless task data assignment 2026-05-22 13:09:18 -07:00
miguel
043b3e1b66 fix(verifier): hydrate persisted agent image paths 2026-05-22 13:09:18 -07:00
miguel
b493fa7fdf fix(cua): keep screenshot provider evidence non-fatal 2026-05-22 13:09:18 -07:00
miguel
16669e80d7 refactor(verifier): collect evidence via agent callbacks 2026-05-22 13:09:18 -07:00
miguel
6caeb1bac5 fix(verifier): guard bus.listenerCount and align export-surface snapshot
- v3AgentHandler / v3CuaAgentHandler use optional-chained listenerCount
  so test mocks without one (captcha-hooks, temperature) don't blow up.
- Add bus stub to the agent-temperature createV3() mock so bus.emit
  doesn't NPE on the new agent_step_finished_event emit.
- Add BUS_EVENTS, shouldPersistTrajectory, writeTrajectoryDir to the
  export-surface snapshot — these are intentional new public exports.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 13:09:18 -07:00
miguel
10b03ca738 style(recorder): prettier — collapse runId fallback onto one line
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 13:09:18 -07:00
miguel
1303315cc9 refactor(verifier): extract writeTrajectoryDir + shouldPersistTrajectory
Lift the on-disk persistence helpers from TrajectoryRecorder into
verifier/trajectory.ts so #2137's harness adapter can share them. Also
drop the recorder's no-op .replace("T","T") and the WHAT-narration
comments per project policy.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 13:09:18 -07:00
miguel
9138ddf88a docs(verifier): trim trajectory event comments 2026-05-22 13:09:18 -07:00
miguel
bb514e3ab5 test(evals): cover trajectory recorder in vitest 2026-05-22 13:09:18 -07:00
miguel
8e9962cc78 docs(verifier): remove rollout comments from trajectory capture 2026-05-22 13:09:18 -07:00
miguel
c25367bbcf chore(evals): remove upstream trajectory references 2026-05-22 13:09:18 -07:00
miguel
40e7ab30b8 fix(verifier): align trajectory naming 2026-05-22 13:09:18 -07:00
miguel
6f27af2528 feat(verifier): record agent trajectories 2026-05-22 13:09:18 -07:00
25 changed files with 2061 additions and 47 deletions

View File

@ -0,0 +1,5 @@
---
"@browserbasehq/stagehand": patch
---
Capture verifier trajectory evidence from agent evidence callbacks for offline scoring.

View File

@ -0,0 +1,73 @@
/**
* captureAriaTreeProbe — capture a truncated accessibility tree of the active
* page for use as tier-2 evidence in the trajectory recorder.
*
* Shared by v3AgentHandler and v3CuaAgentHandler. Listener-gated by the
* callers so ordinary agent runs (no TrajectoryRecorder attached) don't pay
* the cost.
*
* The a11y tree is the same payload the agent's `ariaTree` tool sees, but
* captured by the harness (not the agent) so the verifier has independent
* textual ground truth for grounding non-visual claims — prices, names,
* dates, list contents — without OCR'ing screenshots.
*
* Budget: defaults to ~8000 tokens (32k chars). Per-step a11y captures
* across a ~30-step trajectory at that cap sum to ~240k tokens total,
* which the verifier handles via per-criterion top-K selection. The cap
* is configurable via VERIFIER_ARIATREE_TOKEN_BUDGET so consumers can
* trade RAM/disk for fidelity. Truncated content is marked explicitly so
* the verifier knows it was clipped.
*/
import type { V3 } from "../../v3.js";
const APPROX_CHARS_PER_TOKEN = 4;
const DEFAULT_TOKEN_BUDGET = 8_000;
const DEFAULT_TIMEOUT_MS = 5_000;
interface CaptureAriaTreeOptions {
/** Soft cap on token count (chars/4 approximation). Default 8000. */
tokenBudget?: number;
/** Hard timeout on the capture. Default 5s. */
timeoutMs?: number;
}
/**
* Returns the truncated a11y tree as a plain string, or undefined when
* capture fails. Never throws — a11y capture is best-effort tier-2 evidence,
* not a hard requirement, so failures are silently absorbed (the verifier
* surfaces this via evidence_insufficient).
*/
export async function captureAriaTreeProbe(
v3: V3,
opts: CaptureAriaTreeOptions = {},
): Promise<string | undefined> {
const envBudget = parseInt(
process.env.VERIFIER_ARIATREE_TOKEN_BUDGET ?? "",
10,
);
const tokenBudget =
opts.tokenBudget ??
(Number.isFinite(envBudget) && envBudget > 0
? envBudget
: DEFAULT_TOKEN_BUDGET);
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const maxChars = tokenBudget * APPROX_CHARS_PER_TOKEN;
try {
// v3.extract() without a schema returns { pageText } where pageText is the
// rendered accessibility tree — same path the agent's ariaTree tool uses.
const result = await v3.extract({ timeout: timeoutMs });
const pageText = result?.pageText;
if (typeof pageText !== "string" || pageText.length === 0) return undefined;
if (pageText.length > maxChars) {
return (
pageText.slice(0, maxChars) +
`\n\n[CONTENT TRUNCATED at ~${tokenBudget} tokens — set VERIFIER_ARIATREE_TOKEN_BUDGET to raise]`
);
}
return pageText;
} catch {
return undefined;
}
}

View File

@ -0,0 +1,55 @@
import type { AgentScreenshotEvidenceEvent } from "../../types/public/agentEvidenceEvents.js";
export interface PairedCuaActionStep {
stepIndex: number;
replayScreenshot?: AgentScreenshotEvidenceEvent;
}
export class CuaEvidenceStepTracker {
private nextStepIndex = 0;
private latestScreenshot?: AgentScreenshotEvidenceEvent;
private latestScreenshotConsumed = true;
reset(): void {
this.nextStepIndex = 0;
this.latestScreenshot = undefined;
this.latestScreenshotConsumed = true;
}
recordScreenshot(
screenshot: Buffer,
url: string,
): AgentScreenshotEvidenceEvent {
const event: AgentScreenshotEvidenceEvent = {
type: "screenshot",
stepIndex: this.nextStepIndex++,
screenshot,
url,
evidenceRole: "agent",
};
this.latestScreenshot = event;
this.latestScreenshotConsumed = false;
return event;
}
pairAction(): PairedCuaActionStep {
if (this.latestScreenshot && !this.latestScreenshotConsumed) {
this.latestScreenshotConsumed = true;
return { stepIndex: this.latestScreenshot.stepIndex };
}
const stepIndex = this.nextStepIndex++;
if (this.latestScreenshot) {
return {
stepIndex,
replayScreenshot: { ...this.latestScreenshot, stepIndex },
};
}
return { stepIndex };
}
get latestScreenshotUrl(): string | undefined {
return this.latestScreenshot?.url;
}
}

View File

@ -0,0 +1,90 @@
import type { AgentEvidenceCallback } from "../../types/public/agentEvidenceEvents.js";
import type { LogLine } from "../../types/public/logs.js";
import type { V3 } from "../../v3.js";
import { captureAriaTreeProbe } from "./captureAriaTreeProbe.js";
interface CaptureProbeEvidenceOptions {
v3: V3;
url: string;
logger: (message: LogLine) => void;
warningMessage: string;
}
interface EmitPostStepProbeEvidenceOptions extends CaptureProbeEvidenceOptions {
stepIndices: number | number[];
evidenceCallback?: AgentEvidenceCallback;
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export async function captureProbeEvidence({
v3,
url,
logger,
warningMessage,
}: CaptureProbeEvidenceOptions): Promise<{
url: string;
screenshot?: Buffer;
ariaTree?: string;
}> {
let probeUrl = url;
let screenshot: Buffer | undefined;
try {
const page = await v3.context.awaitActivePage();
probeUrl = page.url();
screenshot = await page.screenshot({ fullPage: false });
} catch (e) {
logger({
category: "agent",
message: `${warningMessage}: ${errorMessage(e)}`,
level: 1,
});
}
const ariaTree = await captureAriaTreeProbe(v3);
return {
url: probeUrl,
...(screenshot ? { screenshot } : {}),
...(ariaTree !== undefined ? { ariaTree } : {}),
};
}
export async function emitPostStepProbeEvidence({
v3,
stepIndices,
url,
evidenceCallback,
logger,
warningMessage,
}: EmitPostStepProbeEvidenceOptions): Promise<void> {
if (!evidenceCallback) return;
const indices = Array.isArray(stepIndices) ? stepIndices : [stepIndices];
if (indices.length === 0) return;
const probe = await captureProbeEvidence({
v3,
url,
logger,
warningMessage,
});
for (const stepIndex of indices) {
if (probe.screenshot) {
await evidenceCallback({
type: "screenshot",
stepIndex,
screenshot: probe.screenshot,
url: probe.url,
evidenceRole: "probe",
});
}
await evidenceCallback({
type: "step_observed",
stepIndex,
url: probe.url,
ariaTree: probe.ariaTree,
});
}
}

View File

@ -0,0 +1,76 @@
import type { AgentStepFinishedEvent } from "../../types/public/agentEvidenceEvents.js";
const ERROR_STRING_LIMIT = 1000;
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function hasOwn(value: Record<string, unknown>, key: string): boolean {
return Object.prototype.hasOwnProperty.call(value, key);
}
function normalizeError(value: unknown): string | undefined {
if (value === undefined || value === null || value === false) {
return undefined;
}
if (value instanceof Error) {
return value.message;
}
if (typeof value === "string") {
return value;
}
if (
typeof value === "number" ||
typeof value === "boolean" ||
typeof value === "bigint"
) {
return String(value);
}
let serialized: string;
try {
serialized = JSON.stringify(value) ?? String(value);
} catch {
serialized = String(value);
}
if (serialized.length <= ERROR_STRING_LIMIT) {
return serialized;
}
return `${serialized.slice(0, ERROR_STRING_LIMIT)}... [truncated]`;
}
function statusCandidates(toolResult: unknown): Record<string, unknown>[] {
if (!isRecord(toolResult)) {
return [];
}
const candidates = [toolResult];
const output = toolResult.output;
if (isRecord(output)) {
candidates.push(output);
}
return candidates;
}
export function inferToolOutput(
toolResult: unknown,
): AgentStepFinishedEvent["toolOutput"] {
const candidates = statusCandidates(toolResult);
const error = candidates
.map((candidate) =>
hasOwn(candidate, "error") ? normalizeError(candidate.error) : undefined,
)
.find((message): message is string => message !== undefined);
const successFalse = candidates.some(
(candidate) => candidate.success === false,
);
const isError = candidates.some((candidate) => Boolean(candidate.isError));
return {
ok: error === undefined && !isError && !successFalse,
result: toolResult,
error,
};
}

View File

@ -31,6 +31,7 @@ import {
AgentModelConfig,
Variables,
} from "../types/public/agent.js";
import type { AgentEvidenceCallback } from "../types/public/agentEvidenceEvents.js";
import { HYBRID_CAPABLE_MODEL_PATTERNS } from "../types/private/agent.js";
import { V3FunctionName } from "../types/public/methods.js";
import { mapToolResultToActions } from "../agent/utils/actionMapping.js";
@ -41,6 +42,11 @@ import {
AgentAbortError,
} from "../types/public/sdkErrors.js";
import { handleDoneToolCall } from "../agent/utils/handleDoneToolCall.js";
import {
captureProbeEvidence,
emitPostStepProbeEvidence,
} from "../agent/utils/postStepProbeEvidence.js";
import { inferToolOutput } from "../agent/utils/toolOutputEvidence.js";
import {
CaptchaSolver,
CAPTCHA_SOLVED_MSG,
@ -51,6 +57,19 @@ function getErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
type FinalAnswerDraft = {
message: string;
output?: Record<string, unknown>;
};
interface StepHandlerOptions {
userCallback?:
| GenerateTextOnStepFinishCallback<ToolSet>
| StreamTextOnStepFinishCallback<ToolSet>;
evidenceCallback?: AgentEvidenceCallback;
onFinalAnswer?: (answer: FinalAnswerDraft) => void;
}
/**
* Prepends a system message with cache control to the messages array.
* The cache control providerOptions are used by Anthropic and ignored by other providers.
@ -244,10 +263,16 @@ export class V3AgentHandler {
private createStepHandler(
state: AgentState,
userCallback?:
| GenerateTextOnStepFinishCallback<ToolSet>
| StreamTextOnStepFinishCallback<ToolSet>,
{
userCallback,
evidenceCallback,
onFinalAnswer,
}: StepHandlerOptions,
) {
// Monotonic step counter scoped to this execute() call. Each tool call in
// the agent loop becomes one trajectory step. The counter feeds stepIndex
// on evidence callback events.
let stepCounter = 0;
return async (event: StepResult<ToolSet>) => {
this.logger({
category: "agent",
@ -255,6 +280,9 @@ export class V3AgentHandler {
level: 2,
});
const stepIndicesInTurn: number[] = [];
let lastFinalAnswer: FinalAnswerDraft | undefined;
if (event.toolCalls && event.toolCalls.length > 0) {
for (let i = 0; i < event.toolCalls.length; i++) {
const toolCall = event.toolCalls[i];
@ -279,6 +307,13 @@ export class V3AgentHandler {
? `${allReasoning} ${doneReasoning}`.trim()
: allReasoning || "Task completed successfully";
}
lastFinalAnswer = {
message: state.finalMessage,
output:
typeof args?.output === "object" && args?.output !== null
? (args.output as Record<string, unknown>)
: undefined,
};
}
const mappedActions = mapToolResultToActions({
toolCallName: toolCall.toolName,
@ -292,8 +327,42 @@ export class V3AgentHandler {
action.timestamp = Date.now();
state.actions.push(action);
}
const stepIndex = stepCounter++;
stepIndicesInTurn.push(stepIndex);
const finishedAt = new Date().toISOString();
await evidenceCallback?.({
type: "step_finished",
stepIndex,
actionName: toolCall.toolName,
actionArgs:
typeof args === "object" && args !== null
? (args as Record<string, unknown>)
: {},
reasoning: event.text ?? "",
toolOutput: inferToolOutput(toolResult),
finishedAt,
});
}
state.currentPageUrl = (await this.v3.context.awaitActivePage()).url();
// Harness probe — take a single screenshot / a11y snapshot per AI SDK
// step and attach it to every tool call in that turn. The observation
// reflects the settled page state after the batch of tool calls; this
// is more faithful than dropping probe evidence for all but the last
// tool call, while still avoiding per-tool screenshot overhead.
await emitPostStepProbeEvidence({
v3: this.v3,
stepIndices: stepIndicesInTurn,
url: state.currentPageUrl,
evidenceCallback,
logger: this.logger,
warningMessage: "Warning: harness probe failed",
});
}
if (lastFinalAnswer) {
onFinalAnswer?.(lastFinalAnswer);
}
if (userCallback) {
@ -321,6 +390,7 @@ export class V3AgentHandler {
completed: false,
currentPageUrl: "",
};
let finalAnswerFromDoneTool: FinalAnswerDraft | undefined;
let messages: ModelMessage[] = [];
let captchaSolver: CaptchaSolver | undefined;
@ -379,7 +449,16 @@ export class V3AgentHandler {
callbacks?.prepareStep,
captchaSolver,
),
onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
onStepFinish: this.createStepHandler(
state,
{
userCallback: callbacks?.onStepFinish,
evidenceCallback: callbacks?.onEvidence,
onFinalAnswer: (answer) => {
finalAnswerFromDoneTool = answer;
},
},
),
abortSignal: preparedOptions.signal,
providerOptions: {
google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
@ -396,6 +475,15 @@ export class V3AgentHandler {
preparedOptions.output,
this.logger,
);
const output = doneResult.output ?? finalAnswerFromDoneTool?.output;
await this.emitFinalEvidence(
state,
{
message: state.finalMessage,
output,
},
callbacks?.onEvidence,
);
return this.consolidateMetricsAndResult(
startTime,
@ -403,7 +491,7 @@ export class V3AgentHandler {
doneResult.messages,
result,
maxSteps,
doneResult.output,
output,
);
} catch (error) {
// Re-throw validation errors that should propagate to the caller
@ -449,6 +537,7 @@ export class V3AgentHandler {
// Highlight cursor defaults to true for hybrid mode, can be overridden
const shouldHighlightCursor =
streamOptions?.highlightCursor ?? this.mode === "hybrid";
let finalAnswerFromDoneTool: FinalAnswerDraft | undefined;
const {
options,
@ -515,7 +604,16 @@ export class V3AgentHandler {
callbacks?.prepareStep,
captchaSolver,
),
onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
onStepFinish: this.createStepHandler(
state,
{
userCallback: callbacks?.onStepFinish,
evidenceCallback: callbacks?.onEvidence,
onFinalAnswer: (answer) => {
finalAnswerFromDoneTool = answer;
},
},
),
onError: (event) => {
captchaSolver?.dispose();
if (callbacks?.onError) {
@ -541,17 +639,29 @@ export class V3AgentHandler {
options.instruction,
options.output,
this.logger,
).then((doneResult) => {
const result = this.consolidateMetricsAndResult(
startTime,
state,
doneResult.messages,
event,
maxSteps,
doneResult.output,
);
resolveResult(result);
});
)
.then(async (doneResult) => {
const output =
doneResult.output ?? finalAnswerFromDoneTool?.output;
await this.emitFinalEvidence(
state,
{
message: state.finalMessage,
output,
},
callbacks?.onEvidence,
);
const result = this.consolidateMetricsAndResult(
startTime,
state,
doneResult.messages,
event,
maxSteps,
output,
);
resolveResult(result);
})
.catch(handleError);
},
onAbort: (event) => {
captchaSolver?.dispose();
@ -580,6 +690,26 @@ export class V3AgentHandler {
return agentStreamResult;
}
private async emitFinalEvidence(
state: AgentState,
finalAnswer: { message: string; output?: Record<string, unknown> },
evidenceCallback?: AgentEvidenceCallback,
): Promise<void> {
if (!evidenceCallback) return;
const observation = await captureProbeEvidence({
v3: this.v3,
url: state.currentPageUrl,
logger: this.logger,
warningMessage: "Warning: final harness probe failed",
});
await evidenceCallback({
type: "final_answer",
...finalAnswer,
observation,
});
}
private consolidateMetricsAndResult(
startTime: number,
state: AgentState,

View File

@ -7,6 +7,12 @@ import { GoogleCUAClient } from "../agent/GoogleCUAClient.js";
import { OpenAICUAClient } from "../agent/OpenAICUAClient.js";
import { mapKeyToPlaywright } from "../agent/utils/cuaKeyMapping.js";
import { ensureXPath } from "../agent/utils/xpath.js";
import {
captureProbeEvidence,
emitPostStepProbeEvidence,
} from "../agent/utils/postStepProbeEvidence.js";
import { CuaEvidenceStepTracker } from "../agent/utils/cuaEvidenceStepTracker.js";
import { inferToolOutput } from "../agent/utils/toolOutputEvidence.js";
import {
ActionExecutionResult,
AgentAction,
@ -16,6 +22,7 @@ import {
SafetyConfirmationHandler,
} from "../types/public/agent.js";
import { LogLine } from "../types/public/logs.js";
import type { AgentEvidenceCallback } from "../types/public/agentEvidenceEvents.js";
import { type Action, V3FunctionName } from "../types/public/methods.js";
import { FlowLogger } from "../flowlogger/FlowLogger.js";
import { toTitleCase } from "../../utils.js";
@ -37,6 +44,8 @@ export class V3CuaAgentHandler {
private captchaSolver: CaptchaSolver | null = null;
private captchaClickGuardRemaining = 0;
private currentInstruction = "";
private readonly cuaEvidenceSteps = new CuaEvidenceStepTracker();
private evidenceCallback?: AgentEvidenceCallback;
constructor(
v3: V3,
@ -76,6 +85,9 @@ export class V3CuaAgentHandler {
this.ensureNotClosed();
const page = await this.v3.context.awaitActivePage();
const screenshotBuffer = await page.screenshot({ fullPage: false });
await this.emitCuaScreenshotNonFatal(screenshotBuffer, page.url());
return screenshotBuffer.toString("base64"); // base64 png
});
@ -120,6 +132,8 @@ export class V3CuaAgentHandler {
(this.options.clientOptions?.waitBetweenActions as number) ||
defaultDelay;
try {
let executionResult: ActionExecutionResult | undefined;
const startedAt = new Date().toISOString();
// Try to inject cursor before each action if enabled
if (this.highlightCursor) {
try {
@ -133,7 +147,7 @@ export class V3CuaAgentHandler {
// takes its own screenshot via screenshotProvider between API turns.
const shouldLog = action.type !== "screenshot";
if (shouldLog) {
await FlowLogger.runWithLogging(
executionResult = await FlowLogger.runWithLogging(
{
eventType: `V3Cua${toTitleCase(action.type)}`, // e.g. "V3CuaClick"
data: {
@ -145,10 +159,13 @@ export class V3CuaAgentHandler {
[action],
);
} else {
await this.executeAction(action);
executionResult = await this.executeAction(action);
}
action.timestamp = Date.now();
if (shouldLog) {
await this.emitCuaActionStep(action, executionResult, startedAt);
}
await new Promise((r) => setTimeout(r, waitBetween));
} catch (error) {
@ -184,6 +201,8 @@ export class V3CuaAgentHandler {
: optionsOrInstruction;
this.setSafetyConfirmationHandler(options.callbacks?.onSafetyConfirmation);
this.evidenceCallback = options.callbacks?.onEvidence;
this.cuaEvidenceSteps.reset();
this.highlightCursor = options.highlightCursor !== false;
this.currentInstruction = options.instruction;
@ -239,7 +258,28 @@ export class V3CuaAgentHandler {
let result: AgentResult;
try {
result = await this.agent.execute({ options, logger: this.logger });
if (this.evidenceCallback) {
let finalUrl = "";
try {
finalUrl = (await this.v3.context.awaitActivePage()).url();
} catch {
finalUrl = this.cuaEvidenceSteps.latestScreenshotUrl ?? "";
}
const observation = await captureProbeEvidence({
v3: this.v3,
url: finalUrl,
logger: this.logger,
warningMessage: "Warning: CUA final probe failed",
});
await this.evidenceCallback({
type: "final_answer",
message: result.message,
output: result.output,
observation,
});
}
} finally {
this.evidenceCallback = undefined;
this.captchaSolver?.dispose();
this.captchaSolver = null;
}
@ -658,6 +698,10 @@ export class V3CuaAgentHandler {
const screenshotBuffer = await page.screenshot({ fullPage: false });
const currentUrl = page.url();
// Mirror the same buffer the CUA client receives as agent evidence.
await this.emitCuaScreenshotNonFatal(screenshotBuffer, currentUrl);
return await this.agentClient.captureScreenshot({
base64Image: screenshotBuffer.toString("base64"),
currentUrl,
@ -767,6 +811,94 @@ export class V3CuaAgentHandler {
}
}
/**
* Emit a pre-action CUA screenshot — the exact buffer the model received
* as input. Tier-1 evidence (agent-mirrored); the tier-2 probe is taken
* separately in emitCuaActionStep after the action runs, so the recorder
* can compare what the model saw against what the page actually showed
* once the keystrokes/clicks landed.
*/
private async emitCuaScreenshot(
screenshot: Buffer,
url: string,
): Promise<void> {
await this.evidenceCallback?.(
this.cuaEvidenceSteps.recordScreenshot(screenshot, url),
);
}
private async emitCuaScreenshotNonFatal(
screenshot: Buffer,
url: string,
): Promise<void> {
try {
await this.emitCuaScreenshot(screenshot, url);
} catch (e) {
this.logger({
category: "agent",
message: `Warning: CUA screenshot evidence callback failed: ${
e instanceof Error ? e.message : String(e)
}`,
level: 1,
});
}
}
private async emitCuaActionStep(
action: AgentAction,
result: ActionExecutionResult | undefined,
startedAt: string,
): Promise<void> {
let pageUrl =
typeof action.pageUrl === "string"
? action.pageUrl
: (this.cuaEvidenceSteps.latestScreenshotUrl ?? "");
try {
pageUrl = (await this.v3.context.awaitActivePage()).url();
} catch {
// Keep the best pre-action URL fallback.
}
const { stepIndex, replayScreenshot } = this.cuaEvidenceSteps.pairAction();
if (replayScreenshot) {
await this.evidenceCallback?.(replayScreenshot);
}
const actionArgs = Object.fromEntries(
Object.entries(action).filter(([key]) => key !== "screenshot"),
);
const reasoning =
typeof action.reasoning === "string"
? action.reasoning
: typeof action.action === "string"
? action.action
: "";
await this.evidenceCallback?.({
type: "step_finished",
stepIndex,
actionName: String(action.type),
actionArgs,
reasoning,
toolOutput: inferToolOutput(result ?? { success: true }),
startedAt,
finishedAt: new Date().toISOString(),
});
// Post-action tier-2 probe. The pre-action screenshot from
// screenshotProvider is what the model SAW; this one shows what the
// page actually LOOKS LIKE after the action ran. Without this the
// verifier has no visual evidence that keystrokes/clicks landed, and
// has to trust the action history alone.
await emitPostStepProbeEvidence({
v3: this.v3,
stepIndices: stepIndex,
url: pageUrl,
evidenceCallback: this.evidenceCallback,
logger: this.logger,
warningMessage: "Warning: CUA post-action probe failed",
});
}
private async injectCursor(): Promise<void> {
try {
const page = await this.v3.context.awaitActivePage();

View File

@ -25,9 +25,14 @@ import { getAISDKLanguageModel } from "./llm/LLMProvider.js";
import { __internalCreateInMemoryAgentCacheHandle } from "./cache/serverAgentCache.js";
import { maybeRunShutdownSupervisorFromArgv } from "./shutdown/supervisor.js";
import {
buildAgentEvidenceFromStepFinished,
loadTrajectoryFromDisk,
mergeAgentEvidence,
nextResultFilename,
normalizeRubric,
redactInlineImagePayloads,
shouldPersistTrajectory,
writeTrajectoryDir,
} from "./verifier/index.js";
export { V3 } from "./v3.js";
@ -90,9 +95,14 @@ export type {
VerifierRawSteps,
} from "./verifier/index.js";
export {
buildAgentEvidenceFromStepFinished,
loadTrajectoryFromDisk,
mergeAgentEvidence,
nextResultFilename,
normalizeRubric,
redactInlineImagePayloads,
shouldPersistTrajectory,
writeTrajectoryDir,
} from "./verifier/index.js";
export { tool } from "ai";
export { getAISDKLanguageModel } from "./llm/LLMProvider.js";
@ -144,9 +154,14 @@ const StagehandDefault = {
toJsonSchema,
connectToMCPServer,
V3Evaluator,
buildAgentEvidenceFromStepFinished,
loadTrajectoryFromDisk,
mergeAgentEvidence,
nextResultFilename,
normalizeRubric,
redactInlineImagePayloads,
shouldPersistTrajectory,
writeTrajectoryDir,
tool,
getAISDKLanguageModel,
__internalCreateInMemoryAgentCacheHandle,

View File

@ -15,6 +15,7 @@ import {
import { LogLine } from "./logs.js";
import { ClientOptions } from "./model.js";
import { StagehandZodObject } from "../../zodCompat.js";
import type { AgentEvidenceCallback } from "./agentEvidenceEvents.js";
// Re-export ModelMessage for consumers who want to use it for conversation continuation
export type { ModelMessage } from "ai";
@ -136,6 +137,11 @@ export interface AgentCallbacks {
onStepFinish?:
| GenerateTextOnStepFinishCallback<ToolSet>
| StreamTextOnStepFinishCallback<ToolSet>;
/**
* Callback called when Stagehand captures agent-run evidence such as
* screenshots, completed tool/action steps, or post-action observations.
*/
onEvidence?: AgentEvidenceCallback;
}
/**

View File

@ -0,0 +1,105 @@
/**
* Evidence events emitted through AgentExecuteOptions.callbacks.onEvidence.
*
* These events describe observations made by Stagehand during an agent run.
* They are intentionally transport-level callback payloads; verifier-specific
* storage and normalization live in the evals/verifier layers.
*/
export type AgentEvidenceRole = "probe" | "agent";
export type AgentEvidenceEvent =
| AgentScreenshotEvidenceEvent
| AgentStepFinishedEvent
| AgentStepObservedEvent
| AgentFinalAnswerEvent;
/**
* Screenshot captured during an agent run.
*
* In DOM/hybrid mode, post-tool screenshots are probe evidence. In CUA mode,
* screenshots captured by the screenshot provider are agent evidence because
* they are the exact bytes sent to the provider.
*/
export interface AgentScreenshotEvidenceEvent {
type: "screenshot";
/** Zero-based index of the step this screenshot corresponds to. */
stepIndex: number;
/** PNG bytes from page.screenshot(). */
screenshot: Buffer;
/** Page URL at the time of capture. */
url: string;
/** Role this screenshot plays in downstream evidence collection. */
evidenceRole?: AgentEvidenceRole;
}
/**
* One completed agent tool/action step.
*/
export interface AgentStepFinishedEvent {
type: "step_finished";
stepIndex: number;
/** Name of the tool/action that ran, e.g. "act", "extract", "click". */
actionName: string;
/** Arguments passed to the tool/action. */
actionArgs: Record<string, unknown>;
/** Agent textual reasoning for the step, when available. */
reasoning: string;
/** Outcome of the tool/action as seen by Stagehand. */
toolOutput: {
ok: boolean;
/** Native return value from the tool/action. */
result: unknown;
error?: string;
};
/** ISO 8601 timestamp at which the step's tool execution started, when available. */
startedAt?: string;
/** ISO 8601 timestamp at which the step finished. */
finishedAt: string;
}
/**
* Independent post-step browser observation.
*/
export interface AgentStepObservedEvent {
type: "step_observed";
stepIndex: number;
/** Page URL after the step's tool/action execution. */
url: string;
/** Accessibility tree snapshot, when captured. */
ariaTree?: string;
/** Viewport scroll context, when captured. */
scroll?: { top: number; pageHeight: number };
}
export interface AgentFinalObservation {
/** Page URL at the time of terminal capture. */
url: string;
/** PNG bytes from page.screenshot(), when capture succeeds. */
screenshot?: Buffer;
/** Accessibility tree snapshot, when captured. */
ariaTree?: string;
/** Viewport scroll context, when captured. */
scroll?: { top: number; pageHeight: number };
}
/** Final answer emitted by the agent, when available. */
export interface AgentFinalAnswerEvent {
type: "final_answer";
/** The agent's final summary message. */
message: string;
/** Optional structured output if the agent's output schema was set. */
output?: Record<string, unknown>;
/**
* Independent terminal browser observation captured after the agent finishes.
*
* This preserves the legacy verifier behavior of evaluating against a final
* page screenshot even when the last agent output is a final answer rather
* than a browser action.
*/
observation?: AgentFinalObservation;
}
export type AgentEvidenceCallback = (
event: AgentEvidenceEvent,
) => PromiseLike<void> | void;

View File

@ -1,4 +1,5 @@
export * from "./agent.js";
export * from "./agentEvidenceEvents.js";
// Export api.ts under namespace to avoid conflicts with methods.ts types
export * as Api from "./api.js";
// Also export BrowserbaseRegion directly for convenience

View File

@ -0,0 +1,118 @@
import type { AgentStepFinishedEvent } from "../types/public/agentEvidenceEvents.js";
import type { AgentEvidence } from "./types.js";
export const REDACTED_INLINE_IMAGE = "[redacted inline image payload]";
const INLINE_IMAGE_KEYS = new Set(["screenshotBase64"]);
function shouldRedactBase64Key(key: string, actionName?: string): boolean {
return (
INLINE_IMAGE_KEYS.has(key) ||
(actionName === "screenshot" && key === "base64")
);
}
export function collectInlineImagePayloads(
value: unknown,
actionName?: string,
out: string[] = [],
): string[] {
if (!value || typeof value !== "object") return out;
if (Buffer.isBuffer(value)) return out;
if (Array.isArray(value)) {
for (const item of value) {
collectInlineImagePayloads(item, actionName, out);
}
return out;
}
for (const [key, nested] of Object.entries(value)) {
if (shouldRedactBase64Key(key, actionName) && typeof nested === "string") {
out.push(nested);
continue;
}
collectInlineImagePayloads(nested, actionName, out);
}
return out;
}
export function redactInlineImagePayloads(
value: unknown,
actionName?: string,
): unknown {
if (!value || typeof value !== "object") return value;
if (Buffer.isBuffer(value)) return value;
if (Array.isArray(value)) {
return value.map((item) => redactInlineImagePayloads(item, actionName));
}
const out: Record<string, unknown> = {};
for (const [key, nested] of Object.entries(value)) {
out[key] =
shouldRedactBase64Key(key, actionName) && typeof nested === "string"
? REDACTED_INLINE_IMAGE
: redactInlineImagePayloads(nested, actionName);
}
return out;
}
export function mergeAgentEvidence(
...parts: Array<AgentEvidence | undefined>
): AgentEvidence {
return {
modalities: parts.flatMap((p) => p?.modalities ?? []),
};
}
export function buildAgentEvidenceFromStepFinished(
event: AgentStepFinishedEvent,
): AgentEvidence {
const modalities: AgentEvidence["modalities"] = [];
if (event.reasoning) {
modalities.push({ type: "text", content: event.reasoning });
}
const result = event.toolOutput.result;
if (result === undefined || result === null) {
return { modalities };
}
if (typeof result === "string") {
modalities.push({ type: "text", content: result });
} else if (
typeof result === "number" ||
typeof result === "boolean" ||
typeof result === "bigint"
) {
modalities.push({ type: "text", content: String(result) });
} else if (Buffer.isBuffer(result)) {
modalities.push({
type: "image",
bytes: result,
mediaType: "image/png",
});
} else if (typeof result === "object") {
for (const imageBase64 of collectInlineImagePayloads(
result,
event.actionName,
)) {
try {
modalities.push({
type: "image",
bytes: Buffer.from(imageBase64, "base64"),
mediaType: "image/png",
});
} catch {
// Malformed base64; skip the image and keep the JSON modality.
}
}
modalities.push({
type: "json",
content: redactInlineImagePayloads(result, event.actionName),
});
}
return { modalities };
}

View File

@ -21,8 +21,17 @@ export type {
VerifierFinding,
VerifierRawSteps,
} from "./types.js";
export {
buildAgentEvidenceFromStepFinished,
collectInlineImagePayloads,
mergeAgentEvidence,
redactInlineImagePayloads,
REDACTED_INLINE_IMAGE,
} from "./evidenceNormalization.js";
export {
loadTrajectoryFromDisk,
nextResultFilename,
normalizeRubric,
shouldPersistTrajectory,
writeTrajectoryDir,
} from "./trajectory.js";

View File

@ -1,3 +1,5 @@
import fs from "node:fs/promises";
import path from "node:path";
import type {
AgentEvidenceModality,
ProbeEvidence,
@ -5,6 +7,7 @@ import type {
Trajectory,
TrajectoryStep,
} from "./types.js";
import { redactInlineImagePayloads } from "./evidenceNormalization.js";
type RawRubricCriterion = {
criterion: unknown;
@ -18,6 +21,10 @@ type RawRubric = {
items?: unknown;
};
type PersistedProbeEvidence = ProbeEvidence & {
screenshotPath?: string;
};
/**
* Convert dataset or generated rubric JSON into the public Stagehand shape.
* Snake-case dataset fields are accepted here so serialized quirks do not leak
@ -92,8 +99,9 @@ function normalizeResultLabel(label?: string): string {
*
* Reverses the recorder's serialization tweaks:
* - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.
* - Image modalities in `agentEvidence.modalities` carry `bytesBase64` on
* disk (human-readable JSON) instead of raw Buffer; we decode back.
* - Image modalities in `agentEvidence.modalities` carry `imagePath` on
* disk instead of raw Buffer; legacy `bytesBase64` fixtures are also
* accepted.
*
* @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.
*/
@ -105,6 +113,7 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
const trajectoryPath = path.join(trajectoryDir, "trajectory.json");
const raw = await fs.readFile(trajectoryPath, "utf8");
const parsed = JSON.parse(raw) as Trajectory & {
finalObservation?: PersistedProbeEvidence;
steps: Array<
TrajectoryStep & {
agentEvidence: {
@ -113,20 +122,24 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
| {
type: "image";
mediaType: string;
// On-disk form (recorder writes base64); accept either to
// tolerate hand-edited fixtures.
// On-disk forms. Current writer externalizes bytes to
// imagePath; bytesBase64 is accepted for older fixtures.
bytes?: unknown;
bytesBase64?: string;
imagePath?: string;
}
| { type: "json"; content: unknown }
>;
};
probeEvidence: ProbeEvidence;
probeEvidence: PersistedProbeEvidence;
}
>;
};
const resolveWithinTrajectoryDir = (candidate: string): string => {
const resolveWithinTrajectoryDir = (
candidate: string,
fieldName = "screenshotPath",
): string => {
const resolved = path.resolve(trajectoryDir, candidate);
const relative = path.relative(trajectoryDir, resolved);
const outside =
@ -136,16 +149,16 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
if (outside) {
throw new Error(
`Trajectory screenshotPath escapes trajectory directory: ${candidate}`,
`Trajectory ${fieldName} escapes trajectory directory: ${candidate}`,
);
}
return resolved;
};
for (const step of parsed.steps) {
// Rehydrate tier-2 probe screenshot from its on-disk file reference.
const probe = step.probeEvidence;
const hydrateProbeScreenshot = async (
probe: PersistedProbeEvidence | undefined,
): Promise<void> => {
if (probe?.screenshotPath && !probe.screenshot) {
const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);
try {
@ -155,25 +168,55 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
// evidence_insufficient path will handle it.
}
}
};
// Decode image modalities from base64 back to Buffer.
for (const step of parsed.steps) {
// Rehydrate tier-2 probe screenshot from its on-disk file reference.
await hydrateProbeScreenshot(step.probeEvidence);
// Decode image modalities from disk references back to Buffer.
if (step.agentEvidence?.modalities) {
step.agentEvidence.modalities = step.agentEvidence.modalities.map((m) => {
// The on-disk shape carries bytesBase64 instead of bytes, so we look
// through `unknown` here rather than rely on the typed union.
const raw = m as unknown as { bytesBase64?: string };
const modalities: AgentEvidenceModality[] = [];
for (const m of step.agentEvidence.modalities) {
// The on-disk shape carries imagePath/bytesBase64 instead of bytes,
// so we look through `unknown` rather than rely on the typed union.
const raw = m as unknown as {
bytesBase64?: string;
imagePath?: string;
};
if (m.type === "image" && typeof raw.bytesBase64 === "string") {
return {
modalities.push({
type: "image" as const,
bytes: Buffer.from(raw.bytesBase64, "base64"),
mediaType: m.mediaType,
};
});
continue;
}
return m as AgentEvidenceModality;
});
if (m.type === "image" && typeof raw.imagePath === "string") {
const resolved = resolveWithinTrajectoryDir(
raw.imagePath,
"imagePath",
);
try {
modalities.push({
type: "image" as const,
bytes: await fs.readFile(resolved),
mediaType: m.mediaType,
});
} catch {
// Missing agent image file: omit that image modality. The
// verifier's evidence_insufficient path will handle missing bytes.
}
continue;
}
modalities.push(m as AgentEvidenceModality);
}
step.agentEvidence.modalities = modalities;
}
}
await hydrateProbeScreenshot(parsed.finalObservation);
return parsed;
}
@ -187,3 +230,164 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
export function nextResultFilename(label?: string): string {
return `result_${normalizeResultLabel(label)}.json`;
}
/**
* Default persistence policy: explicit override, then env, then "on unless CI".
*/
export function shouldPersistTrajectory(
override: boolean | undefined,
): boolean {
if (override !== undefined) return override;
const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();
if (env === "1" || env === "true") return true;
if (env === "0" || env === "false") return false;
return !process.env.CI;
}
/**
* Write the on-disk trajectory layout under `dir`:
*
* <dir>/
* ├── task_data.json
* ├── trajectory.json (screenshots referenced by path)
* ├── screenshots/
* │ ├── probe/<N>.png
* │ └── agent/<N>[_M].png
* ├── times.json
* ├── scores/ (empty; populated separately)
* └── core.log
*
* Image bytes are externalized to PNG files; the in-memory Trajectory is left
* untouched so callers can keep using it after persistence.
*/
export async function writeTrajectoryDir(
dir: string,
trajectory: Trajectory,
): Promise<void> {
await fs.mkdir(dir, { recursive: true });
await fs.mkdir(path.join(dir, "screenshots", "probe"), { recursive: true });
await fs.mkdir(path.join(dir, "screenshots", "agent"), { recursive: true });
const serializableSteps: unknown[] = [];
for (const step of trajectory.steps) {
const probe: ProbeEvidence = { ...step.probeEvidence };
if (probe.screenshot) {
const relPath = `screenshots/probe/${step.index + 1}.png`;
await fs.writeFile(path.join(dir, relPath), probe.screenshot);
probe.screenshotPath = relPath;
delete probe.screenshot;
}
const imageModalities = step.agentEvidence.modalities.filter(
(m) => m.type === "image",
);
const multipleImages = imageModalities.length > 1;
let imageSeq = 0;
const modalities: unknown[] = [];
for (const m of step.agentEvidence.modalities) {
if (m.type !== "image") {
modalities.push(
m.type === "json"
? {
...m,
content: redactInlineImagePayloads(m.content, step.actionName),
}
: m,
);
continue;
}
const suffix = multipleImages ? `_${imageSeq}` : "";
const relPath = `screenshots/agent/${step.index + 1}${suffix}.png`;
await fs.writeFile(path.join(dir, relPath), m.bytes);
modalities.push({
type: "image",
imagePath: relPath,
mediaType: m.mediaType,
});
imageSeq += 1;
}
serializableSteps.push({
...step,
probeEvidence: probe,
agentEvidence: { modalities },
toolOutput: {
...step.toolOutput,
result: redactInlineImagePayloads(
step.toolOutput.result,
step.actionName,
),
},
});
}
const finalObservation: ProbeEvidence | undefined =
trajectory.finalObservation === undefined
? undefined
: { ...trajectory.finalObservation };
if (finalObservation?.screenshot) {
const relPath = "screenshots/probe/final.png";
await fs.writeFile(path.join(dir, relPath), finalObservation.screenshot);
finalObservation.screenshotPath = relPath;
delete finalObservation.screenshot;
}
// Image modalities carry imagePath instead of raw bytes on disk; cast
// through unknown rather than widen Trajectory's type contract.
const serialized = {
...trajectory,
steps: serializableSteps,
...(finalObservation ? { finalObservation } : {}),
} as unknown;
await fs.writeFile(
path.join(dir, "trajectory.json"),
JSON.stringify(serialized, null, 2),
);
await fs.writeFile(
path.join(dir, "task_data.json"),
JSON.stringify(
{
task: trajectory.task,
status: trajectory.status,
finalAnswer: trajectory.finalAnswer ?? null,
},
null,
2,
),
);
await fs.writeFile(
path.join(dir, "times.json"),
JSON.stringify(
{
timing: trajectory.timing,
usage: trajectory.usage,
stepCount: trajectory.steps.length,
},
null,
2,
),
);
await fs.mkdir(path.join(dir, "scores"), { recursive: true });
await fs.writeFile(path.join(dir, "core.log"), coreLog(trajectory));
}
function coreLog(trajectory: Trajectory): string {
return (
trajectory.steps
.map((step) =>
JSON.stringify({
step: step.index,
action: step.actionName,
url: step.probeEvidence.url ?? null,
ok: step.toolOutput.ok,
reasoning: step.reasoning || undefined,
startedAt: step.startedAt,
finishedAt: step.finishedAt,
}),
)
.join("\n") + "\n"
);
}

View File

@ -136,7 +136,7 @@ export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
* .trajectories/<run-id>/<task-id>/
* ├── task_data.json — TaskSpec + result metadata
* ├── trajectory.json — this object, with screenshotPath instead of bytes
* ├── screenshot_1.png — probeEvidence.screenshot for step 1, etc.
* ├── screenshots/ — step probe/agent images plus final observation
* ├── scores/
* │ └── result.json — Result from V3Evaluator.verify()
* ├── core.log — captured action log
@ -146,6 +146,8 @@ export interface Trajectory {
task: TaskSpec;
steps: TrajectoryStep[];
finalAnswer?: string;
/** Terminal page observation captured after the agent finishes. */
finalObservation?: ProbeEvidence;
status: TrajectoryStatus;
usage: TrajectoryUsage;
timing: { startedAt: string; endedAt: string };

View File

@ -227,6 +227,10 @@ function collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {
}
}
if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {
screenshots.push(trajectory.finalObservation.screenshot);
}
return screenshots;
}
@ -234,12 +238,14 @@ function renderLegacyAgentReasoning(
trajectory: Trajectory,
): string | undefined {
const stepLines = (trajectory.steps ?? []).map((step) => {
const status = step.toolOutput?.ok === false ? "Tool status: failed" : "";
const output = step.toolOutput?.error
? `Tool error: ${step.toolOutput.error}`
: `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;
return [
`Step ${step.index}: ${step.actionName}`,
step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,
status || undefined,
output,
]
.filter(Boolean)

View File

@ -60,6 +60,7 @@ class FakeCuaClient {
public contextNotes: string[] = [];
public preStepHook?: () => Promise<void>;
public actionHandler?: (action: Record<string, unknown>) => Promise<void>;
public screenshotProvider?: () => Promise<string>;
public executeImpl = vi.fn(async (options: unknown) => {
void options;
return {
@ -72,7 +73,9 @@ class FakeCuaClient {
public captureScreenshot = vi.fn(async () => null);
public setViewport = vi.fn();
public setCurrentUrl = vi.fn();
public setScreenshotProvider = vi.fn();
public setScreenshotProvider = vi.fn((provider: () => Promise<string>) => {
this.screenshotProvider = provider;
});
public setSafetyConfirmationHandler = vi.fn();
setActionHandler(
@ -247,7 +250,6 @@ describe("agent captcha hooks", () => {
context: {
awaitActivePage: async () => page,
},
bus: { emit: vi.fn() },
isCaptchaAutoSolveEnabled: true,
isAdvancedStealth: false,
configuredViewport: { width: 1288, height: 711 },
@ -316,7 +318,6 @@ describe("agent captcha hooks", () => {
context: {
awaitActivePage: async () => page,
},
bus: { emit: vi.fn() },
isCaptchaAutoSolveEnabled: true,
isAdvancedStealth: false,
configuredViewport: { width: 1288, height: 711 },
@ -392,7 +393,6 @@ describe("agent captcha hooks", () => {
context: {
awaitActivePage: async () => page,
},
bus: { emit: vi.fn() },
isCaptchaAutoSolveEnabled: true,
isAdvancedStealth: false,
configuredViewport: { width: 1288, height: 711 },
@ -474,7 +474,6 @@ describe("v3 cua handler screenshot behavior", () => {
context: {
awaitActivePage: async () => page,
},
bus: { emit: vi.fn() },
isCaptchaAutoSolveEnabled: false,
isAdvancedStealth: false,
configuredViewport: { width: 1288, height: 711 },
@ -504,4 +503,58 @@ describe("v3 cua handler screenshot behavior", () => {
// the CUA client takes a single screenshot after all actions itself.
expect(screenshotSpy).not.toHaveBeenCalled();
});
it("still returns provider screenshots when screenshot evidence callbacks fail", async () => {
const screenshotBase64 = Buffer.from("fake-image").toString("base64");
const onEvidence = vi.fn(async (event: { type: string }) => {
if (event.type === "screenshot") {
throw new Error("recorder failed");
}
});
fakeCuaClient.executeImpl = vi.fn(async () => {
await expect(fakeCuaClient.screenshotProvider?.()).resolves.toBe(
screenshotBase64,
);
return {
success: true,
message: "ok",
actions: [],
completed: true,
};
});
const handler = new V3CuaAgentHandler(
{
context: {
awaitActivePage: async () => page,
},
isCaptchaAutoSolveEnabled: false,
isAdvancedStealth: false,
configuredViewport: { width: 1288, height: 711 },
isAgentReplayActive: () => false,
updateMetrics: vi.fn(),
} as never,
logger,
{
modelName: "openai/gpt-5.4",
clientOptions: { waitBetweenActions: 1 },
} as never,
);
await handler.execute({
instruction: "describe the page",
highlightCursor: false,
callbacks: { onEvidence },
});
expect(onEvidence).toHaveBeenCalledWith(
expect.objectContaining({ type: "screenshot" }),
);
expect(
logs.some((line) =>
line.message.includes("CUA screenshot evidence callback failed"),
),
).toBe(true);
});
});

View File

@ -0,0 +1,55 @@
import { describe, expect, it } from "vitest";
import { CuaEvidenceStepTracker } from "../../lib/v3/agent/utils/cuaEvidenceStepTracker.js";
describe("CuaEvidenceStepTracker", () => {
it("pairs a fresh provider screenshot with the next action", () => {
const tracker = new CuaEvidenceStepTracker();
const screenshot = Buffer.from("screen");
const event = tracker.recordScreenshot(screenshot, "https://example.com");
const paired = tracker.pairAction();
expect(event).toMatchObject({
type: "screenshot",
stepIndex: 0,
evidenceRole: "agent",
url: "https://example.com",
});
expect(paired).toEqual({ stepIndex: 0 });
});
it("allocates an action step without screenshot evidence", () => {
const tracker = new CuaEvidenceStepTracker();
expect(tracker.pairAction()).toEqual({ stepIndex: 0 });
});
it("replays the latest consumed screenshot for later actions", () => {
const tracker = new CuaEvidenceStepTracker();
const screenshot = Buffer.from("screen");
tracker.recordScreenshot(screenshot, "https://example.com/start");
tracker.pairAction();
const paired = tracker.pairAction();
expect(paired.stepIndex).toBe(1);
expect(paired.replayScreenshot).toMatchObject({
type: "screenshot",
stepIndex: 1,
evidenceRole: "agent",
url: "https://example.com/start",
});
expect(paired.replayScreenshot?.screenshot).toEqual(screenshot);
});
it("resets step allocation and pending screenshot state", () => {
const tracker = new CuaEvidenceStepTracker();
tracker.recordScreenshot(Buffer.from("screen"), "https://example.com");
tracker.reset();
expect(tracker.pairAction()).toEqual({ stepIndex: 0 });
expect(tracker.latestScreenshotUrl).toBeUndefined();
});
});

View File

@ -32,6 +32,8 @@ const publicApiShape = {
V3: Stagehand.V3,
V3Evaluator: Stagehand.V3Evaluator,
V3FunctionName: Stagehand.V3FunctionName,
buildAgentEvidenceFromStepFinished:
Stagehand.buildAgentEvidenceFromStepFinished,
connectToMCPServer: Stagehand.connectToMCPServer,
default: StagehandDefaultExport,
defaultExtractSchema: Stagehand.defaultExtractSchema,
@ -44,18 +46,22 @@ const publicApiShape = {
jsonSchemaToZod: Stagehand.jsonSchemaToZod,
loadApiKeyFromEnv: Stagehand.loadApiKeyFromEnv,
loadTrajectoryFromDisk: Stagehand.loadTrajectoryFromDisk,
mergeAgentEvidence: Stagehand.mergeAgentEvidence,
localBrowserLaunchOptionsSchema: Stagehand.localBrowserLaunchOptionsSchema,
modelToAgentProviderMap: Stagehand.modelToAgentProviderMap,
nextResultFilename: Stagehand.nextResultFilename,
normalizeRubric: Stagehand.normalizeRubric,
pageTextSchema: Stagehand.pageTextSchema,
providerEnvVarMap: Stagehand.providerEnvVarMap,
redactInlineImagePayloads: Stagehand.redactInlineImagePayloads,
shouldPersistTrajectory: Stagehand.shouldPersistTrajectory,
toGeminiSchema: Stagehand.toGeminiSchema,
toJsonSchema: Stagehand.toJsonSchema,
tool: Stagehand.tool,
transformSchema: Stagehand.transformSchema,
trimTrailingTextNode: Stagehand.trimTrailingTextNode,
validateZodSchema: Stagehand.validateZodSchema,
writeTrajectoryDir: Stagehand.writeTrajectoryDir,
...publicErrorTypes,
} as const;

View File

@ -0,0 +1,58 @@
import { describe, expect, it } from "vitest";
import { inferToolOutput } from "../../lib/v3/agent/utils/toolOutputEvidence.js";
describe("inferToolOutput", () => {
it("preserves raw results while normalizing top-level failure status", () => {
const result = { success: false };
expect(inferToolOutput(result)).toEqual({
ok: false,
result,
error: undefined,
});
});
it("normalizes one-level AI SDK output wrappers", () => {
const result = {
toolCallId: "call-1",
output: { success: false, error: { message: "not found" } },
};
expect(inferToolOutput(result)).toEqual({
ok: false,
result,
error: '{"message":"not found"}',
});
});
it("handles isError and non-string errors", () => {
const result = { isError: true, error: new Error("bad input") };
expect(inferToolOutput(result)).toEqual({
ok: false,
result,
error: "bad input",
});
});
it("normalizes non-json error values", () => {
const result = { error: Symbol("bad input") };
expect(inferToolOutput(result)).toEqual({
ok: false,
result,
error: "Symbol(bad input)",
});
});
it("does not recursively treat page data as tool status", () => {
const result = { data: { success: false, error: "page field" } };
expect(inferToolOutput(result)).toEqual({
ok: true,
result,
error: undefined,
});
});
});

View File

@ -82,6 +82,39 @@ describe("V3Evaluator verifier facade", () => {
expect(result.perCriterion).toBeUndefined();
});
it("passes final observation screenshots to the legacy verifier adapter", async () => {
const taskSpec: TaskSpec = {
id: "final-observation",
instruction: "Complete the task",
};
const finalScreenshot = Buffer.from("final screenshot");
const trajectory = {
...makeTrajectory(taskSpec),
finalObservation: {
url: "https://example.com/done",
screenshot: finalScreenshot,
},
};
const ask = vi.fn().mockResolvedValue({
evaluation: "YES",
reasoning: "The final screenshot shows completion.",
});
const evaluator = new V3Evaluator({} as V3, {
backend: "legacy",
});
Object.defineProperty(evaluator, "legacyEvaluator", {
value: { ask },
});
await evaluator.verify(trajectory);
expect(ask).toHaveBeenCalledWith(
expect.objectContaining({
screenshot: [finalScreenshot],
}),
);
});
it("keeps legacy tool output detail until the overall reasoning budget is reached", async () => {
const taskSpec: TaskSpec = {
id: "reasoning-budget",

View File

@ -0,0 +1,20 @@
import { describe, expect, it } from "vitest";
import { buildAgentEvidenceFromStepFinished } from "../../lib/v3/verifier/evidenceNormalization.js";
describe("buildAgentEvidenceFromStepFinished", () => {
it("captures primitive tool results as text evidence", () => {
const evidence = buildAgentEvidenceFromStepFinished({
type: "step_finished",
stepIndex: 0,
actionName: "check",
actionArgs: {},
reasoning: "",
toolOutput: { ok: true, result: false },
startedAt: new Date(0).toISOString(),
finishedAt: new Date(1).toISOString(),
});
expect(evidence.modalities).toEqual([{ type: "text", content: "false" }]);
});
});

View File

@ -1,4 +1,4 @@
import { mkdtemp, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@ -8,6 +8,7 @@ import {
loadTrajectoryFromDisk,
nextResultFilename,
normalizeRubric,
writeTrajectoryDir,
} from "../../lib/v3/verifier/trajectory.js";
describe("verifier trajectory utilities", () => {
@ -63,8 +64,67 @@ describe("verifier trajectory utilities", () => {
it("loads trajectory screenshots and image modalities from disk", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
const screenshot = Buffer.from("probe screenshot");
const finalScreenshot = Buffer.from("final screenshot");
const agentImage = Buffer.from("agent image");
await writeFile(path.join(dir, "screenshot_1.png"), screenshot);
await writeFile(path.join(dir, "final.png"), finalScreenshot);
await mkdir(path.join(dir, "screenshots", "agent"), { recursive: true });
await writeFile(
path.join(dir, "screenshots", "agent", "1.png"),
agentImage,
);
await writeFile(
path.join(dir, "trajectory.json"),
JSON.stringify({
task: { id: "task", instruction: "Do the task" },
status: "complete",
usage: { input_tokens: 0, output_tokens: 0 },
timing: {
startedAt: new Date(0).toISOString(),
endedAt: new Date(0).toISOString(),
},
steps: [
{
index: 0,
actionName: "act",
actionArgs: {},
reasoning: "",
agentEvidence: {
modalities: [
{
type: "image",
mediaType: "image/png",
imagePath: "screenshots/agent/1.png",
},
],
},
probeEvidence: { screenshotPath: "screenshot_1.png" },
toolOutput: { ok: true, result: null },
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
},
],
finalObservation: {
url: "https://example.com/done",
screenshotPath: "final.png",
},
}),
);
const trajectory = await loadTrajectoryFromDisk(dir);
const modality = trajectory.steps[0].agentEvidence.modalities[0];
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(screenshot);
expect(trajectory.finalObservation?.screenshot).toEqual(finalScreenshot);
expect(modality.type).toBe("image");
if (modality.type === "image") {
expect(modality.bytes).toEqual(agentImage);
}
});
it("loads legacy base64 image modalities from disk", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
const agentImage = Buffer.from("legacy agent image");
await writeFile(
path.join(dir, "trajectory.json"),
JSON.stringify({
@ -90,7 +150,7 @@ describe("verifier trajectory utilities", () => {
},
],
},
probeEvidence: { screenshotPath: "screenshot_1.png" },
probeEvidence: {},
toolOutput: { ok: true, result: null },
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
@ -102,13 +162,83 @@ describe("verifier trajectory utilities", () => {
const trajectory = await loadTrajectoryFromDisk(dir);
const modality = trajectory.steps[0].agentEvidence.modalities[0];
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(screenshot);
expect(modality.type).toBe("image");
if (modality.type === "image") {
expect(modality.bytes).toEqual(agentImage);
}
});
it("redacts inline screenshot payloads when writing trajectories", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
const inlineScreenshot =
Buffer.from("inline screenshot").toString("base64");
await writeTrajectoryDir(dir, {
task: { id: "task", instruction: "Do the task" },
status: "complete",
usage: { input_tokens: 0, output_tokens: 0 },
timing: {
startedAt: new Date(0).toISOString(),
endedAt: new Date(0).toISOString(),
},
finalObservation: {
url: "https://example.com/done",
screenshot: Buffer.from("final screenshot"),
},
steps: [
{
index: 0,
actionName: "click",
actionArgs: {},
reasoning: "",
agentEvidence: {
modalities: [
{
type: "json",
content: {
output: {
success: true,
screenshotBase64: inlineScreenshot,
},
},
},
],
},
probeEvidence: {},
toolOutput: {
ok: true,
result: {
output: {
success: true,
screenshotBase64: inlineScreenshot,
},
},
},
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
},
],
});
const raw = await readFile(path.join(dir, "trajectory.json"), "utf8");
const trajectory = JSON.parse(raw);
expect(raw).not.toContain(inlineScreenshot);
expect(
trajectory.steps[0].agentEvidence.modalities[0].content.output
.screenshotBase64,
).toBe("[redacted inline image payload]");
expect(trajectory.steps[0].toolOutput.result.output.screenshotBase64).toBe(
"[redacted inline image payload]",
);
expect(trajectory.finalObservation.screenshotPath).toBe(
"screenshots/probe/final.png",
);
await expect(
readFile(path.join(dir, "screenshots", "probe", "final.png")),
).resolves.toEqual(Buffer.from("final screenshot"));
});
it("rejects screenshot paths outside the trajectory directory", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
await writeFile(

View File

@ -0,0 +1,305 @@
import fs from "node:fs/promises";
import path from "node:path";
import {
buildAgentEvidenceFromStepFinished,
mergeAgentEvidence,
redactInlineImagePayloads,
shouldPersistTrajectory,
writeTrajectoryDir,
} from "@browserbasehq/stagehand";
import type {
AgentEvidence,
AgentEvidenceEvent,
AgentFinalAnswerEvent,
AgentScreenshotEvidenceEvent,
AgentStepFinishedEvent,
AgentStepObservedEvent,
ProbeEvidence,
TaskSpec,
Trajectory,
TrajectoryStatus,
TrajectoryStep,
TrajectoryUsage,
EvaluationResult,
} from "@browserbasehq/stagehand";
interface PartialStep {
index: number;
actionName: string;
actionArgs: Record<string, unknown>;
reasoning: string;
agentEvidence: AgentEvidence;
probeEvidence: ProbeEvidence;
toolOutput: { ok: boolean; result: unknown; error?: string };
startedAt: string;
finishedAt: string;
}
export interface TrajectoryRecorderOptions {
taskSpec: TaskSpec;
/**
* Root directory under which trajectory dirs are written. Each task run
* gets a subdirectory named by runId/task.id.
* Defaults to `<cwd>/.trajectories`.
*/
outputRoot?: string;
/** Run identifier (e.g., ISO timestamp + env). Defaults to a fresh timestamp. */
runId?: string;
/**
* Override the env-gated persistence default. `true` always persists,
* `false` never does, `undefined` defers to VERIFIER_PERSIST_TRAJECTORIES.
*/
persist?: boolean;
}
export interface TrajectoryFinishOptions {
status: TrajectoryStatus;
finalAnswer?: string;
usage?: Partial<TrajectoryUsage>;
}
const ZERO_USAGE: TrajectoryUsage = {
input_tokens: 0,
output_tokens: 0,
};
export class TrajectoryRecorder {
private readonly taskSpec: TaskSpec;
private readonly runId: string;
private readonly outputDir: string;
private readonly persistEnabled: boolean;
// Events can arrive out-of-order across step indices; same-step events all
// fire in one microtask.
private readonly partialSteps = new Map<number, Partial<PartialStep>>();
private finalAnswerEvent?: AgentFinalAnswerEvent;
private finalObservation?: ProbeEvidence;
private startedAt = "";
private endedAt = "";
private onScreenshot(e: AgentScreenshotEvidenceEvent): void {
const partial = this.ensurePartial(e.stepIndex);
// Default to probe when the emit site doesn't tag a role: matches
// v3AgentHandler's post-step screenshot. For CUA the pre-action shot is
// NOT a probe — emitCuaActionStep fills that role post-action.
const role = e.evidenceRole ?? "probe";
if (role === "probe") {
const probe: ProbeEvidence = { ...(partial.probeEvidence ?? {}) };
probe.screenshot = e.screenshot;
probe.url = e.url;
partial.probeEvidence = probe;
} else if (!partial.probeEvidence?.url) {
// Capture URL even for tier-1-only events; a later post-action URL
// can still overwrite it.
partial.probeEvidence = {
...(partial.probeEvidence ?? {}),
url: e.url,
};
}
if (role === "agent") {
partial.agentEvidence = mergeAgentEvidence(partial.agentEvidence, {
modalities: [
{ type: "image", bytes: e.screenshot, mediaType: "image/png" },
],
});
}
}
private onStepFinished(e: AgentStepFinishedEvent): void {
const partial = this.ensurePartial(e.stepIndex);
partial.actionName = e.actionName;
partial.actionArgs = e.actionArgs;
partial.reasoning = e.reasoning;
partial.toolOutput = {
...e.toolOutput,
result: redactInlineImagePayloads(e.toolOutput.result, e.actionName),
};
partial.startedAt = e.startedAt ?? e.finishedAt;
partial.finishedAt = e.finishedAt;
partial.agentEvidence = mergeAgentEvidence(
partial.agentEvidence,
buildAgentEvidenceFromStepFinished(e),
);
}
private onStepObserved(e: AgentStepObservedEvent): void {
const partial = this.ensurePartial(e.stepIndex);
const probe: ProbeEvidence = { ...(partial.probeEvidence ?? {}) };
probe.url = e.url;
if (e.ariaTree !== undefined) probe.ariaTree = e.ariaTree;
if (e.scroll !== undefined) probe.scroll = e.scroll;
partial.probeEvidence = probe;
}
private onFinalAnswer(e: AgentFinalAnswerEvent): void {
this.finalAnswerEvent = e;
if (e.observation) {
this.finalObservation = {
url: e.observation.url,
...(e.observation.screenshot
? { screenshot: e.observation.screenshot }
: {}),
...(e.observation.ariaTree !== undefined
? { ariaTree: e.observation.ariaTree }
: {}),
...(e.observation.scroll !== undefined
? { scroll: e.observation.scroll }
: {}),
};
}
}
constructor(opts: TrajectoryRecorderOptions) {
this.taskSpec = opts.taskSpec;
this.runId = opts.runId ?? new Date().toISOString().replace(/[:.]/g, "-");
const root = opts.outputRoot ?? path.join(process.cwd(), ".trajectories");
this.outputDir = path.join(root, this.runId, opts.taskSpec.id);
this.persistEnabled = shouldPersistTrajectory(opts.persist);
}
/** Mark the beginning of collection. Call once before agent.execute(). */
start(): void {
if (this.startedAt) return;
this.startedAt = new Date().toISOString();
}
/** Ingest an evidence callback event from agent.execute(). */
record(event: AgentEvidenceEvent): void {
if (!this.startedAt) this.start();
switch (event.type) {
case "screenshot":
this.onScreenshot(event);
break;
case "step_finished":
this.onStepFinished(event);
break;
case "step_observed":
this.onStepObserved(event);
break;
case "final_answer":
this.onFinalAnswer(event);
break;
}
}
/**
* Detach listeners, assemble the Trajectory, and (if persistence is on)
* write the on-disk layout. Idempotent.
*/
async finish(opts: TrajectoryFinishOptions): Promise<Trajectory> {
if (!this.startedAt) this.start();
this.endedAt = new Date().toISOString();
const steps = this.assembleSteps();
const trajectory: Trajectory = {
task: this.taskSpec,
steps,
finalAnswer: opts.finalAnswer ?? this.finalAnswerEvent?.message,
...(this.finalObservation
? { finalObservation: this.finalObservation }
: {}),
status: opts.status,
usage: { ...ZERO_USAGE, ...(opts.usage ?? {}) },
timing: { startedAt: this.startedAt, endedAt: this.endedAt },
};
if (this.persistEnabled) {
await writeTrajectoryDir(this.outputDir, trajectory);
}
return trajectory;
}
/** Throw away in-memory state without writing to disk. Used on early abort. */
cancel(): void {
this.partialSteps.clear();
this.finalAnswerEvent = undefined;
this.finalObservation = undefined;
}
/** Where the trajectory dir lives (whether or not it was persisted). */
get directory(): string {
return this.outputDir;
}
/** Whether this recorder wrote the trajectory directory on finish(). */
get persisted(): boolean {
return this.persistEnabled;
}
/**
* Persist evaluator result next to the trajectory. No-op when trajectory
* persistence is disabled.
*/
async persistResult(
result: EvaluationResult,
filename = "result.json",
): Promise<void> {
if (!this.persistEnabled) return;
const scoresDir = path.join(this.outputDir, "scores");
await fs.mkdir(scoresDir, { recursive: true });
await fs.writeFile(
path.join(scoresDir, filename),
JSON.stringify(result, null, 2),
);
const taskDataPath = path.join(this.outputDir, "task_data.json");
let taskData: Record<string, unknown>;
try {
taskData = JSON.parse(await fs.readFile(taskDataPath, "utf8")) as Record<
string,
unknown
>;
} catch {
taskData = { task: this.taskSpec };
}
await fs.writeFile(
taskDataPath,
JSON.stringify({ ...taskData, result }, null, 2),
);
}
private ensurePartial(stepIndex: number): Partial<PartialStep> {
let p = this.partialSteps.get(stepIndex);
if (!p) {
p = { index: stepIndex };
this.partialSteps.set(stepIndex, p);
}
return p;
}
private assembleSteps(): TrajectoryStep[] {
const out: TrajectoryStep[] = [];
const indices = [...this.partialSteps.keys()].sort((a, b) => a - b);
for (const i of indices) {
const p = this.partialSteps.get(i)!;
if (
p.actionName === undefined ||
p.toolOutput === undefined ||
p.startedAt === undefined ||
p.finishedAt === undefined
) {
// Provider-only screenshot refreshes are transport evidence for the
// next CUA action. If no action arrives for this index, there is no
// completed trajectory step to persist.
continue;
}
out.push({
index: i,
actionName: p.actionName,
actionArgs: p.actionArgs ?? {},
reasoning: p.reasoning ?? "",
agentEvidence: p.agentEvidence ?? { modalities: [] },
probeEvidence: p.probeEvidence ?? {},
toolOutput: p.toolOutput,
startedAt: p.startedAt,
finishedAt: p.finishedAt,
});
}
return out;
}
}

View File

@ -0,0 +1,327 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { TaskSpec } from "@browserbasehq/stagehand";
import { TrajectoryRecorder } from "../../framework/trajectoryRecorder.js";
const tempDirs: string[] = [];
afterEach(async () => {
while (tempDirs.length > 0) {
const dir = tempDirs.pop();
if (dir) await fs.rm(dir, { recursive: true, force: true });
}
});
function makeTempDir(): Promise<string> {
return fs
.mkdtemp(path.join(os.tmpdir(), "trajectory-recorder-"))
.then((dir) => {
tempDirs.push(dir);
return dir;
});
}
function makeTaskSpec(): TaskSpec {
return {
id: "recorder-task",
instruction: "Compare economy and business fares.",
initUrl: "https://example.com",
precomputedRubric: {
items: [
{
criterion: "Report fare delta",
description: "Report the difference between two fares.",
maxPoints: 1,
},
],
},
};
}
describe("TrajectoryRecorder", () => {
it("assembles trajectory evidence from callback events", async () => {
const recorder = new TrajectoryRecorder({
taskSpec: makeTaskSpec(),
persist: false,
});
const screenshot = Buffer.from("screen-1");
recorder.start();
recorder.record({
type: "screenshot",
stepIndex: 0,
screenshot,
url: "https://example.com/search",
evidenceRole: "agent",
});
recorder.record({
type: "screenshot",
stepIndex: 0,
screenshot,
url: "https://example.com/search",
evidenceRole: "probe",
});
recorder.record({
type: "step_finished",
stepIndex: 0,
actionName: "extract",
actionArgs: { instruction: "Read fares" },
reasoning: "Read visible fare cells.",
toolOutput: {
ok: true,
result: { economy: "$100", business: "$250" },
},
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
});
recorder.record({
type: "step_observed",
stepIndex: 0,
url: "https://example.com/search",
ariaTree: "RootWebArea\nStaticText: Economy $100",
});
recorder.record({
type: "final_answer",
message: "Business is $150 more than economy.",
observation: {
url: "https://example.com/checkout",
screenshot: Buffer.from("final-screen"),
ariaTree: "RootWebArea\nStaticText: Complete",
},
});
const trajectory = await recorder.finish({
status: "complete",
usage: { input_tokens: 10, output_tokens: 5 },
});
expect(trajectory.steps).toHaveLength(1);
expect(trajectory.steps[0]).toMatchObject({
index: 0,
actionName: "extract",
actionArgs: { instruction: "Read fares" },
reasoning: "Read visible fare cells.",
toolOutput: {
ok: true,
result: { economy: "$100", business: "$250" },
},
probeEvidence: {
url: "https://example.com/search",
ariaTree: "RootWebArea\nStaticText: Economy $100",
},
});
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(screenshot);
expect(trajectory.steps[0].startedAt).toBe(new Date(0).toISOString());
expect(trajectory.steps[0].agentEvidence.modalities).toEqual(
expect.arrayContaining([
{ type: "image", bytes: screenshot, mediaType: "image/png" },
{ type: "text", content: "Read visible fare cells." },
{ type: "json", content: { economy: "$100", business: "$250" } },
]),
);
expect(trajectory.finalAnswer).toBe("Business is $150 more than economy.");
expect(trajectory.finalObservation).toMatchObject({
url: "https://example.com/checkout",
ariaTree: "RootWebArea\nStaticText: Complete",
});
expect(trajectory.finalObservation?.screenshot).toEqual(
Buffer.from("final-screen"),
);
});
it("persists trajectory files and evaluator results", async () => {
const outputRoot = await makeTempDir();
const recorder = new TrajectoryRecorder({
taskSpec: makeTaskSpec(),
outputRoot,
runId: "run-1",
persist: true,
});
const screenshot = Buffer.from("screen-1");
recorder.start();
recorder.record({
type: "screenshot",
stepIndex: 0,
screenshot,
url: "https://example.com/search",
evidenceRole: "agent",
});
recorder.record({
type: "screenshot",
stepIndex: 0,
screenshot,
url: "https://example.com/search",
evidenceRole: "probe",
});
recorder.record({
type: "step_finished",
stepIndex: 0,
actionName: "act",
actionArgs: { instruction: "Search fares" },
reasoning: "Search for fares.",
toolOutput: { ok: true, result: "done" },
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
});
recorder.record({
type: "step_observed",
stepIndex: 0,
url: "https://example.com/search",
});
recorder.record({
type: "final_answer",
message: "Complete.",
observation: {
url: "https://example.com/complete",
screenshot: Buffer.from("final-screen"),
},
});
await recorder.finish({ status: "complete" });
await recorder.persistResult({
outcomeSuccess: true,
explanation: "The task was completed.",
});
const taskDir = path.join(outputRoot, "run-1", "recorder-task");
await expect(fs.readdir(taskDir)).resolves.toEqual(
expect.arrayContaining([
"core.log",
"scores",
"screenshots",
"task_data.json",
"times.json",
"trajectory.json",
]),
);
await expect(
fs.readFile(path.join(taskDir, "screenshots", "probe", "1.png")),
).resolves.toEqual(screenshot);
await expect(
fs.readFile(path.join(taskDir, "screenshots", "probe", "final.png")),
).resolves.toEqual(Buffer.from("final-screen"));
await expect(
fs.readFile(path.join(taskDir, "screenshots", "agent", "1.png")),
).resolves.toEqual(screenshot);
await expect(
fs.readFile(path.join(taskDir, "scores", "result.json"), "utf8"),
).resolves.toContain('"outcomeSuccess": true');
const trajectory = JSON.parse(
await fs.readFile(path.join(taskDir, "trajectory.json"), "utf8"),
);
expect(trajectory.steps[0].probeEvidence.screenshotPath).toBe(
"screenshots/probe/1.png",
);
expect(trajectory.finalObservation.screenshotPath).toBe(
"screenshots/probe/final.png",
);
expect(trajectory.steps[0].agentEvidence.modalities).toContainEqual({
type: "image",
imagePath: "screenshots/agent/1.png",
mediaType: "image/png",
});
const taskData = JSON.parse(
await fs.readFile(path.join(taskDir, "task_data.json"), "utf8"),
);
expect(taskData.result).toMatchObject({
outcomeSuccess: true,
explanation: "The task was completed.",
});
});
it("normalizes missing step startedAt to finishedAt", async () => {
const recorder = new TrajectoryRecorder({
taskSpec: makeTaskSpec(),
persist: false,
});
const finishedAt = new Date(1).toISOString();
recorder.record({
type: "step_finished",
stepIndex: 0,
actionName: "extract",
actionArgs: { instruction: "Read fares" },
reasoning: "",
toolOutput: { ok: true, result: false },
finishedAt,
});
const trajectory = await recorder.finish({ status: "complete" });
expect(trajectory.steps[0].startedAt).toBe(finishedAt);
expect(trajectory.steps[0].finishedAt).toBe(finishedAt);
});
it("lifts inline screenshot payloads into image evidence and redacts JSON", async () => {
const inlineScreenshot =
Buffer.from("inline screenshot").toString("base64");
const recorder = new TrajectoryRecorder({
taskSpec: makeTaskSpec(),
persist: false,
});
recorder.record({
type: "step_finished",
stepIndex: 0,
actionName: "click",
actionArgs: { describe: "Open fare details" },
reasoning: "Click the fare details button.",
toolOutput: {
ok: true,
result: {
output: {
success: true,
describe: "Open fare details",
screenshotBase64: inlineScreenshot,
},
},
},
startedAt: new Date(0).toISOString(),
finishedAt: new Date(0).toISOString(),
});
const trajectory = await recorder.finish({ status: "complete" });
const step = trajectory.steps[0];
const rawTrajectory = JSON.stringify(trajectory);
const imageModalities = step.agentEvidence.modalities.filter(
(m) => m.type === "image",
);
const jsonModality = step.agentEvidence.modalities.find(
(m) => m.type === "json",
);
expect(rawTrajectory).not.toContain(inlineScreenshot);
expect(step.toolOutput.result).toMatchObject({
output: {
success: true,
describe: "Open fare details",
screenshotBase64: "[redacted inline image payload]",
},
});
expect(jsonModality).toMatchObject({
type: "json",
content: {
output: {
screenshotBase64: "[redacted inline image payload]",
},
},
});
expect(imageModalities).toHaveLength(1);
expect(imageModalities[0]).toMatchObject({
type: "image",
mediaType: "image/png",
});
if (imageModalities[0].type === "image") {
expect(imageModalities[0].bytes).toEqual(
Buffer.from(inlineScreenshot, "base64"),
);
}
});
});