Compare commits
7 Commits
codex/sdk-
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1db5f1c193 | ||
|
|
ce21468147 | ||
|
|
49575d62f5 | ||
|
|
bf9d44989c | ||
|
|
aff5528c8d | ||
|
|
947e6e4249 | ||
|
|
e102a89c90 |
5
.changeset/opus-4-8-cua.md
Normal file
5
.changeset/opus-4-8-cua.md
Normal file
@ -0,0 +1,5 @@
|
||||
---
|
||||
"@browserbasehq/stagehand": patch
|
||||
---
|
||||
|
||||
Add Claude Opus 4.8 to the supported CUA model whitelist.
|
||||
5
.changeset/true-pillows-prove.md
Normal file
5
.changeset/true-pillows-prove.md
Normal file
@ -0,0 +1,5 @@
|
||||
---
|
||||
"@browserbasehq/stagehand-server-v3": patch
|
||||
---
|
||||
|
||||
Add SEA binary --version build metadata output.
|
||||
5
.changeset/verifier-trajectory-events.md
Normal file
5
.changeset/verifier-trajectory-events.md
Normal file
@ -0,0 +1,5 @@
|
||||
---
|
||||
"@browserbasehq/stagehand": patch
|
||||
---
|
||||
|
||||
Capture verifier trajectory evidence from agent evidence callbacks for offline scoring.
|
||||
6
.changeset/vertex-model-forwarding.md
Normal file
6
.changeset/vertex-model-forwarding.md
Normal file
@ -0,0 +1,6 @@
|
||||
---
|
||||
"@browserbasehq/stagehand": minor
|
||||
"@browserbasehq/stagehand-server-v3": minor
|
||||
---
|
||||
|
||||
Forward constructor and request model configuration when initializing API-backed sessions.
|
||||
@ -56,7 +56,7 @@
|
||||
"prettier": "^3.2.5",
|
||||
"source-map": "^0.7.4",
|
||||
"tsx": "^4.19.4",
|
||||
"turbo": "^2.8.10",
|
||||
"turbo": "^2.9.15",
|
||||
"typescript": "5.8.3",
|
||||
"typescript-eslint": "^8.56.1"
|
||||
},
|
||||
@ -78,7 +78,9 @@
|
||||
"jws": "^4.0.1",
|
||||
"lodash": "^4.18.0",
|
||||
"minimatch": "^3.1.4",
|
||||
"uuid": "^11.1.1"
|
||||
"uuid": "^11.1.1",
|
||||
"ws": "^8.20.1",
|
||||
"qs": "^6.15.2"
|
||||
}
|
||||
},
|
||||
"lint-staged": {
|
||||
|
||||
@ -51,7 +51,7 @@
|
||||
"node-html-markdown": "^1.3.0",
|
||||
"pino": "^9.6.0",
|
||||
"pino-pretty": "^13.0.0",
|
||||
"ws": "^8.18.0"
|
||||
"ws": "^8.21.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.30",
|
||||
|
||||
@ -23,6 +23,7 @@ export const modelToAgentProviderMap: Record<string, AgentProviderType> = {
|
||||
"claude-sonnet-4-5-20250929": "anthropic",
|
||||
"claude-opus-4-5-20251101": "anthropic",
|
||||
"claude-opus-4-6": "anthropic",
|
||||
"claude-opus-4-8": "anthropic",
|
||||
"claude-sonnet-4-6": "anthropic",
|
||||
"claude-haiku-4-5": "anthropic",
|
||||
"claude-haiku-4-5-20251001": "anthropic",
|
||||
|
||||
@ -453,6 +453,7 @@ export class AnthropicCUAClient extends AgentClient {
|
||||
|
||||
// Check if this is a Claude 4.6+ model that supports adaptive thinking
|
||||
const isAdaptiveThinkingModel = [
|
||||
"claude-opus-4-8",
|
||||
"claude-opus-4-6",
|
||||
"claude-sonnet-4-6",
|
||||
].includes(modelBase);
|
||||
|
||||
73
packages/core/lib/v3/agent/utils/captureAriaTreeProbe.ts
Normal file
73
packages/core/lib/v3/agent/utils/captureAriaTreeProbe.ts
Normal file
@ -0,0 +1,73 @@
|
||||
/**
|
||||
* captureAriaTreeProbe — capture a truncated accessibility tree of the active
|
||||
* page for use as tier-2 evidence in the trajectory recorder.
|
||||
*
|
||||
* Shared by v3AgentHandler and v3CuaAgentHandler. Listener-gated by the
|
||||
* callers so ordinary agent runs (no TrajectoryRecorder attached) don't pay
|
||||
* the cost.
|
||||
*
|
||||
* The a11y tree is the same payload the agent's `ariaTree` tool sees, but
|
||||
* captured by the harness (not the agent) so the verifier has independent
|
||||
* textual ground truth for grounding non-visual claims — prices, names,
|
||||
* dates, list contents — without OCR'ing screenshots.
|
||||
*
|
||||
* Budget: defaults to ~8000 tokens (32k chars). Per-step a11y captures
|
||||
* across a ~30-step trajectory at that cap sum to ~240k tokens total,
|
||||
* which the verifier handles via per-criterion top-K selection. The cap
|
||||
* is configurable via VERIFIER_ARIATREE_TOKEN_BUDGET so consumers can
|
||||
* trade RAM/disk for fidelity. Truncated content is marked explicitly so
|
||||
* the verifier knows it was clipped.
|
||||
*/
|
||||
import type { V3 } from "../../v3.js";
|
||||
|
||||
const APPROX_CHARS_PER_TOKEN = 4;
|
||||
const DEFAULT_TOKEN_BUDGET = 8_000;
|
||||
const DEFAULT_TIMEOUT_MS = 5_000;
|
||||
|
||||
interface CaptureAriaTreeOptions {
|
||||
/** Soft cap on token count (chars/4 approximation). Default 8000. */
|
||||
tokenBudget?: number;
|
||||
/** Hard timeout on the capture. Default 5s. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the truncated a11y tree as a plain string, or undefined when
|
||||
* capture fails. Never throws — a11y capture is best-effort tier-2 evidence,
|
||||
* not a hard requirement, so failures are silently absorbed (the verifier
|
||||
* surfaces this via evidence_insufficient).
|
||||
*/
|
||||
export async function captureAriaTreeProbe(
|
||||
v3: V3,
|
||||
opts: CaptureAriaTreeOptions = {},
|
||||
): Promise<string | undefined> {
|
||||
const envBudget = parseInt(
|
||||
process.env.VERIFIER_ARIATREE_TOKEN_BUDGET ?? "",
|
||||
10,
|
||||
);
|
||||
const tokenBudget =
|
||||
opts.tokenBudget ??
|
||||
(Number.isFinite(envBudget) && envBudget > 0
|
||||
? envBudget
|
||||
: DEFAULT_TOKEN_BUDGET);
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const maxChars = tokenBudget * APPROX_CHARS_PER_TOKEN;
|
||||
|
||||
try {
|
||||
// v3.extract() without a schema returns { pageText } where pageText is the
|
||||
// rendered accessibility tree — same path the agent's ariaTree tool uses.
|
||||
const result = await v3.extract({ timeout: timeoutMs });
|
||||
const pageText = result?.pageText;
|
||||
if (typeof pageText !== "string" || pageText.length === 0) return undefined;
|
||||
|
||||
if (pageText.length > maxChars) {
|
||||
return (
|
||||
pageText.slice(0, maxChars) +
|
||||
`\n\n[CONTENT TRUNCATED at ~${tokenBudget} tokens — set VERIFIER_ARIATREE_TOKEN_BUDGET to raise]`
|
||||
);
|
||||
}
|
||||
return pageText;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
81
packages/core/lib/v3/agent/utils/postStepProbeEvidence.ts
Normal file
81
packages/core/lib/v3/agent/utils/postStepProbeEvidence.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import type { AgentEvidenceCallback } from "../../types/public/agentEvidenceEvents.js";
|
||||
import type { LogLine } from "../../types/public/logs.js";
|
||||
import type { V3 } from "../../v3.js";
|
||||
import { captureAriaTreeProbe } from "./captureAriaTreeProbe.js";
|
||||
|
||||
interface CaptureProbeEvidenceOptions {
|
||||
v3: V3;
|
||||
url: string;
|
||||
logger: (message: LogLine) => void;
|
||||
warningMessage: string;
|
||||
}
|
||||
|
||||
interface EmitPostStepProbeEvidenceOptions extends CaptureProbeEvidenceOptions {
|
||||
evidenceCallback?: AgentEvidenceCallback;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
export async function captureProbeEvidence({
|
||||
v3,
|
||||
url,
|
||||
logger,
|
||||
warningMessage,
|
||||
}: CaptureProbeEvidenceOptions): Promise<{
|
||||
url: string;
|
||||
screenshot?: Buffer;
|
||||
ariaTree?: string;
|
||||
}> {
|
||||
let probeUrl = url;
|
||||
let screenshot: Buffer | undefined;
|
||||
try {
|
||||
const page = await v3.context.awaitActivePage();
|
||||
probeUrl = page.url();
|
||||
screenshot = await page.screenshot({ fullPage: false });
|
||||
} catch (e) {
|
||||
logger({
|
||||
category: "agent",
|
||||
message: `${warningMessage}: ${errorMessage(e)}`,
|
||||
level: 1,
|
||||
});
|
||||
}
|
||||
|
||||
const ariaTree = await captureAriaTreeProbe(v3);
|
||||
return {
|
||||
url: probeUrl,
|
||||
...(screenshot ? { screenshot } : {}),
|
||||
...(ariaTree !== undefined ? { ariaTree } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function emitPostStepProbeEvidence({
|
||||
v3,
|
||||
url,
|
||||
evidenceCallback,
|
||||
logger,
|
||||
warningMessage,
|
||||
}: EmitPostStepProbeEvidenceOptions): Promise<void> {
|
||||
if (!evidenceCallback) return;
|
||||
|
||||
const probe = await captureProbeEvidence({
|
||||
v3,
|
||||
url,
|
||||
logger,
|
||||
warningMessage,
|
||||
});
|
||||
if (probe.screenshot) {
|
||||
await evidenceCallback({
|
||||
type: "screenshot",
|
||||
screenshot: probe.screenshot,
|
||||
url: probe.url,
|
||||
evidenceRole: "probe",
|
||||
});
|
||||
}
|
||||
await evidenceCallback({
|
||||
type: "step_observed",
|
||||
url: probe.url,
|
||||
ariaTree: probe.ariaTree,
|
||||
});
|
||||
}
|
||||
76
packages/core/lib/v3/agent/utils/toolOutputEvidence.ts
Normal file
76
packages/core/lib/v3/agent/utils/toolOutputEvidence.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import type { AgentStepFinishedEvent } from "../../types/public/agentEvidenceEvents.js";
|
||||
|
||||
const ERROR_STRING_LIMIT = 1000;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function hasOwn(value: Record<string, unknown>, key: string): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(value, key);
|
||||
}
|
||||
|
||||
function normalizeError(value: unknown): string | undefined {
|
||||
if (value === undefined || value === null || value === false) {
|
||||
return undefined;
|
||||
}
|
||||
if (value instanceof Error) {
|
||||
return value.message;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (
|
||||
typeof value === "number" ||
|
||||
typeof value === "boolean" ||
|
||||
typeof value === "bigint"
|
||||
) {
|
||||
return String(value);
|
||||
}
|
||||
|
||||
let serialized: string;
|
||||
try {
|
||||
serialized = JSON.stringify(value) ?? String(value);
|
||||
} catch {
|
||||
serialized = String(value);
|
||||
}
|
||||
if (serialized.length <= ERROR_STRING_LIMIT) {
|
||||
return serialized;
|
||||
}
|
||||
return `${serialized.slice(0, ERROR_STRING_LIMIT)}... [truncated]`;
|
||||
}
|
||||
|
||||
function statusCandidates(toolResult: unknown): Record<string, unknown>[] {
|
||||
if (!isRecord(toolResult)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const candidates = [toolResult];
|
||||
const output = toolResult.output;
|
||||
if (isRecord(output)) {
|
||||
candidates.push(output);
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export function inferToolOutput(
|
||||
toolResult: unknown,
|
||||
): AgentStepFinishedEvent["toolOutput"] {
|
||||
const candidates = statusCandidates(toolResult);
|
||||
const error = candidates
|
||||
.map((candidate) =>
|
||||
hasOwn(candidate, "error") ? normalizeError(candidate.error) : undefined,
|
||||
)
|
||||
.find((message): message is string => message !== undefined);
|
||||
|
||||
const successFalse = candidates.some(
|
||||
(candidate) => candidate.success === false,
|
||||
);
|
||||
const isError = candidates.some((candidate) => Boolean(candidate.isError));
|
||||
|
||||
return {
|
||||
ok: error === undefined && !isError && !successFalse,
|
||||
result: toolResult,
|
||||
error,
|
||||
};
|
||||
}
|
||||
27
packages/core/lib/v3/agent/utils/wrapEvidenceCallback.ts
Normal file
27
packages/core/lib/v3/agent/utils/wrapEvidenceCallback.ts
Normal file
@ -0,0 +1,27 @@
|
||||
import type { AgentEvidenceCallback } from "../../types/public/agentEvidenceEvents.js";
|
||||
import type { LogLine } from "../../types/public/logs.js";
|
||||
|
||||
// onEvidence is a user-supplied observability hook (trajectory recording,
|
||||
// verifier capture, etc.). Wrap it once at the boundary where the handler
|
||||
// receives it so a throwing user callback can never abort the agent loop —
|
||||
// internal emit sites can then call the wrapped callback directly without
|
||||
// per-site try/catch.
|
||||
export function wrapEvidenceCallback(
|
||||
callback: AgentEvidenceCallback | undefined,
|
||||
logger: (message: LogLine) => void,
|
||||
): AgentEvidenceCallback | undefined {
|
||||
if (!callback) return undefined;
|
||||
return async (event) => {
|
||||
try {
|
||||
await callback(event);
|
||||
} catch (e) {
|
||||
logger({
|
||||
category: "agent",
|
||||
message: `Warning: onEvidence callback failed for ${event.type}: ${
|
||||
e instanceof Error ? e.message : String(e)
|
||||
}`,
|
||||
level: 1,
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
@ -106,6 +106,8 @@ interface ClientSessionStartParams extends Api.SessionStartRequest {
|
||||
* Optional: when omitted, requests are sent without the x-model-api-key header
|
||||
* and the server is expected to handle model authentication on its own. */
|
||||
modelApiKey?: string;
|
||||
/** Default model config for later action requests. Not sent to /sessions/start. */
|
||||
defaultModelConfig?: ModelConfiguration;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -115,6 +117,11 @@ type ApiResponse<T> =
|
||||
| { success: true; data: T }
|
||||
| { success: false; message: string };
|
||||
|
||||
type PreparedModelConfig = { modelName: string; apiKey?: string } & Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
|
||||
/**
|
||||
* Union of all API request body types for type-safe execute() calls
|
||||
*/
|
||||
@ -180,6 +187,7 @@ export class StagehandAPIClient {
|
||||
private sessionId?: string;
|
||||
private modelApiKey?: string;
|
||||
private modelProvider?: string;
|
||||
private defaultModelConfig?: PreparedModelConfig;
|
||||
private region?: BrowserbaseRegion;
|
||||
private logger: (message: LogLine) => void;
|
||||
private fetchWithCookies;
|
||||
@ -205,6 +213,7 @@ export class StagehandAPIClient {
|
||||
async init({
|
||||
modelName,
|
||||
modelApiKey,
|
||||
defaultModelConfig,
|
||||
domSettleTimeoutMs,
|
||||
verbose,
|
||||
systemPrompt,
|
||||
@ -218,6 +227,9 @@ export class StagehandAPIClient {
|
||||
this.modelProvider = modelName?.includes("/")
|
||||
? modelName.split("/")[0]
|
||||
: undefined;
|
||||
this.defaultModelConfig = defaultModelConfig
|
||||
? this.prepareModelConfig(defaultModelConfig)
|
||||
: undefined;
|
||||
|
||||
// Store the region for multi-region API URL resolution
|
||||
this.region = browserbaseSessionCreateParams?.region;
|
||||
@ -288,12 +300,18 @@ export class StagehandAPIClient {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { page: _, serverCache: enableCache, ...restOptions } = options;
|
||||
serverCache = enableCache;
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
} else if (this.defaultModelConfig) {
|
||||
restOptions.model = this.getDefaultModelConfig();
|
||||
}
|
||||
if (Object.keys(restOptions).length > 0) {
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
}
|
||||
wireOptions = restOptions as unknown as Api.ActRequest["options"];
|
||||
}
|
||||
} else if (this.defaultModelConfig) {
|
||||
wireOptions = {
|
||||
model: this.getDefaultModelConfig(),
|
||||
} as unknown as Api.ActRequest["options"];
|
||||
}
|
||||
|
||||
// Build wire-format request body
|
||||
@ -326,12 +344,18 @@ export class StagehandAPIClient {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { page: _, serverCache: enableCache, ...restOptions } = options;
|
||||
serverCache = enableCache;
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
} else if (this.defaultModelConfig) {
|
||||
restOptions.model = this.getDefaultModelConfig();
|
||||
}
|
||||
if (Object.keys(restOptions).length > 0) {
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
}
|
||||
wireOptions = restOptions as unknown as Api.ExtractRequest["options"];
|
||||
}
|
||||
} else if (this.defaultModelConfig) {
|
||||
wireOptions = {
|
||||
model: this.getDefaultModelConfig(),
|
||||
} as unknown as Api.ExtractRequest["options"];
|
||||
}
|
||||
|
||||
// Build wire-format request body
|
||||
@ -361,12 +385,18 @@ export class StagehandAPIClient {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const { page: _, serverCache: enableCache, ...restOptions } = options;
|
||||
serverCache = enableCache;
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
} else if (this.defaultModelConfig) {
|
||||
restOptions.model = this.getDefaultModelConfig();
|
||||
}
|
||||
if (Object.keys(restOptions).length > 0) {
|
||||
if (restOptions.model) {
|
||||
restOptions.model = this.prepareModelConfig(restOptions.model);
|
||||
}
|
||||
wireOptions = restOptions as unknown as Api.ObserveRequest["options"];
|
||||
}
|
||||
} else if (this.defaultModelConfig) {
|
||||
wireOptions = {
|
||||
model: this.getDefaultModelConfig(),
|
||||
} as unknown as Api.ObserveRequest["options"];
|
||||
}
|
||||
|
||||
// Build wire-format request body
|
||||
@ -388,7 +418,22 @@ export class StagehandAPIClient {
|
||||
options?: Api.NavigateRequest["options"],
|
||||
frameId?: string,
|
||||
): Promise<SerializableResponse | null> {
|
||||
const requestBody: Api.NavigateRequest = { url, options, frameId };
|
||||
const publicOptions = { ...(options ?? {}) } as NonNullable<
|
||||
Api.NavigateRequest["options"]
|
||||
> & { model?: unknown };
|
||||
delete publicOptions.model;
|
||||
|
||||
const wireOptions = {
|
||||
...publicOptions,
|
||||
...(this.defaultModelConfig
|
||||
? { model: this.getDefaultModelConfig() }
|
||||
: {}),
|
||||
};
|
||||
const requestBody: Api.NavigateRequest = {
|
||||
url,
|
||||
options: Object.keys(wireOptions).length > 0 ? wireOptions : undefined,
|
||||
frameId,
|
||||
};
|
||||
|
||||
return this.execute<SerializableResponse | null>({
|
||||
method: "navigate",
|
||||
@ -425,7 +470,7 @@ export class StagehandAPIClient {
|
||||
cua: agentConfig.mode === undefined ? agentConfig.cua : undefined,
|
||||
model: agentConfig.model
|
||||
? this.prepareModelConfig(agentConfig.model)
|
||||
: undefined,
|
||||
: this.getDefaultModelConfig(),
|
||||
executionModel: agentConfig.executionModel
|
||||
? this.prepareModelConfig(agentConfig.executionModel)
|
||||
: undefined,
|
||||
@ -605,40 +650,52 @@ export class StagehandAPIClient {
|
||||
* In API mode, we only attempt to load an API key from env vars when the
|
||||
* model provider differs from the one used to init the session.
|
||||
*/
|
||||
private prepareModelConfig(
|
||||
model: ModelConfiguration,
|
||||
): { modelName: string; apiKey?: string } & Record<string, unknown> {
|
||||
private prepareModelConfig(model: ModelConfiguration): PreparedModelConfig {
|
||||
if (typeof model === "string") {
|
||||
// Extract provider from model string (e.g., "openai/gpt-5-nano" -> "openai")
|
||||
const provider = model.includes("/") ? model.split("/")[0] : undefined;
|
||||
const provider = this.getModelProvider(model);
|
||||
const inheritedDefault =
|
||||
provider && provider === this.modelProvider
|
||||
? this.getDefaultModelConfig()
|
||||
: undefined;
|
||||
const apiKey =
|
||||
provider && provider !== this.modelProvider
|
||||
? (loadApiKeyFromEnv(provider, this.logger) ?? this.modelApiKey)
|
||||
: this.modelApiKey;
|
||||
return {
|
||||
...inheritedDefault,
|
||||
modelName: model,
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
if (!model.apiKey) {
|
||||
const provider = model.modelName?.includes("/")
|
||||
? model.modelName.split("/")[0]
|
||||
const provider = this.getModelProvider(model.modelName);
|
||||
const inheritedDefault =
|
||||
provider && provider === this.modelProvider
|
||||
? this.getDefaultModelConfig()
|
||||
: undefined;
|
||||
const apiKey =
|
||||
provider && provider !== this.modelProvider
|
||||
? (loadApiKeyFromEnv(provider, this.logger) ?? this.modelApiKey)
|
||||
: this.modelApiKey;
|
||||
return {
|
||||
...model,
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
};
|
||||
}
|
||||
const apiKey =
|
||||
!model.apiKey && provider && provider !== this.modelProvider
|
||||
? (loadApiKeyFromEnv(provider, this.logger) ?? this.modelApiKey)
|
||||
: !model.apiKey
|
||||
? this.modelApiKey
|
||||
: undefined;
|
||||
|
||||
return model as { modelName: string; apiKey: string } & Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
return {
|
||||
...inheritedDefault,
|
||||
...model,
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
} as PreparedModelConfig;
|
||||
}
|
||||
|
||||
private getDefaultModelConfig(): PreparedModelConfig | undefined {
|
||||
return this.defaultModelConfig
|
||||
? ({ ...this.defaultModelConfig } as PreparedModelConfig)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
private getModelProvider(modelName: string | undefined): string | undefined {
|
||||
return modelName?.includes("/") ? modelName.split("/")[0] : undefined;
|
||||
}
|
||||
|
||||
private consumeFinishedEventData<T>(): T | null {
|
||||
|
||||
@ -31,6 +31,7 @@ import {
|
||||
AgentModelConfig,
|
||||
Variables,
|
||||
} from "../types/public/agent.js";
|
||||
import type { AgentEvidenceCallback } from "../types/public/agentEvidenceEvents.js";
|
||||
import { HYBRID_CAPABLE_MODEL_PATTERNS } from "../types/private/agent.js";
|
||||
import { V3FunctionName } from "../types/public/methods.js";
|
||||
import { mapToolResultToActions } from "../agent/utils/actionMapping.js";
|
||||
@ -41,6 +42,12 @@ import {
|
||||
AgentAbortError,
|
||||
} from "../types/public/sdkErrors.js";
|
||||
import { handleDoneToolCall } from "../agent/utils/handleDoneToolCall.js";
|
||||
import {
|
||||
captureProbeEvidence,
|
||||
emitPostStepProbeEvidence,
|
||||
} from "../agent/utils/postStepProbeEvidence.js";
|
||||
import { wrapEvidenceCallback } from "../agent/utils/wrapEvidenceCallback.js";
|
||||
import { inferToolOutput } from "../agent/utils/toolOutputEvidence.js";
|
||||
import {
|
||||
CaptchaSolver,
|
||||
CAPTCHA_SOLVED_MSG,
|
||||
@ -51,6 +58,19 @@ function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
type FinalAnswerDraft = {
|
||||
message: string;
|
||||
output?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
interface StepHandlerOptions {
|
||||
userCallback?:
|
||||
| GenerateTextOnStepFinishCallback<ToolSet>
|
||||
| StreamTextOnStepFinishCallback<ToolSet>;
|
||||
evidenceCallback?: AgentEvidenceCallback;
|
||||
onFinalAnswer?: (answer: FinalAnswerDraft) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepends a system message with cache control to the messages array.
|
||||
* The cache control providerOptions are used by Anthropic and ignored by other providers.
|
||||
@ -244,9 +264,7 @@ export class V3AgentHandler {
|
||||
|
||||
private createStepHandler(
|
||||
state: AgentState,
|
||||
userCallback?:
|
||||
| GenerateTextOnStepFinishCallback<ToolSet>
|
||||
| StreamTextOnStepFinishCallback<ToolSet>,
|
||||
{ userCallback, evidenceCallback, onFinalAnswer }: StepHandlerOptions,
|
||||
) {
|
||||
return async (event: StepResult<ToolSet>) => {
|
||||
this.logger({
|
||||
@ -255,6 +273,8 @@ export class V3AgentHandler {
|
||||
level: 2,
|
||||
});
|
||||
|
||||
let lastFinalAnswer: FinalAnswerDraft | undefined;
|
||||
|
||||
if (event.toolCalls && event.toolCalls.length > 0) {
|
||||
for (let i = 0; i < event.toolCalls.length; i++) {
|
||||
const toolCall = event.toolCalls[i];
|
||||
@ -279,6 +299,13 @@ export class V3AgentHandler {
|
||||
? `${allReasoning} ${doneReasoning}`.trim()
|
||||
: allReasoning || "Task completed successfully";
|
||||
}
|
||||
lastFinalAnswer = {
|
||||
message: state.finalMessage,
|
||||
output:
|
||||
typeof args?.output === "object" && args?.output !== null
|
||||
? (args.output as Record<string, unknown>)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
const mappedActions = mapToolResultToActions({
|
||||
toolCallName: toolCall.toolName,
|
||||
@ -292,8 +319,35 @@ export class V3AgentHandler {
|
||||
action.timestamp = Date.now();
|
||||
state.actions.push(action);
|
||||
}
|
||||
|
||||
await evidenceCallback?.({
|
||||
type: "step_finished",
|
||||
actionName: toolCall.toolName,
|
||||
actionArgs:
|
||||
typeof args === "object" && args !== null
|
||||
? (args as Record<string, unknown>)
|
||||
: {},
|
||||
reasoning: event.text ?? "",
|
||||
toolOutput: inferToolOutput(toolResult),
|
||||
});
|
||||
}
|
||||
state.currentPageUrl = (await this.v3.context.awaitActivePage()).url();
|
||||
|
||||
// Harness probe — one screenshot / a11y snapshot per AI SDK step.
|
||||
// The recorder applies the probe to every step_finished received
|
||||
// since the previous probe, so a multi-tool turn shares the same
|
||||
// post-turn observation.
|
||||
await emitPostStepProbeEvidence({
|
||||
v3: this.v3,
|
||||
url: state.currentPageUrl,
|
||||
evidenceCallback,
|
||||
logger: this.logger,
|
||||
warningMessage: "Warning: harness probe failed",
|
||||
});
|
||||
}
|
||||
|
||||
if (lastFinalAnswer) {
|
||||
onFinalAnswer?.(lastFinalAnswer);
|
||||
}
|
||||
|
||||
if (userCallback) {
|
||||
@ -321,6 +375,7 @@ export class V3AgentHandler {
|
||||
completed: false,
|
||||
currentPageUrl: "",
|
||||
};
|
||||
let finalAnswerFromDoneTool: FinalAnswerDraft | undefined;
|
||||
|
||||
let messages: ModelMessage[] = [];
|
||||
let captchaSolver: CaptchaSolver | undefined;
|
||||
@ -368,6 +423,11 @@ export class V3AgentHandler {
|
||||
}
|
||||
}
|
||||
|
||||
const evidenceCallback = wrapEvidenceCallback(
|
||||
callbacks?.onEvidence,
|
||||
this.logger,
|
||||
);
|
||||
|
||||
const result = await this.llmClient.generateText({
|
||||
model: wrappedModel,
|
||||
messages: prependSystemMessage(systemPrompt, messages),
|
||||
@ -379,7 +439,13 @@ export class V3AgentHandler {
|
||||
callbacks?.prepareStep,
|
||||
captchaSolver,
|
||||
),
|
||||
onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
|
||||
onStepFinish: this.createStepHandler(state, {
|
||||
userCallback: callbacks?.onStepFinish,
|
||||
evidenceCallback,
|
||||
onFinalAnswer: (answer) => {
|
||||
finalAnswerFromDoneTool = answer;
|
||||
},
|
||||
}),
|
||||
abortSignal: preparedOptions.signal,
|
||||
providerOptions: {
|
||||
google: { mediaResolution: "MEDIA_RESOLUTION_HIGH" },
|
||||
@ -396,6 +462,15 @@ export class V3AgentHandler {
|
||||
preparedOptions.output,
|
||||
this.logger,
|
||||
);
|
||||
const output = doneResult.output ?? finalAnswerFromDoneTool?.output;
|
||||
await this.emitFinalEvidence(
|
||||
state,
|
||||
{
|
||||
message: state.finalMessage,
|
||||
output,
|
||||
},
|
||||
evidenceCallback,
|
||||
);
|
||||
|
||||
return this.consolidateMetricsAndResult(
|
||||
startTime,
|
||||
@ -403,7 +478,7 @@ export class V3AgentHandler {
|
||||
doneResult.messages,
|
||||
result,
|
||||
maxSteps,
|
||||
doneResult.output,
|
||||
output,
|
||||
);
|
||||
} catch (error) {
|
||||
// Re-throw validation errors that should propagate to the caller
|
||||
@ -449,6 +524,7 @@ export class V3AgentHandler {
|
||||
// Highlight cursor defaults to true for hybrid mode, can be overridden
|
||||
const shouldHighlightCursor =
|
||||
streamOptions?.highlightCursor ?? this.mode === "hybrid";
|
||||
let finalAnswerFromDoneTool: FinalAnswerDraft | undefined;
|
||||
|
||||
const {
|
||||
options,
|
||||
@ -503,6 +579,11 @@ export class V3AgentHandler {
|
||||
rejectResult(error);
|
||||
};
|
||||
|
||||
const evidenceCallback = wrapEvidenceCallback(
|
||||
callbacks?.onEvidence,
|
||||
this.logger,
|
||||
);
|
||||
|
||||
let streamResult: ReturnType<typeof this.llmClient.streamText>;
|
||||
try {
|
||||
streamResult = this.llmClient.streamText({
|
||||
@ -515,7 +596,13 @@ export class V3AgentHandler {
|
||||
callbacks?.prepareStep,
|
||||
captchaSolver,
|
||||
),
|
||||
onStepFinish: this.createStepHandler(state, callbacks?.onStepFinish),
|
||||
onStepFinish: this.createStepHandler(state, {
|
||||
userCallback: callbacks?.onStepFinish,
|
||||
evidenceCallback,
|
||||
onFinalAnswer: (answer) => {
|
||||
finalAnswerFromDoneTool = answer;
|
||||
},
|
||||
}),
|
||||
onError: (event) => {
|
||||
captchaSolver?.dispose();
|
||||
if (callbacks?.onError) {
|
||||
@ -541,17 +628,29 @@ export class V3AgentHandler {
|
||||
options.instruction,
|
||||
options.output,
|
||||
this.logger,
|
||||
).then((doneResult) => {
|
||||
const result = this.consolidateMetricsAndResult(
|
||||
startTime,
|
||||
state,
|
||||
doneResult.messages,
|
||||
event,
|
||||
maxSteps,
|
||||
doneResult.output,
|
||||
);
|
||||
resolveResult(result);
|
||||
});
|
||||
)
|
||||
.then(async (doneResult) => {
|
||||
const output =
|
||||
doneResult.output ?? finalAnswerFromDoneTool?.output;
|
||||
await this.emitFinalEvidence(
|
||||
state,
|
||||
{
|
||||
message: state.finalMessage,
|
||||
output,
|
||||
},
|
||||
evidenceCallback,
|
||||
);
|
||||
const result = this.consolidateMetricsAndResult(
|
||||
startTime,
|
||||
state,
|
||||
doneResult.messages,
|
||||
event,
|
||||
maxSteps,
|
||||
output,
|
||||
);
|
||||
resolveResult(result);
|
||||
})
|
||||
.catch(handleError);
|
||||
},
|
||||
onAbort: (event) => {
|
||||
captchaSolver?.dispose();
|
||||
@ -580,6 +679,26 @@ export class V3AgentHandler {
|
||||
return agentStreamResult;
|
||||
}
|
||||
|
||||
private async emitFinalEvidence(
|
||||
state: AgentState,
|
||||
finalAnswer: { message: string; output?: Record<string, unknown> },
|
||||
evidenceCallback?: AgentEvidenceCallback,
|
||||
): Promise<void> {
|
||||
if (!evidenceCallback) return;
|
||||
|
||||
const observation = await captureProbeEvidence({
|
||||
v3: this.v3,
|
||||
url: state.currentPageUrl,
|
||||
logger: this.logger,
|
||||
warningMessage: "Warning: final harness probe failed",
|
||||
});
|
||||
await evidenceCallback({
|
||||
type: "final_answer",
|
||||
...finalAnswer,
|
||||
observation,
|
||||
});
|
||||
}
|
||||
|
||||
private consolidateMetricsAndResult(
|
||||
startTime: number,
|
||||
state: AgentState,
|
||||
|
||||
@ -7,6 +7,12 @@ import { GoogleCUAClient } from "../agent/GoogleCUAClient.js";
|
||||
import { OpenAICUAClient } from "../agent/OpenAICUAClient.js";
|
||||
import { mapKeyToPlaywright } from "../agent/utils/cuaKeyMapping.js";
|
||||
import { ensureXPath } from "../agent/utils/xpath.js";
|
||||
import {
|
||||
captureProbeEvidence,
|
||||
emitPostStepProbeEvidence,
|
||||
} from "../agent/utils/postStepProbeEvidence.js";
|
||||
import { wrapEvidenceCallback } from "../agent/utils/wrapEvidenceCallback.js";
|
||||
import { inferToolOutput } from "../agent/utils/toolOutputEvidence.js";
|
||||
import {
|
||||
ActionExecutionResult,
|
||||
AgentAction,
|
||||
@ -16,6 +22,10 @@ import {
|
||||
SafetyConfirmationHandler,
|
||||
} from "../types/public/agent.js";
|
||||
import { LogLine } from "../types/public/logs.js";
|
||||
import type {
|
||||
AgentEvidenceCallback,
|
||||
AgentStepFinishedEvent,
|
||||
} from "../types/public/agentEvidenceEvents.js";
|
||||
import { type Action, V3FunctionName } from "../types/public/methods.js";
|
||||
import { FlowLogger } from "../flowlogger/FlowLogger.js";
|
||||
import { toTitleCase } from "../../utils.js";
|
||||
@ -37,6 +47,8 @@ export class V3CuaAgentHandler {
|
||||
private captchaSolver: CaptchaSolver | null = null;
|
||||
private captchaClickGuardRemaining = 0;
|
||||
private currentInstruction = "";
|
||||
private lastAgentScreenshotUrl?: string;
|
||||
private evidenceCallback?: AgentEvidenceCallback;
|
||||
|
||||
constructor(
|
||||
v3: V3,
|
||||
@ -76,6 +88,9 @@ export class V3CuaAgentHandler {
|
||||
this.ensureNotClosed();
|
||||
const page = await this.v3.context.awaitActivePage();
|
||||
const screenshotBuffer = await page.screenshot({ fullPage: false });
|
||||
|
||||
await this.emitCuaScreenshot(screenshotBuffer, page.url());
|
||||
|
||||
return screenshotBuffer.toString("base64"); // base64 png
|
||||
});
|
||||
|
||||
@ -119,7 +134,12 @@ export class V3CuaAgentHandler {
|
||||
const waitBetween =
|
||||
(this.options.clientOptions?.waitBetweenActions as number) ||
|
||||
defaultDelay;
|
||||
// Skip logging for screenshot actions - they're no-ops; the CUA client
|
||||
// takes its own screenshot via screenshotProvider between API turns.
|
||||
// Computed outside the try so the catch can still record a failed step.
|
||||
const shouldLog = action.type !== "screenshot";
|
||||
try {
|
||||
let executionResult: ActionExecutionResult | undefined;
|
||||
// Try to inject cursor before each action if enabled
|
||||
if (this.highlightCursor) {
|
||||
try {
|
||||
@ -129,11 +149,8 @@ export class V3CuaAgentHandler {
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
// Skip logging for screenshot actions - they're no-ops; the CUA client
|
||||
// takes its own screenshot via screenshotProvider between API turns.
|
||||
const shouldLog = action.type !== "screenshot";
|
||||
if (shouldLog) {
|
||||
await FlowLogger.runWithLogging(
|
||||
executionResult = await FlowLogger.runWithLogging(
|
||||
{
|
||||
eventType: `V3Cua${toTitleCase(action.type)}`, // e.g. "V3CuaClick"
|
||||
data: {
|
||||
@ -145,10 +162,16 @@ export class V3CuaAgentHandler {
|
||||
[action],
|
||||
);
|
||||
} else {
|
||||
await this.executeAction(action);
|
||||
executionResult = await this.executeAction(action);
|
||||
}
|
||||
|
||||
action.timestamp = Date.now();
|
||||
if (shouldLog && this.evidenceCallback) {
|
||||
await this.emitCuaActionStep(
|
||||
action,
|
||||
inferToolOutput(executionResult ?? { success: true }),
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, waitBetween));
|
||||
} catch (error) {
|
||||
@ -158,6 +181,30 @@ export class V3CuaAgentHandler {
|
||||
message: `Error executing action ${action.type}: ${msg}`,
|
||||
level: 0,
|
||||
});
|
||||
// Record the failed action as an ok:false step (with a best-effort
|
||||
// post-failure probe, since a throwing action can still partially
|
||||
// mutate the page) before rethrowing — otherwise the failure is
|
||||
// dropped from the persisted trajectory. Evidence emission must never
|
||||
// mask the original action error.
|
||||
if (shouldLog && this.evidenceCallback) {
|
||||
try {
|
||||
await this.emitCuaActionStep(action, {
|
||||
ok: false,
|
||||
result: undefined,
|
||||
error: msg,
|
||||
});
|
||||
} catch (evidenceError) {
|
||||
this.logger({
|
||||
category: "agent",
|
||||
message: `Failed to record failed-action evidence: ${
|
||||
evidenceError instanceof Error
|
||||
? evidenceError.message
|
||||
: String(evidenceError)
|
||||
}`,
|
||||
level: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
@ -184,6 +231,11 @@ export class V3CuaAgentHandler {
|
||||
: optionsOrInstruction;
|
||||
|
||||
this.setSafetyConfirmationHandler(options.callbacks?.onSafetyConfirmation);
|
||||
this.evidenceCallback = wrapEvidenceCallback(
|
||||
options.callbacks?.onEvidence,
|
||||
this.logger,
|
||||
);
|
||||
this.lastAgentScreenshotUrl = undefined;
|
||||
|
||||
this.highlightCursor = options.highlightCursor !== false;
|
||||
this.currentInstruction = options.instruction;
|
||||
@ -239,7 +291,28 @@ export class V3CuaAgentHandler {
|
||||
let result: AgentResult;
|
||||
try {
|
||||
result = await this.agent.execute({ options, logger: this.logger });
|
||||
if (this.evidenceCallback) {
|
||||
let finalUrl = "";
|
||||
try {
|
||||
finalUrl = (await this.v3.context.awaitActivePage()).url();
|
||||
} catch {
|
||||
finalUrl = this.lastAgentScreenshotUrl ?? "";
|
||||
}
|
||||
const observation = await captureProbeEvidence({
|
||||
v3: this.v3,
|
||||
url: finalUrl,
|
||||
logger: this.logger,
|
||||
warningMessage: "Warning: CUA final probe failed",
|
||||
});
|
||||
await this.evidenceCallback({
|
||||
type: "final_answer",
|
||||
message: result.message,
|
||||
output: result.output,
|
||||
observation,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
this.evidenceCallback = undefined;
|
||||
this.captchaSolver?.dispose();
|
||||
this.captchaSolver = null;
|
||||
}
|
||||
@ -658,6 +731,10 @@ export class V3CuaAgentHandler {
|
||||
const screenshotBuffer = await page.screenshot({ fullPage: false });
|
||||
|
||||
const currentUrl = page.url();
|
||||
|
||||
// Mirror the same buffer the CUA client receives as agent evidence.
|
||||
await this.emitCuaScreenshot(screenshotBuffer, currentUrl);
|
||||
|
||||
return await this.agentClient.captureScreenshot({
|
||||
base64Image: screenshotBuffer.toString("base64"),
|
||||
currentUrl,
|
||||
@ -767,6 +844,75 @@ export class V3CuaAgentHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a pre-action CUA screenshot — the exact buffer the model received
|
||||
* as input. Tier-1 evidence (agent-mirrored); the tier-2 probe is taken
|
||||
* separately in emitCuaActionStep after the action runs, so the recorder
|
||||
* can compare what the model saw against what the page actually showed
|
||||
* once the keystrokes/clicks landed.
|
||||
*/
|
||||
private async emitCuaScreenshot(
|
||||
screenshot: Buffer,
|
||||
url: string,
|
||||
): Promise<void> {
|
||||
if (!this.evidenceCallback) return;
|
||||
this.lastAgentScreenshotUrl = url;
|
||||
await this.evidenceCallback({
|
||||
type: "screenshot",
|
||||
screenshot,
|
||||
url,
|
||||
evidenceRole: "agent",
|
||||
});
|
||||
}
|
||||
|
||||
private async emitCuaActionStep(
|
||||
action: AgentAction,
|
||||
toolOutput: AgentStepFinishedEvent["toolOutput"],
|
||||
): Promise<void> {
|
||||
let pageUrl =
|
||||
typeof action.pageUrl === "string"
|
||||
? action.pageUrl
|
||||
: (this.lastAgentScreenshotUrl ?? "");
|
||||
try {
|
||||
pageUrl = (await this.v3.context.awaitActivePage()).url();
|
||||
} catch {
|
||||
// Keep the best pre-action URL fallback.
|
||||
}
|
||||
|
||||
const actionArgs = Object.fromEntries(
|
||||
Object.entries(action).filter(
|
||||
([key]) => key !== "screenshot" && key !== "timestamp",
|
||||
),
|
||||
);
|
||||
const reasoning =
|
||||
typeof action.reasoning === "string"
|
||||
? action.reasoning
|
||||
: typeof action.action === "string"
|
||||
? action.action
|
||||
: "";
|
||||
|
||||
await this.evidenceCallback?.({
|
||||
type: "step_finished",
|
||||
actionName: String(action.type),
|
||||
actionArgs,
|
||||
reasoning,
|
||||
toolOutput,
|
||||
});
|
||||
|
||||
// Post-action tier-2 probe. The pre-action screenshot from
|
||||
// screenshotProvider is what the model SAW; this one shows what the
|
||||
// page actually LOOKS LIKE after the action ran. Without this the
|
||||
// verifier has no visual evidence that keystrokes/clicks landed, and
|
||||
// has to trust the action history alone.
|
||||
await emitPostStepProbeEvidence({
|
||||
v3: this.v3,
|
||||
url: pageUrl,
|
||||
evidenceCallback: this.evidenceCallback,
|
||||
logger: this.logger,
|
||||
warningMessage: "Warning: CUA post-action probe failed",
|
||||
});
|
||||
}
|
||||
|
||||
private async injectCursor(): Promise<void> {
|
||||
try {
|
||||
const page = await this.v3.context.awaitActivePage();
|
||||
|
||||
@ -25,9 +25,14 @@ import { getAISDKLanguageModel } from "./llm/LLMProvider.js";
|
||||
import { __internalCreateInMemoryAgentCacheHandle } from "./cache/serverAgentCache.js";
|
||||
import { maybeRunShutdownSupervisorFromArgv } from "./shutdown/supervisor.js";
|
||||
import {
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
loadTrajectoryFromDisk,
|
||||
mergeAgentEvidence,
|
||||
nextResultFilename,
|
||||
normalizeRubric,
|
||||
redactInlineImagePayloads,
|
||||
shouldPersistTrajectory,
|
||||
writeTrajectoryDir,
|
||||
} from "./verifier/index.js";
|
||||
|
||||
export { V3 } from "./v3.js";
|
||||
@ -90,9 +95,14 @@ export type {
|
||||
VerifierRawSteps,
|
||||
} from "./verifier/index.js";
|
||||
export {
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
loadTrajectoryFromDisk,
|
||||
mergeAgentEvidence,
|
||||
nextResultFilename,
|
||||
normalizeRubric,
|
||||
redactInlineImagePayloads,
|
||||
shouldPersistTrajectory,
|
||||
writeTrajectoryDir,
|
||||
} from "./verifier/index.js";
|
||||
export { tool } from "ai";
|
||||
export { getAISDKLanguageModel } from "./llm/LLMProvider.js";
|
||||
@ -144,9 +154,14 @@ const StagehandDefault = {
|
||||
toJsonSchema,
|
||||
connectToMCPServer,
|
||||
V3Evaluator,
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
loadTrajectoryFromDisk,
|
||||
mergeAgentEvidence,
|
||||
nextResultFilename,
|
||||
normalizeRubric,
|
||||
redactInlineImagePayloads,
|
||||
shouldPersistTrajectory,
|
||||
writeTrajectoryDir,
|
||||
tool,
|
||||
getAISDKLanguageModel,
|
||||
__internalCreateInMemoryAgentCacheHandle,
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
import type { LanguageModelV2Middleware } from "@ai-sdk/provider";
|
||||
import {
|
||||
ExperimentalNotConfiguredError,
|
||||
UnsupportedAISDKModelProviderError,
|
||||
UnsupportedModelError,
|
||||
UnsupportedModelProviderError,
|
||||
@ -70,6 +69,37 @@ const AISDKProvidersWithAPIKey: Record<string, AISDKCustomProvider> = {
|
||||
gateway: createGateway,
|
||||
};
|
||||
|
||||
type AISDKProviderClientOptions = ClientOptions & Record<string, unknown>;
|
||||
|
||||
export function toAISDKClientOptions(
|
||||
subProvider: string,
|
||||
clientOptions?: ClientOptions,
|
||||
): AISDKProviderClientOptions | undefined {
|
||||
if (!clientOptions || subProvider !== "vertex") {
|
||||
return clientOptions as AISDKProviderClientOptions | undefined;
|
||||
}
|
||||
|
||||
const { auth, providerOptions, ...rest } = clientOptions;
|
||||
const vertexOptions = providerOptions?.vertex;
|
||||
|
||||
return {
|
||||
...rest,
|
||||
...(vertexOptions ?? {}),
|
||||
...(auth?.type === "googleServiceAccount"
|
||||
? {
|
||||
googleAuthOptions: {
|
||||
credentials: auth.credentials,
|
||||
...(auth.scopes ? { scopes: auth.scopes } : {}),
|
||||
...(auth.projectId ? { projectId: auth.projectId } : {}),
|
||||
...(auth.universeDomain
|
||||
? { universeDomain: auth.universeDomain }
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
const modelToProviderMap: { [key in AvailableModel]: ModelProvider } = {
|
||||
"gpt-4.1": "openai",
|
||||
"gpt-4.1-mini": "openai",
|
||||
@ -106,9 +136,12 @@ export function getAISDKLanguageModel(
|
||||
clientOptions?: ClientOptions,
|
||||
middleware?: LanguageModelV2Middleware,
|
||||
) {
|
||||
const aiSdkClientOptions = toAISDKClientOptions(subProvider, clientOptions);
|
||||
const hasValidOptions =
|
||||
clientOptions &&
|
||||
Object.values(clientOptions).some((v) => v !== undefined && v !== null);
|
||||
aiSdkClientOptions &&
|
||||
Object.values(aiSdkClientOptions).some(
|
||||
(v) => v !== undefined && v !== null,
|
||||
);
|
||||
|
||||
let model;
|
||||
if (hasValidOptions) {
|
||||
@ -119,7 +152,7 @@ export function getAISDKLanguageModel(
|
||||
Object.keys(AISDKProvidersWithAPIKey),
|
||||
);
|
||||
}
|
||||
const provider = creator(clientOptions);
|
||||
const provider = creator(aiSdkClientOptions as ClientOptions);
|
||||
model = provider(subModelName);
|
||||
} else {
|
||||
const provider = AISDKProviders[subProvider];
|
||||
@ -163,13 +196,6 @@ export class LLMProvider {
|
||||
const firstSlashIndex = modelName.indexOf("/");
|
||||
const subProvider = modelName.substring(0, firstSlashIndex);
|
||||
const subModelName = modelName.substring(firstSlashIndex + 1);
|
||||
if (
|
||||
subProvider === "vertex" &&
|
||||
!options?.disableAPI &&
|
||||
!options?.experimental
|
||||
) {
|
||||
throw new ExperimentalNotConfiguredError("Vertex provider");
|
||||
}
|
||||
|
||||
const effectiveMiddleware = options?.middleware ?? this.middleware;
|
||||
const languageModel = getAISDKLanguageModel(
|
||||
|
||||
@ -15,6 +15,7 @@ import {
|
||||
import { LogLine } from "./logs.js";
|
||||
import { ClientOptions } from "./model.js";
|
||||
import { StagehandZodObject } from "../../zodCompat.js";
|
||||
import type { AgentEvidenceCallback } from "./agentEvidenceEvents.js";
|
||||
|
||||
// Re-export ModelMessage for consumers who want to use it for conversation continuation
|
||||
export type { ModelMessage } from "ai";
|
||||
@ -136,6 +137,11 @@ export interface AgentCallbacks {
|
||||
onStepFinish?:
|
||||
| GenerateTextOnStepFinishCallback<ToolSet>
|
||||
| StreamTextOnStepFinishCallback<ToolSet>;
|
||||
/**
|
||||
* Callback called when Stagehand captures agent-run evidence such as
|
||||
* screenshots, completed tool/action steps, or post-action observations.
|
||||
*/
|
||||
onEvidence?: AgentEvidenceCallback;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -456,6 +462,7 @@ export const AVAILABLE_CUA_MODELS = [
|
||||
"openai/computer-use-preview-2025-03-11",
|
||||
"anthropic/claude-opus-4-5-20251101",
|
||||
"anthropic/claude-opus-4-6",
|
||||
"anthropic/claude-opus-4-8",
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"anthropic/claude-haiku-4-5",
|
||||
"anthropic/claude-haiku-4-5-20251001",
|
||||
|
||||
99
packages/core/lib/v3/types/public/agentEvidenceEvents.ts
Normal file
99
packages/core/lib/v3/types/public/agentEvidenceEvents.ts
Normal file
@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Evidence events emitted through AgentExecuteOptions.callbacks.onEvidence.
|
||||
*
|
||||
* These events describe observations made by Stagehand during an agent run.
|
||||
* They are emitted in temporal order; consumers should treat the stream as
|
||||
* sequential. An agent-role screenshot applies to every subsequent
|
||||
* step_finished until a newer agent-role screenshot replaces it — a CUA
|
||||
* provider may choose multiple actions from a single screenshot, so each of
|
||||
* those steps shares that frame. A step_observed/probe applies to all
|
||||
* step_finished events received since the last probe. Verifier-specific
|
||||
* storage and normalization live in the evals/verifier layers.
|
||||
*/
|
||||
|
||||
export type AgentEvidenceRole = "probe" | "agent";
|
||||
|
||||
export type AgentEvidenceEvent =
|
||||
| AgentScreenshotEvidenceEvent
|
||||
| AgentStepFinishedEvent
|
||||
| AgentStepObservedEvent
|
||||
| AgentFinalAnswerEvent;
|
||||
|
||||
/**
|
||||
* Screenshot captured during an agent run.
|
||||
*
|
||||
* In DOM/hybrid mode, post-tool screenshots are probe evidence. In CUA mode,
|
||||
* screenshots captured by the screenshot provider are agent evidence because
|
||||
* they are the exact bytes sent to the provider.
|
||||
*/
|
||||
export interface AgentScreenshotEvidenceEvent {
|
||||
type: "screenshot";
|
||||
/** PNG bytes from page.screenshot(). */
|
||||
screenshot: Buffer;
|
||||
/** Page URL at the time of capture. */
|
||||
url: string;
|
||||
/** Role this screenshot plays in downstream evidence collection. */
|
||||
evidenceRole: AgentEvidenceRole;
|
||||
}
|
||||
|
||||
/**
|
||||
* One completed agent tool/action step.
|
||||
*/
|
||||
export interface AgentStepFinishedEvent {
|
||||
type: "step_finished";
|
||||
/** Name of the tool/action that ran, e.g. "act", "extract", "click". */
|
||||
actionName: string;
|
||||
/** Arguments passed to the tool/action. */
|
||||
actionArgs: Record<string, unknown>;
|
||||
/** Agent textual reasoning for the step, when available. */
|
||||
reasoning: string;
|
||||
/** Outcome of the tool/action as seen by Stagehand. */
|
||||
toolOutput: {
|
||||
ok: boolean;
|
||||
/** Native return value from the tool/action. */
|
||||
result: unknown;
|
||||
error?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Independent post-step browser observation. Emitted once per agent turn;
|
||||
* consumers apply it to every step_finished received since the previous probe.
|
||||
*/
|
||||
export interface AgentStepObservedEvent {
|
||||
type: "step_observed";
|
||||
/** Page URL after the step's tool/action execution. */
|
||||
url: string;
|
||||
/** Accessibility tree snapshot, when captured. */
|
||||
ariaTree?: string;
|
||||
}
|
||||
|
||||
export interface AgentFinalObservation {
|
||||
/** Page URL at the time of terminal capture. */
|
||||
url: string;
|
||||
/** PNG bytes from page.screenshot(), when capture succeeds. */
|
||||
screenshot?: Buffer;
|
||||
/** Accessibility tree snapshot, when captured. */
|
||||
ariaTree?: string;
|
||||
}
|
||||
|
||||
/** Final answer emitted by the agent, when available. */
|
||||
export interface AgentFinalAnswerEvent {
|
||||
type: "final_answer";
|
||||
/** The agent's final summary message. */
|
||||
message: string;
|
||||
/** Optional structured output if the agent's output schema was set. */
|
||||
output?: Record<string, unknown>;
|
||||
/**
|
||||
* Independent terminal browser observation captured after the agent finishes.
|
||||
*
|
||||
* This preserves the legacy verifier behavior of evaluating against a final
|
||||
* page screenshot even when the last agent output is a final answer rather
|
||||
* than a browser action.
|
||||
*/
|
||||
observation?: AgentFinalObservation;
|
||||
}
|
||||
|
||||
export type AgentEvidenceCallback = (
|
||||
event: AgentEvidenceEvent,
|
||||
) => PromiseLike<void> | void;
|
||||
@ -73,9 +73,13 @@ export const GoogleServiceAccountCredentialsSchema = z
|
||||
.strict()
|
||||
.meta({ id: "GoogleServiceAccountCredentials" });
|
||||
|
||||
export const GoogleAuthOptionsSchema = z
|
||||
export const GoogleServiceAccountAuthSchema = z
|
||||
.object({
|
||||
credentials: GoogleServiceAccountCredentialsSchema.optional().meta({
|
||||
type: z.literal("googleServiceAccount").meta({
|
||||
description:
|
||||
"Use inline Google Cloud service account credentials for provider authentication",
|
||||
}),
|
||||
credentials: GoogleServiceAccountCredentialsSchema.meta({
|
||||
description: "Google Cloud service account credentials",
|
||||
}),
|
||||
scopes: z
|
||||
@ -92,18 +96,44 @@ export const GoogleAuthOptionsSchema = z
|
||||
}),
|
||||
})
|
||||
.strict()
|
||||
.meta({ id: "GoogleAuthOptions" });
|
||||
.meta({ id: "GoogleServiceAccountAuth" });
|
||||
|
||||
export const ModelConfigObjectSchema = z
|
||||
export const ModelAuthSchema = z
|
||||
.discriminatedUnion("type", [GoogleServiceAccountAuthSchema])
|
||||
.meta({ id: "ModelAuth" });
|
||||
|
||||
export const VertexProviderOptionsSchema = z
|
||||
.object({
|
||||
project: z.string().meta({
|
||||
description: "Google Cloud project ID for Vertex AI models",
|
||||
example: "my-gcp-project",
|
||||
}),
|
||||
location: z.string().meta({
|
||||
description: "Google Cloud location for Vertex AI models",
|
||||
example: "us-central1",
|
||||
}),
|
||||
baseURL: z.string().url().optional().meta({
|
||||
description: "Base URL for the Vertex AI provider",
|
||||
}),
|
||||
headers: z.record(z.string(), z.string()).optional().meta({
|
||||
description:
|
||||
"Custom headers sent with every request to the Vertex AI provider",
|
||||
}),
|
||||
})
|
||||
.strict()
|
||||
.meta({ id: "VertexProviderOptions" });
|
||||
|
||||
export const ModelProviderOptionsSchema = z
|
||||
.object({
|
||||
vertex: VertexProviderOptionsSchema.meta({
|
||||
description: "Vertex AI provider-specific settings",
|
||||
}),
|
||||
})
|
||||
.strict()
|
||||
.meta({ id: "ModelProviderOptions" });
|
||||
|
||||
const ModelConfigBaseSchema = z
|
||||
.object({
|
||||
provider: z
|
||||
.enum(["openai", "anthropic", "google", "microsoft", "bedrock", "vertex"])
|
||||
.optional()
|
||||
.meta({
|
||||
description:
|
||||
"AI provider for the model (or provide a baseURL endpoint instead)",
|
||||
example: "openai",
|
||||
}),
|
||||
modelName: z.string().meta({
|
||||
description:
|
||||
"Model name string with provider prefix (e.g., 'openai/gpt-5-nano')",
|
||||
@ -121,19 +151,38 @@ export const ModelConfigObjectSchema = z
|
||||
description:
|
||||
"Custom headers sent with every request to the model provider",
|
||||
}),
|
||||
project: z.string().optional().meta({
|
||||
description: "Google Cloud project ID for Vertex AI models",
|
||||
example: "my-gcp-project",
|
||||
}),
|
||||
location: z.string().optional().meta({
|
||||
description: "Google Cloud location for Vertex AI models",
|
||||
example: "us-central1",
|
||||
}),
|
||||
googleAuthOptions: GoogleAuthOptionsSchema.optional().meta({
|
||||
description:
|
||||
"google-auth-library options used to authenticate Vertex AI models",
|
||||
}),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const GenericModelConfigObjectSchema = ModelConfigBaseSchema.extend({
|
||||
provider: z
|
||||
.enum(["openai", "anthropic", "google", "microsoft", "bedrock"])
|
||||
.optional()
|
||||
.meta({
|
||||
description:
|
||||
"AI provider for the model (or provide a baseURL endpoint instead)",
|
||||
example: "openai",
|
||||
}),
|
||||
})
|
||||
.strict()
|
||||
.meta({ id: "GenericModelConfigObject" });
|
||||
|
||||
export const VertexModelConfigObjectSchema = ModelConfigBaseSchema.extend({
|
||||
provider: z.literal("vertex").meta({
|
||||
description: "Vertex AI model provider",
|
||||
}),
|
||||
auth: ModelAuthSchema.meta({
|
||||
description: "Vertex provider authentication configuration",
|
||||
}),
|
||||
providerOptions: ModelProviderOptionsSchema.meta({
|
||||
description: "Vertex provider-specific model configuration",
|
||||
}),
|
||||
})
|
||||
.strict()
|
||||
.meta({ id: "VertexModelConfigObject" });
|
||||
|
||||
export const ModelConfigObjectSchema = z
|
||||
.union([VertexModelConfigObjectSchema, GenericModelConfigObjectSchema])
|
||||
.meta({ id: "ModelConfigObject" });
|
||||
|
||||
/** Model configuration */
|
||||
@ -1161,6 +1210,21 @@ export const Operations = {
|
||||
// Shared types
|
||||
export type Action = z.infer<typeof ActionSchema>;
|
||||
export type ModelConfig = z.infer<typeof ModelConfigSchema>;
|
||||
export type GenericModelConfigObject = z.infer<
|
||||
typeof GenericModelConfigObjectSchema
|
||||
>;
|
||||
export type VertexModelConfigObject = z.infer<
|
||||
typeof VertexModelConfigObjectSchema
|
||||
>;
|
||||
export type GoogleServiceAccountCredentials = z.infer<
|
||||
typeof GoogleServiceAccountCredentialsSchema
|
||||
>;
|
||||
export type GoogleServiceAccountAuth = z.infer<
|
||||
typeof GoogleServiceAccountAuthSchema
|
||||
>;
|
||||
export type ModelAuth = z.infer<typeof ModelAuthSchema>;
|
||||
export type VertexProviderOptions = z.infer<typeof VertexProviderOptionsSchema>;
|
||||
export type ModelProviderOptions = z.infer<typeof ModelProviderOptionsSchema>;
|
||||
export type BrowserConfig = z.infer<typeof BrowserConfigSchema>;
|
||||
export type SessionIdParams = z.infer<typeof SessionIdParamsSchema>;
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
export * from "./agent.js";
|
||||
export * from "./agentEvidenceEvents.js";
|
||||
// Export api.ts under namespace to avoid conflicts with methods.ts types
|
||||
export * as Api from "./api.js";
|
||||
// Also export BrowserbaseRegion directly for convenience
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import type { ClientOptions as AnthropicClientOptionsBase } from "@anthropic-ai/sdk";
|
||||
import type { GoogleVertexProviderSettings as GoogleVertexProviderSettingsBase } from "@ai-sdk/google-vertex";
|
||||
import type {
|
||||
LanguageModelV2,
|
||||
LanguageModelV2Middleware,
|
||||
@ -31,19 +30,26 @@ export interface GoogleServiceAccountCredentials {
|
||||
universe_domain?: string;
|
||||
}
|
||||
|
||||
export interface GoogleVertexAuthOptions {
|
||||
credentials?: GoogleServiceAccountCredentials;
|
||||
export interface GoogleServiceAccountAuth {
|
||||
type: "googleServiceAccount";
|
||||
credentials: GoogleServiceAccountCredentials;
|
||||
scopes?: string | string[];
|
||||
projectId?: string;
|
||||
universeDomain?: string;
|
||||
}
|
||||
|
||||
export type GoogleVertexProviderSettings = Pick<
|
||||
GoogleVertexProviderSettingsBase,
|
||||
"project" | "location" | "headers" | "baseURL"
|
||||
> & {
|
||||
googleAuthOptions?: GoogleVertexAuthOptions;
|
||||
};
|
||||
export type ModelAuth = GoogleServiceAccountAuth;
|
||||
|
||||
export interface VertexProviderOptions {
|
||||
project: string;
|
||||
location: string;
|
||||
baseURL?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ModelProviderOptions {
|
||||
vertex: VertexProviderOptions;
|
||||
}
|
||||
|
||||
export type AnthropicJsonSchemaObject = {
|
||||
definitions?: {
|
||||
@ -113,13 +119,11 @@ export type ModelProvider =
|
||||
*/
|
||||
export type ThinkingEffort = "none" | "low" | "medium" | "high" | "max";
|
||||
|
||||
export type ClientOptions = (
|
||||
| OpenAIClientOptions
|
||||
| AnthropicClientOptions
|
||||
| GoogleVertexProviderSettings
|
||||
) & {
|
||||
export type ClientOptions = (OpenAIClientOptions | AnthropicClientOptions) & {
|
||||
apiKey?: string;
|
||||
provider?: AgentProviderType;
|
||||
auth?: ModelAuth;
|
||||
providerOptions?: ModelProviderOptions;
|
||||
baseURL?: string;
|
||||
/** OpenAI organization ID */
|
||||
organization?: string;
|
||||
|
||||
@ -538,6 +538,26 @@ export class V3 {
|
||||
);
|
||||
}
|
||||
|
||||
private getApiDefaultModelConfig(): ModelConfiguration | undefined {
|
||||
const clientOptions = Object.fromEntries(
|
||||
Object.entries(this.modelClientOptions).filter(
|
||||
([key, value]) => key !== "apiKey" && value !== undefined,
|
||||
),
|
||||
);
|
||||
|
||||
if (Object.keys(clientOptions).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
modelName: this.modelName,
|
||||
...clientOptions,
|
||||
...((this.modelClientOptions as { apiKey?: string }).apiKey
|
||||
? { apiKey: (this.modelClientOptions as { apiKey?: string }).apiKey }
|
||||
: {}),
|
||||
} as ModelConfiguration;
|
||||
}
|
||||
|
||||
private resolveLlmClient(model?: ModelConfiguration): LLMClient {
|
||||
if (!model) {
|
||||
return this.llmClient;
|
||||
@ -1043,6 +1063,7 @@ export class V3 {
|
||||
const { sessionId, available } = await this.apiClient.init({
|
||||
modelName: this.modelName,
|
||||
modelApiKey: this.modelClientOptions.apiKey,
|
||||
defaultModelConfig: this.getApiDefaultModelConfig(),
|
||||
domSettleTimeoutMs: this.domSettleTimeoutMs,
|
||||
verbose: this.verbose,
|
||||
systemPrompt: this.opts.systemPrompt,
|
||||
|
||||
118
packages/core/lib/v3/verifier/evidenceNormalization.ts
Normal file
118
packages/core/lib/v3/verifier/evidenceNormalization.ts
Normal file
@ -0,0 +1,118 @@
|
||||
import type { AgentStepFinishedEvent } from "../types/public/agentEvidenceEvents.js";
|
||||
import type { AgentEvidence } from "./types.js";
|
||||
|
||||
export const REDACTED_INLINE_IMAGE = "[redacted inline image payload]";
|
||||
|
||||
const INLINE_IMAGE_KEYS = new Set(["screenshotBase64"]);
|
||||
|
||||
function shouldRedactBase64Key(key: string, actionName?: string): boolean {
|
||||
return (
|
||||
INLINE_IMAGE_KEYS.has(key) ||
|
||||
(actionName === "screenshot" && key === "base64")
|
||||
);
|
||||
}
|
||||
|
||||
export function collectInlineImagePayloads(
|
||||
value: unknown,
|
||||
actionName?: string,
|
||||
out: string[] = [],
|
||||
): string[] {
|
||||
if (!value || typeof value !== "object") return out;
|
||||
if (Buffer.isBuffer(value)) return out;
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
collectInlineImagePayloads(item, actionName, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
if (shouldRedactBase64Key(key, actionName) && typeof nested === "string") {
|
||||
out.push(nested);
|
||||
continue;
|
||||
}
|
||||
collectInlineImagePayloads(nested, actionName, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function redactInlineImagePayloads(
|
||||
value: unknown,
|
||||
actionName?: string,
|
||||
): unknown {
|
||||
if (!value || typeof value !== "object") return value;
|
||||
if (Buffer.isBuffer(value)) return value;
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => redactInlineImagePayloads(item, actionName));
|
||||
}
|
||||
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, nested] of Object.entries(value)) {
|
||||
out[key] =
|
||||
shouldRedactBase64Key(key, actionName) && typeof nested === "string"
|
||||
? REDACTED_INLINE_IMAGE
|
||||
: redactInlineImagePayloads(nested, actionName);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function mergeAgentEvidence(
|
||||
...parts: Array<AgentEvidence | undefined>
|
||||
): AgentEvidence {
|
||||
return {
|
||||
modalities: parts.flatMap((p) => p?.modalities ?? []),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAgentEvidenceFromStepFinished(
|
||||
event: AgentStepFinishedEvent,
|
||||
): AgentEvidence {
|
||||
const modalities: AgentEvidence["modalities"] = [];
|
||||
if (event.reasoning) {
|
||||
modalities.push({ type: "text", content: event.reasoning });
|
||||
}
|
||||
|
||||
const result = event.toolOutput.result;
|
||||
if (result === undefined || result === null) {
|
||||
return { modalities };
|
||||
}
|
||||
|
||||
if (typeof result === "string") {
|
||||
modalities.push({ type: "text", content: result });
|
||||
} else if (
|
||||
typeof result === "number" ||
|
||||
typeof result === "boolean" ||
|
||||
typeof result === "bigint"
|
||||
) {
|
||||
modalities.push({ type: "text", content: String(result) });
|
||||
} else if (Buffer.isBuffer(result)) {
|
||||
modalities.push({
|
||||
type: "image",
|
||||
bytes: result,
|
||||
mediaType: "image/png",
|
||||
});
|
||||
} else if (typeof result === "object") {
|
||||
for (const imageBase64 of collectInlineImagePayloads(
|
||||
result,
|
||||
event.actionName,
|
||||
)) {
|
||||
try {
|
||||
modalities.push({
|
||||
type: "image",
|
||||
bytes: Buffer.from(imageBase64, "base64"),
|
||||
mediaType: "image/png",
|
||||
});
|
||||
} catch {
|
||||
// Malformed base64; skip the image and keep the JSON modality.
|
||||
}
|
||||
}
|
||||
modalities.push({
|
||||
type: "json",
|
||||
content: redactInlineImagePayloads(result, event.actionName),
|
||||
});
|
||||
}
|
||||
|
||||
return { modalities };
|
||||
}
|
||||
@ -21,8 +21,17 @@ export type {
|
||||
VerifierFinding,
|
||||
VerifierRawSteps,
|
||||
} from "./types.js";
|
||||
export {
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
collectInlineImagePayloads,
|
||||
mergeAgentEvidence,
|
||||
redactInlineImagePayloads,
|
||||
REDACTED_INLINE_IMAGE,
|
||||
} from "./evidenceNormalization.js";
|
||||
export {
|
||||
loadTrajectoryFromDisk,
|
||||
nextResultFilename,
|
||||
normalizeRubric,
|
||||
shouldPersistTrajectory,
|
||||
writeTrajectoryDir,
|
||||
} from "./trajectory.js";
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
AgentEvidenceModality,
|
||||
ProbeEvidence,
|
||||
@ -5,6 +7,7 @@ import type {
|
||||
Trajectory,
|
||||
TrajectoryStep,
|
||||
} from "./types.js";
|
||||
import { redactInlineImagePayloads } from "./evidenceNormalization.js";
|
||||
|
||||
type RawRubricCriterion = {
|
||||
criterion: unknown;
|
||||
@ -18,6 +21,10 @@ type RawRubric = {
|
||||
items?: unknown;
|
||||
};
|
||||
|
||||
type PersistedProbeEvidence = ProbeEvidence & {
|
||||
screenshotPath?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Convert dataset or generated rubric JSON into the public Stagehand shape.
|
||||
* Snake-case dataset fields are accepted here so serialized quirks do not leak
|
||||
@ -92,8 +99,9 @@ function normalizeResultLabel(label?: string): string {
|
||||
*
|
||||
* Reverses the recorder's serialization tweaks:
|
||||
* - `probeEvidence.screenshotPath` → read file into `probeEvidence.screenshot`.
|
||||
* - Image modalities in `agentEvidence.modalities` carry `bytesBase64` on
|
||||
* disk (human-readable JSON) instead of raw Buffer; we decode back.
|
||||
* - Image modalities in `agentEvidence.modalities` carry `imagePath` on
|
||||
* disk instead of raw Buffer; legacy `bytesBase64` fixtures are also
|
||||
* accepted.
|
||||
*
|
||||
* @param dir absolute or cwd-relative path to a `<run-id>/<task-id>/` directory.
|
||||
*/
|
||||
@ -105,6 +113,7 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
|
||||
const trajectoryPath = path.join(trajectoryDir, "trajectory.json");
|
||||
const raw = await fs.readFile(trajectoryPath, "utf8");
|
||||
const parsed = JSON.parse(raw) as Trajectory & {
|
||||
finalObservation?: PersistedProbeEvidence;
|
||||
steps: Array<
|
||||
TrajectoryStep & {
|
||||
agentEvidence: {
|
||||
@ -113,20 +122,24 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
|
||||
| {
|
||||
type: "image";
|
||||
mediaType: string;
|
||||
// On-disk form (recorder writes base64); accept either to
|
||||
// tolerate hand-edited fixtures.
|
||||
// On-disk forms. Current writer externalizes bytes to
|
||||
// imagePath; bytesBase64 is accepted for older fixtures.
|
||||
bytes?: unknown;
|
||||
bytesBase64?: string;
|
||||
imagePath?: string;
|
||||
}
|
||||
| { type: "json"; content: unknown }
|
||||
>;
|
||||
};
|
||||
probeEvidence: ProbeEvidence;
|
||||
probeEvidence: PersistedProbeEvidence;
|
||||
}
|
||||
>;
|
||||
};
|
||||
|
||||
const resolveWithinTrajectoryDir = (candidate: string): string => {
|
||||
const resolveWithinTrajectoryDir = (
|
||||
candidate: string,
|
||||
fieldName = "screenshotPath",
|
||||
): string => {
|
||||
const resolved = path.resolve(trajectoryDir, candidate);
|
||||
const relative = path.relative(trajectoryDir, resolved);
|
||||
const outside =
|
||||
@ -136,16 +149,16 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
|
||||
|
||||
if (outside) {
|
||||
throw new Error(
|
||||
`Trajectory screenshotPath escapes trajectory directory: ${candidate}`,
|
||||
`Trajectory ${fieldName} escapes trajectory directory: ${candidate}`,
|
||||
);
|
||||
}
|
||||
|
||||
return resolved;
|
||||
};
|
||||
|
||||
for (const step of parsed.steps) {
|
||||
// Rehydrate tier-2 probe screenshot from its on-disk file reference.
|
||||
const probe = step.probeEvidence;
|
||||
const hydrateProbeScreenshot = async (
|
||||
probe: PersistedProbeEvidence | undefined,
|
||||
): Promise<void> => {
|
||||
if (probe?.screenshotPath && !probe.screenshot) {
|
||||
const resolved = resolveWithinTrajectoryDir(probe.screenshotPath);
|
||||
try {
|
||||
@ -155,25 +168,55 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
|
||||
// evidence_insufficient path will handle it.
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Decode image modalities from base64 back to Buffer.
|
||||
for (const step of parsed.steps) {
|
||||
// Rehydrate tier-2 probe screenshot from its on-disk file reference.
|
||||
await hydrateProbeScreenshot(step.probeEvidence);
|
||||
|
||||
// Decode image modalities from disk references back to Buffer.
|
||||
if (step.agentEvidence?.modalities) {
|
||||
step.agentEvidence.modalities = step.agentEvidence.modalities.map((m) => {
|
||||
// The on-disk shape carries bytesBase64 instead of bytes, so we look
|
||||
// through `unknown` here rather than rely on the typed union.
|
||||
const raw = m as unknown as { bytesBase64?: string };
|
||||
const modalities: AgentEvidenceModality[] = [];
|
||||
for (const m of step.agentEvidence.modalities) {
|
||||
// The on-disk shape carries imagePath/bytesBase64 instead of bytes,
|
||||
// so we look through `unknown` rather than rely on the typed union.
|
||||
const raw = m as unknown as {
|
||||
bytesBase64?: string;
|
||||
imagePath?: string;
|
||||
};
|
||||
if (m.type === "image" && typeof raw.bytesBase64 === "string") {
|
||||
return {
|
||||
modalities.push({
|
||||
type: "image" as const,
|
||||
bytes: Buffer.from(raw.bytesBase64, "base64"),
|
||||
mediaType: m.mediaType,
|
||||
};
|
||||
});
|
||||
continue;
|
||||
}
|
||||
return m as AgentEvidenceModality;
|
||||
});
|
||||
if (m.type === "image" && typeof raw.imagePath === "string") {
|
||||
const resolved = resolveWithinTrajectoryDir(
|
||||
raw.imagePath,
|
||||
"imagePath",
|
||||
);
|
||||
try {
|
||||
modalities.push({
|
||||
type: "image" as const,
|
||||
bytes: await fs.readFile(resolved),
|
||||
mediaType: m.mediaType,
|
||||
});
|
||||
} catch {
|
||||
// Missing agent image file: omit that image modality. The
|
||||
// verifier's evidence_insufficient path will handle missing bytes.
|
||||
}
|
||||
continue;
|
||||
}
|
||||
modalities.push(m as AgentEvidenceModality);
|
||||
}
|
||||
step.agentEvidence.modalities = modalities;
|
||||
}
|
||||
}
|
||||
|
||||
await hydrateProbeScreenshot(parsed.finalObservation);
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
@ -187,3 +230,162 @@ export async function loadTrajectoryFromDisk(dir: string): Promise<Trajectory> {
|
||||
export function nextResultFilename(label?: string): string {
|
||||
return `result_${normalizeResultLabel(label)}.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default persistence policy: explicit override, then env, then "on unless CI".
|
||||
*/
|
||||
export function shouldPersistTrajectory(
|
||||
override: boolean | undefined,
|
||||
): boolean {
|
||||
if (override !== undefined) return override;
|
||||
const env = process.env.VERIFIER_PERSIST_TRAJECTORIES?.toLowerCase();
|
||||
if (env === "1" || env === "true") return true;
|
||||
if (env === "0" || env === "false") return false;
|
||||
return !process.env.CI;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the on-disk trajectory layout under `dir`:
|
||||
*
|
||||
* <dir>/
|
||||
* ├── task_data.json
|
||||
* ├── trajectory.json (screenshots referenced by path)
|
||||
* ├── screenshots/
|
||||
* │ ├── probe/<N>.png
|
||||
* │ └── agent/<N>[_M].png
|
||||
* ├── scores/ (empty; populated separately)
|
||||
* └── core.log
|
||||
*
|
||||
* Image bytes are externalized to PNG files; the in-memory Trajectory is left
|
||||
* untouched so callers can keep using it after persistence.
|
||||
*/
|
||||
export async function writeTrajectoryDir(
|
||||
dir: string,
|
||||
trajectory: Trajectory,
|
||||
): Promise<void> {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.mkdir(path.join(dir, "screenshots", "probe"), { recursive: true });
|
||||
await fs.mkdir(path.join(dir, "screenshots", "agent"), { recursive: true });
|
||||
|
||||
const serializableSteps: unknown[] = [];
|
||||
// A single post-turn probe is fanned across every step of a multi-tool turn,
|
||||
// and a single agent screenshot is shared across every action a CUA provider
|
||||
// chose from it, so the same Buffer is shared by reference. Dedupe by
|
||||
// identity: write the PNG once and point every sharing step at the same file.
|
||||
const probePathByBuffer = new Map<Buffer, string>();
|
||||
const agentPathByBuffer = new Map<Buffer, string>();
|
||||
for (const [i, step] of trajectory.steps.entries()) {
|
||||
const probe: ProbeEvidence = { ...step.probeEvidence };
|
||||
if (probe.screenshot) {
|
||||
let relPath = probePathByBuffer.get(probe.screenshot);
|
||||
if (!relPath) {
|
||||
relPath = `screenshots/probe/${i + 1}.png`;
|
||||
await fs.writeFile(path.join(dir, relPath), probe.screenshot);
|
||||
probePathByBuffer.set(probe.screenshot, relPath);
|
||||
}
|
||||
probe.screenshotPath = relPath;
|
||||
delete probe.screenshot;
|
||||
}
|
||||
|
||||
const imageModalities = step.agentEvidence.modalities.filter(
|
||||
(m) => m.type === "image",
|
||||
);
|
||||
const multipleImages = imageModalities.length > 1;
|
||||
let imageSeq = 0;
|
||||
const modalities: unknown[] = [];
|
||||
for (const m of step.agentEvidence.modalities) {
|
||||
if (m.type !== "image") {
|
||||
modalities.push(
|
||||
m.type === "json"
|
||||
? {
|
||||
...m,
|
||||
content: redactInlineImagePayloads(m.content, step.actionName),
|
||||
}
|
||||
: m,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let relPath = agentPathByBuffer.get(m.bytes);
|
||||
if (!relPath) {
|
||||
const suffix = multipleImages ? `_${imageSeq}` : "";
|
||||
relPath = `screenshots/agent/${i + 1}${suffix}.png`;
|
||||
await fs.writeFile(path.join(dir, relPath), m.bytes);
|
||||
agentPathByBuffer.set(m.bytes, relPath);
|
||||
}
|
||||
modalities.push({
|
||||
type: "image",
|
||||
imagePath: relPath,
|
||||
mediaType: m.mediaType,
|
||||
});
|
||||
imageSeq += 1;
|
||||
}
|
||||
serializableSteps.push({
|
||||
...step,
|
||||
probeEvidence: probe,
|
||||
agentEvidence: { modalities },
|
||||
toolOutput: {
|
||||
...step.toolOutput,
|
||||
result: redactInlineImagePayloads(
|
||||
step.toolOutput.result,
|
||||
step.actionName,
|
||||
),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const finalObservation: ProbeEvidence | undefined =
|
||||
trajectory.finalObservation === undefined
|
||||
? undefined
|
||||
: { ...trajectory.finalObservation };
|
||||
if (finalObservation?.screenshot) {
|
||||
const relPath = "screenshots/probe/final.png";
|
||||
await fs.writeFile(path.join(dir, relPath), finalObservation.screenshot);
|
||||
finalObservation.screenshotPath = relPath;
|
||||
delete finalObservation.screenshot;
|
||||
}
|
||||
|
||||
// Image modalities carry imagePath instead of raw bytes on disk; cast
|
||||
// through unknown rather than widen Trajectory's type contract.
|
||||
const serialized = {
|
||||
...trajectory,
|
||||
steps: serializableSteps,
|
||||
...(finalObservation ? { finalObservation } : {}),
|
||||
} as unknown;
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(dir, "trajectory.json"),
|
||||
JSON.stringify(serialized, null, 2),
|
||||
);
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(dir, "task_data.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
task: trajectory.task,
|
||||
status: trajectory.status,
|
||||
finalAnswer: trajectory.finalAnswer ?? null,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
await fs.mkdir(path.join(dir, "scores"), { recursive: true });
|
||||
await fs.writeFile(path.join(dir, "core.log"), coreLog(trajectory));
|
||||
}
|
||||
|
||||
function coreLog(trajectory: Trajectory): string {
|
||||
return (
|
||||
trajectory.steps
|
||||
.map((step, i) =>
|
||||
JSON.stringify({
|
||||
step: i,
|
||||
action: step.actionName,
|
||||
url: step.probeEvidence.url ?? null,
|
||||
ok: step.toolOutput.ok,
|
||||
reasoning: step.reasoning || undefined,
|
||||
}),
|
||||
)
|
||||
.join("\n") + "\n"
|
||||
);
|
||||
}
|
||||
|
||||
@ -89,8 +89,6 @@ export interface ProbeEvidence {
|
||||
screenshot?: Buffer;
|
||||
/** Reference to the persisted screenshot file under the trajectory dir. */
|
||||
screenshotPath?: string;
|
||||
/** Viewport scroll context. Lets the verifier reason about whether the agent saw the full page. */
|
||||
scroll?: { top: number; pageHeight: number };
|
||||
/** Accessibility tree snapshot. */
|
||||
ariaTree?: string;
|
||||
/** Verifier-requested probes, keyed by criterion id. */
|
||||
@ -111,7 +109,6 @@ export interface ToolOutput {
|
||||
|
||||
/** One step in a trajectory: action + reasoning + evidence + outcome. */
|
||||
export interface TrajectoryStep {
|
||||
index: number;
|
||||
actionName: string;
|
||||
actionArgs: Record<string, unknown>;
|
||||
/** From AgentAction.reasoning. May be empty for tools that don't surface reasoning. */
|
||||
@ -119,10 +116,6 @@ export interface TrajectoryStep {
|
||||
agentEvidence: AgentEvidence;
|
||||
probeEvidence: ProbeEvidence;
|
||||
toolOutput: ToolOutput;
|
||||
/** ISO 8601 timestamp when the step's tool execution started. */
|
||||
startedAt: string;
|
||||
/** ISO 8601 timestamp when the step's tool execution finished. */
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
/** Terminal status of the agent run. */
|
||||
@ -136,19 +129,19 @@ export type TrajectoryStatus = "complete" | "aborted" | "stalled" | "error";
|
||||
* .trajectories/<run-id>/<task-id>/
|
||||
* ├── task_data.json — TaskSpec + result metadata
|
||||
* ├── trajectory.json — this object, with screenshotPath instead of bytes
|
||||
* ├── screenshot_1.png — probeEvidence.screenshot for step 1, etc.
|
||||
* ├── screenshots/ — step probe/agent images plus final observation
|
||||
* ├── scores/
|
||||
* │ └── result.json — Result from V3Evaluator.verify()
|
||||
* ├── core.log — captured action log
|
||||
* └── times.json — step timing + token usage
|
||||
* └── core.log — captured action log
|
||||
*/
|
||||
export interface Trajectory {
|
||||
task: TaskSpec;
|
||||
steps: TrajectoryStep[];
|
||||
finalAnswer?: string;
|
||||
/** Terminal page observation captured after the agent finishes. */
|
||||
finalObservation?: ProbeEvidence;
|
||||
status: TrajectoryStatus;
|
||||
usage: TrajectoryUsage;
|
||||
timing: { startedAt: string; endedAt: string };
|
||||
}
|
||||
|
||||
/** Score for a single rubric criterion after evidence analysis + rescoring. */
|
||||
|
||||
@ -227,19 +227,25 @@ function collectLegacyScreenshots(trajectory: Trajectory): Buffer[] {
|
||||
}
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(trajectory.finalObservation?.screenshot)) {
|
||||
screenshots.push(trajectory.finalObservation.screenshot);
|
||||
}
|
||||
|
||||
return screenshots;
|
||||
}
|
||||
|
||||
function renderLegacyAgentReasoning(
|
||||
trajectory: Trajectory,
|
||||
): string | undefined {
|
||||
const stepLines = (trajectory.steps ?? []).map((step) => {
|
||||
const stepLines = (trajectory.steps ?? []).map((step, i) => {
|
||||
const status = step.toolOutput?.ok === false ? "Tool status: failed" : "";
|
||||
const output = step.toolOutput?.error
|
||||
? `Tool error: ${step.toolOutput.error}`
|
||||
: `Tool output: ${stringifyForPrompt(step.toolOutput?.result)}`;
|
||||
return [
|
||||
`Step ${step.index}: ${step.actionName}`,
|
||||
`Step ${i}: ${step.actionName}`,
|
||||
step.reasoning ? `Reasoning: ${step.reasoning}` : undefined,
|
||||
status || undefined,
|
||||
output,
|
||||
]
|
||||
.filter(Boolean)
|
||||
|
||||
@ -90,7 +90,7 @@
|
||||
"pino": "^9.6.0",
|
||||
"pino-pretty": "^13.0.0",
|
||||
"uuid": "^11.1.1",
|
||||
"ws": "^8.18.0",
|
||||
"ws": "^8.21.0",
|
||||
"zod-to-json-schema": "^3.25.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
|
||||
@ -60,6 +60,7 @@ class FakeCuaClient {
|
||||
public contextNotes: string[] = [];
|
||||
public preStepHook?: () => Promise<void>;
|
||||
public actionHandler?: (action: Record<string, unknown>) => Promise<void>;
|
||||
public screenshotProvider?: () => Promise<string>;
|
||||
public executeImpl = vi.fn(async (options: unknown) => {
|
||||
void options;
|
||||
return {
|
||||
@ -72,7 +73,9 @@ class FakeCuaClient {
|
||||
public captureScreenshot = vi.fn(async () => null);
|
||||
public setViewport = vi.fn();
|
||||
public setCurrentUrl = vi.fn();
|
||||
public setScreenshotProvider = vi.fn();
|
||||
public setScreenshotProvider = vi.fn((provider: () => Promise<string>) => {
|
||||
this.screenshotProvider = provider;
|
||||
});
|
||||
public setSafetyConfirmationHandler = vi.fn();
|
||||
|
||||
setActionHandler(
|
||||
@ -247,7 +250,6 @@ describe("agent captcha hooks", () => {
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
bus: { emit: vi.fn() },
|
||||
isCaptchaAutoSolveEnabled: true,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
@ -316,7 +318,6 @@ describe("agent captcha hooks", () => {
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
bus: { emit: vi.fn() },
|
||||
isCaptchaAutoSolveEnabled: true,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
@ -392,7 +393,6 @@ describe("agent captcha hooks", () => {
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
bus: { emit: vi.fn() },
|
||||
isCaptchaAutoSolveEnabled: true,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
@ -474,7 +474,6 @@ describe("v3 cua handler screenshot behavior", () => {
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
bus: { emit: vi.fn() },
|
||||
isCaptchaAutoSolveEnabled: false,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
@ -504,4 +503,113 @@ describe("v3 cua handler screenshot behavior", () => {
|
||||
// the CUA client takes a single screenshot after all actions itself.
|
||||
expect(screenshotSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still returns provider screenshots when screenshot evidence callbacks fail", async () => {
|
||||
const screenshotBase64 = Buffer.from("fake-image").toString("base64");
|
||||
const onEvidence = vi.fn(async (event: { type: string }) => {
|
||||
if (event.type === "screenshot") {
|
||||
throw new Error("recorder failed");
|
||||
}
|
||||
});
|
||||
|
||||
fakeCuaClient.executeImpl = vi.fn(async () => {
|
||||
await expect(fakeCuaClient.screenshotProvider?.()).resolves.toBe(
|
||||
screenshotBase64,
|
||||
);
|
||||
return {
|
||||
success: true,
|
||||
message: "ok",
|
||||
actions: [],
|
||||
completed: true,
|
||||
};
|
||||
});
|
||||
|
||||
const handler = new V3CuaAgentHandler(
|
||||
{
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
isCaptchaAutoSolveEnabled: false,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
isAgentReplayActive: () => false,
|
||||
updateMetrics: vi.fn(),
|
||||
} as never,
|
||||
logger,
|
||||
{
|
||||
modelName: "openai/gpt-5.4",
|
||||
clientOptions: { waitBetweenActions: 1 },
|
||||
} as never,
|
||||
);
|
||||
|
||||
await handler.execute({
|
||||
instruction: "describe the page",
|
||||
highlightCursor: false,
|
||||
callbacks: { onEvidence },
|
||||
});
|
||||
|
||||
expect(onEvidence).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ type: "screenshot" }),
|
||||
);
|
||||
expect(
|
||||
logs.some((line) =>
|
||||
line.message.includes("onEvidence callback failed for screenshot"),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("records a failed action as step_finished {ok:false} and rethrows the original error", async () => {
|
||||
const events: Array<{ type: string; [k: string]: unknown }> = [];
|
||||
const onEvidence = vi.fn(async (event: { type: string }) => {
|
||||
events.push(event as { type: string });
|
||||
});
|
||||
|
||||
fakeCuaClient.executeImpl = vi.fn(async () => {
|
||||
await fakeCuaClient.actionHandler?.({
|
||||
type: "click",
|
||||
button: "left",
|
||||
x: 5,
|
||||
y: 9,
|
||||
});
|
||||
return { success: true, message: "ok", actions: [], completed: true };
|
||||
});
|
||||
|
||||
const handler = new V3CuaAgentHandler(
|
||||
{
|
||||
context: {
|
||||
awaitActivePage: async () => page,
|
||||
},
|
||||
isCaptchaAutoSolveEnabled: false,
|
||||
isAdvancedStealth: false,
|
||||
configuredViewport: { width: 1288, height: 711 },
|
||||
isAgentReplayActive: () => false,
|
||||
updateMetrics: vi.fn(),
|
||||
} as never,
|
||||
logger,
|
||||
{
|
||||
modelName: "openai/gpt-5.4",
|
||||
clientOptions: { waitBetweenActions: 1 },
|
||||
} as never,
|
||||
);
|
||||
vi.spyOn(
|
||||
handler as unknown as {
|
||||
executeAction: (action: Record<string, unknown>) => Promise<unknown>;
|
||||
},
|
||||
"executeAction",
|
||||
).mockRejectedValue(new Error("click failed"));
|
||||
|
||||
await expect(
|
||||
handler.execute({
|
||||
instruction: "click the thing",
|
||||
highlightCursor: false,
|
||||
callbacks: { onEvidence },
|
||||
}),
|
||||
).rejects.toThrow("click failed");
|
||||
|
||||
const stepFinished = events.find((e) => e.type === "step_finished");
|
||||
expect(stepFinished).toMatchObject({
|
||||
actionName: "click",
|
||||
toolOutput: { ok: false, error: "click failed" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -36,7 +36,30 @@ describe("AnthropicCUAClient adaptive thinking", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Claude 4.6 models (adaptive thinking)", () => {
|
||||
describe("Claude 4.6+ models (adaptive thinking)", () => {
|
||||
it("should use thinking.type: 'adaptive' for claude-opus-4-8 when thinkingEffort is set", async () => {
|
||||
const client = new AnthropicCUAClient(
|
||||
"anthropic",
|
||||
"claude-opus-4-8",
|
||||
undefined,
|
||||
{
|
||||
apiKey: "test-key",
|
||||
thinkingEffort: "high",
|
||||
},
|
||||
);
|
||||
client.setViewport(1280, 720);
|
||||
|
||||
await client.getAction([{ role: "user", content: "test" }]);
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
thinking: { type: "adaptive" },
|
||||
output_config: { effort: "high" },
|
||||
temperature: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("should use thinking.type: 'adaptive' for claude-opus-4-6 when thinkingEffort is set", async () => {
|
||||
const client = new AnthropicCUAClient(
|
||||
"anthropic",
|
||||
|
||||
342
packages/core/tests/unit/api-client-default-model-config.test.ts
Normal file
342
packages/core/tests/unit/api-client-default-model-config.test.ts
Normal file
@ -0,0 +1,342 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { StagehandAPIClient } from "../../lib/v3/api.js";
|
||||
import type { ModelConfiguration } from "../../lib/v3/types/public/model.js";
|
||||
|
||||
const vertexModel = {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
} as unknown as ModelConfiguration;
|
||||
|
||||
function createClientWithExecuteMock() {
|
||||
const client = new StagehandAPIClient({
|
||||
apiKey: "bb-test",
|
||||
logger: vi.fn(),
|
||||
});
|
||||
const executeMock = vi.fn().mockResolvedValue({});
|
||||
|
||||
(
|
||||
client as unknown as {
|
||||
execute: typeof executeMock;
|
||||
}
|
||||
).execute = executeMock;
|
||||
|
||||
return { client, executeMock };
|
||||
}
|
||||
|
||||
describe("StagehandAPIClient default model config", () => {
|
||||
let originalFetch: typeof globalThis.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
data: { sessionId: "sess-default-model", available: true },
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("sends constructor Vertex model config on navigate bootstrap, act, observe, extract, and agent execute requests", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
defaultModelConfig: vertexModel,
|
||||
});
|
||||
|
||||
await client.goto("https://example.com");
|
||||
await client.act({ input: "click the login button" });
|
||||
await client.observe({ instruction: "find the login button" });
|
||||
await client.extract({ instruction: "extract the headline" });
|
||||
await client.agentExecute({ mode: "dom" }, "click the login button");
|
||||
|
||||
expect(executeMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
method: "navigate",
|
||||
args: expect.objectContaining({
|
||||
options: {
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(executeMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
method: "act",
|
||||
args: expect.objectContaining({
|
||||
options: {
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(executeMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
method: "observe",
|
||||
args: expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(executeMock).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
expect.objectContaining({
|
||||
method: "extract",
|
||||
args: expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(executeMock).toHaveBeenNthCalledWith(
|
||||
5,
|
||||
expect.objectContaining({
|
||||
method: "agentExecute",
|
||||
args: expect.objectContaining({
|
||||
agentConfig: expect.objectContaining({
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers a per-call model over the constructor default model config", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
defaultModelConfig: vertexModel,
|
||||
});
|
||||
|
||||
await client.act({
|
||||
input: "click the login button",
|
||||
options: {
|
||||
model: {
|
||||
modelName: "openai/gpt-4.1-mini",
|
||||
apiKey: "sk-per-call",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(executeMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
args: expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
model: {
|
||||
modelName: "openai/gpt-4.1-mini",
|
||||
apiKey: "sk-per-call",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("inherits constructor Vertex auth for a same-provider string model override", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
defaultModelConfig: vertexModel,
|
||||
});
|
||||
|
||||
await client.observe({
|
||||
instruction: "find the login button",
|
||||
options: {
|
||||
model: "vertex/gemini-2.0-flash",
|
||||
},
|
||||
});
|
||||
|
||||
expect(executeMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
method: "observe",
|
||||
args: expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.0-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not treat model as a public navigate option", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "openai/gpt-4.1-mini",
|
||||
modelApiKey: "sk-header-only",
|
||||
});
|
||||
|
||||
await client.goto("https://example.com", {
|
||||
model: {
|
||||
modelName: "openai/gpt-4o",
|
||||
apiKey: "sk-per-call",
|
||||
},
|
||||
} as unknown as Parameters<StagehandAPIClient["goto"]>[1]);
|
||||
|
||||
expect(executeMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
method: "navigate",
|
||||
args: expect.objectContaining({
|
||||
options: undefined,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps constructor bootstrap config when a caller passes an unsupported navigate model", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
defaultModelConfig: vertexModel,
|
||||
});
|
||||
|
||||
await client.goto("https://example.com", {
|
||||
model: {
|
||||
modelName: "openai/gpt-4o",
|
||||
apiKey: "sk-per-call",
|
||||
},
|
||||
} as unknown as Parameters<StagehandAPIClient["goto"]>[1]);
|
||||
|
||||
expect(executeMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
method: "navigate",
|
||||
args: expect.objectContaining({
|
||||
options: {
|
||||
model: expect.objectContaining({
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: expect.objectContaining({
|
||||
type: "googleServiceAccount",
|
||||
}),
|
||||
providerOptions: {
|
||||
vertex: expect.objectContaining({
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
}),
|
||||
},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not add a body model config when the constructor only provides a model API key", async () => {
|
||||
const { client, executeMock } = createClientWithExecuteMock();
|
||||
|
||||
await client.init({
|
||||
modelName: "openai/gpt-4.1-mini",
|
||||
modelApiKey: "sk-header-only",
|
||||
});
|
||||
|
||||
await client.act({ input: "click the login button" });
|
||||
|
||||
expect(executeMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
args: expect.objectContaining({
|
||||
options: undefined,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@ -73,9 +73,8 @@ describe("API model config schemas", () => {
|
||||
const vertexModel = {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
googleAuthOptions: {
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
type: "service_account",
|
||||
project_id: "test-gcp-project",
|
||||
@ -89,6 +88,12 @@ describe("API model config schemas", () => {
|
||||
projectId: "test-gcp-project",
|
||||
universeDomain: "googleapis.com",
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it("preserves Vertex auth params for act requests", () => {
|
||||
@ -111,15 +116,20 @@ describe("API model config schemas", () => {
|
||||
model: {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
googleAuthOptions: {
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@ -131,7 +141,11 @@ describe("API model config schemas", () => {
|
||||
if (typeof parsedModel !== "object" || parsedModel === null) {
|
||||
throw new Error("Expected object model config");
|
||||
}
|
||||
expect(parsedModel.googleAuthOptions).toEqual({
|
||||
if (!("auth" in parsedModel)) {
|
||||
throw new Error("Expected Vertex auth config");
|
||||
}
|
||||
expect(parsedModel.auth).toEqual({
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
@ -140,6 +154,67 @@ describe("API model config schemas", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("requires explicit Vertex provider configs to include service account auth and provider options", () => {
|
||||
for (const model of [
|
||||
{
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
},
|
||||
]) {
|
||||
const result = Api.ActRequestSchema.safeParse({
|
||||
input: "click the search button",
|
||||
options: { model },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("requires provider vertex when passing Vertex auth and provider options", () => {
|
||||
const result = Api.ActRequestSchema.safeParse({
|
||||
input: "click the search button",
|
||||
options: {
|
||||
model: {
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves Vertex auth params for agent model configs", () => {
|
||||
const result = Api.AgentExecuteRequestSchema.safeParse({
|
||||
agentConfig: {
|
||||
@ -158,6 +233,20 @@ describe("API model config schemas", () => {
|
||||
});
|
||||
|
||||
it("rejects Vertex key file paths and external credential sources", () => {
|
||||
const topLevelLegacyShape = Api.ActRequestSchema.safeParse({
|
||||
input: "click the search button",
|
||||
options: {
|
||||
model: {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
googleAuthOptions: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(topLevelLegacyShape.success).toBe(false);
|
||||
|
||||
for (const blockedAuthOptions of [
|
||||
{ keyFilename: "/etc/passwd" },
|
||||
{ keyFile: "/etc/passwd" },
|
||||
@ -169,7 +258,7 @@ describe("API model config schemas", () => {
|
||||
model: {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
googleAuthOptions: blockedAuthOptions,
|
||||
auth: blockedAuthOptions,
|
||||
},
|
||||
},
|
||||
});
|
||||
@ -183,7 +272,8 @@ describe("API model config schemas", () => {
|
||||
model: {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
googleAuthOptions: {
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
type: "external_account",
|
||||
credential_source: {
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getAISDKLanguageModel } from "../../lib/v3/llm/LLMProvider.js";
|
||||
import {
|
||||
getAISDKLanguageModel,
|
||||
LLMProvider,
|
||||
toAISDKClientOptions,
|
||||
} from "../../lib/v3/llm/LLMProvider.js";
|
||||
|
||||
describe("getAISDKLanguageModel", () => {
|
||||
describe("ollama provider", () => {
|
||||
@ -68,3 +72,67 @@ describe("getAISDKLanguageModel", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLMProvider", () => {
|
||||
it("allows Vertex models without experimental mode", () => {
|
||||
const provider = new LLMProvider(() => {});
|
||||
|
||||
expect(() =>
|
||||
provider.getClient(
|
||||
"vertex/gemini-2.5-flash" as never,
|
||||
{
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "stagehand@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
{ experimental: false, disableAPI: false },
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("adapts canonical Vertex auth into AI SDK googleAuthOptions", () => {
|
||||
expect(
|
||||
toAISDKClientOptions("vertex", {
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "stagehand@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n",
|
||||
},
|
||||
projectId: "test-project",
|
||||
universeDomain: "googleapis.com",
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
project: "test-project",
|
||||
location: "us-central1",
|
||||
googleAuthOptions: {
|
||||
credentials: {
|
||||
client_email: "stagehand@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n",
|
||||
},
|
||||
projectId: "test-project",
|
||||
universeDomain: "googleapis.com",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -32,6 +32,8 @@ const publicApiShape = {
|
||||
V3: Stagehand.V3,
|
||||
V3Evaluator: Stagehand.V3Evaluator,
|
||||
V3FunctionName: Stagehand.V3FunctionName,
|
||||
buildAgentEvidenceFromStepFinished:
|
||||
Stagehand.buildAgentEvidenceFromStepFinished,
|
||||
connectToMCPServer: Stagehand.connectToMCPServer,
|
||||
default: StagehandDefaultExport,
|
||||
defaultExtractSchema: Stagehand.defaultExtractSchema,
|
||||
@ -44,18 +46,22 @@ const publicApiShape = {
|
||||
jsonSchemaToZod: Stagehand.jsonSchemaToZod,
|
||||
loadApiKeyFromEnv: Stagehand.loadApiKeyFromEnv,
|
||||
loadTrajectoryFromDisk: Stagehand.loadTrajectoryFromDisk,
|
||||
mergeAgentEvidence: Stagehand.mergeAgentEvidence,
|
||||
localBrowserLaunchOptionsSchema: Stagehand.localBrowserLaunchOptionsSchema,
|
||||
modelToAgentProviderMap: Stagehand.modelToAgentProviderMap,
|
||||
nextResultFilename: Stagehand.nextResultFilename,
|
||||
normalizeRubric: Stagehand.normalizeRubric,
|
||||
pageTextSchema: Stagehand.pageTextSchema,
|
||||
providerEnvVarMap: Stagehand.providerEnvVarMap,
|
||||
redactInlineImagePayloads: Stagehand.redactInlineImagePayloads,
|
||||
shouldPersistTrajectory: Stagehand.shouldPersistTrajectory,
|
||||
toGeminiSchema: Stagehand.toGeminiSchema,
|
||||
toJsonSchema: Stagehand.toJsonSchema,
|
||||
tool: Stagehand.tool,
|
||||
transformSchema: Stagehand.transformSchema,
|
||||
trimTrailingTextNode: Stagehand.trimTrailingTextNode,
|
||||
validateZodSchema: Stagehand.validateZodSchema,
|
||||
writeTrajectoryDir: Stagehand.writeTrajectoryDir,
|
||||
...publicErrorTypes,
|
||||
} as const;
|
||||
|
||||
|
||||
@ -3,17 +3,30 @@ import * as Stagehand from "@browserbasehq/stagehand";
|
||||
|
||||
describe("LLM and Agents public API types", () => {
|
||||
describe("ModelConfiguration", () => {
|
||||
it("accepts Vertex headers in model config", () => {
|
||||
const googleConfig = {
|
||||
modelName: "google/gemini-3-flash-preview",
|
||||
project: "test-project",
|
||||
location: "global",
|
||||
it("accepts Vertex provider options in model config", () => {
|
||||
const vertexConfig = {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-3-flash-preview",
|
||||
headers: {
|
||||
"X-Goog-Priority": "high",
|
||||
},
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-project",
|
||||
location: "global",
|
||||
},
|
||||
},
|
||||
} satisfies Stagehand.ModelConfiguration;
|
||||
|
||||
void googleConfig;
|
||||
void vertexConfig;
|
||||
});
|
||||
});
|
||||
|
||||
@ -44,6 +57,7 @@ describe("LLM and Agents public API types", () => {
|
||||
"openai/gpt-5.5",
|
||||
"anthropic/claude-opus-4-5-20251101",
|
||||
"anthropic/claude-opus-4-6",
|
||||
"anthropic/claude-opus-4-8",
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"anthropic/claude-haiku-4-5",
|
||||
"anthropic/claude-haiku-4-5-20251001",
|
||||
@ -61,6 +75,12 @@ describe("LLM and Agents public API types", () => {
|
||||
>();
|
||||
void expectedModels; // Mark as used to satisfy ESLint
|
||||
});
|
||||
|
||||
it("includes Claude Opus 4.8 at runtime", () => {
|
||||
expect(Stagehand.AVAILABLE_CUA_MODELS).toContain(
|
||||
"anthropic/claude-opus-4-8",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AgentProvider", () => {
|
||||
@ -274,6 +294,12 @@ describe("LLM and Agents public API types", () => {
|
||||
typeof Stagehand.modelToAgentProviderMap
|
||||
>().toExtend<ExpectedModelToAgentProviderMap>();
|
||||
});
|
||||
|
||||
it("routes Claude Opus 4.8 to Anthropic", () => {
|
||||
expect(Stagehand.modelToAgentProviderMap["claude-opus-4-8"]).toBe(
|
||||
"anthropic",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Response", () => {
|
||||
|
||||
47
packages/core/tests/unit/tool-output-evidence.test.ts
Normal file
47
packages/core/tests/unit/tool-output-evidence.test.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { inferToolOutput } from "../../lib/v3/agent/utils/toolOutputEvidence.js";
|
||||
|
||||
describe("inferToolOutput", () => {
|
||||
it.each<[string, unknown, boolean, string | undefined]>([
|
||||
[
|
||||
"preserves raw results while normalizing top-level failure status",
|
||||
{ success: false },
|
||||
false,
|
||||
undefined,
|
||||
],
|
||||
[
|
||||
"normalizes one-level AI SDK output wrappers",
|
||||
{
|
||||
toolCallId: "call-1",
|
||||
output: { success: false, error: { message: "not found" } },
|
||||
},
|
||||
false,
|
||||
'{"message":"not found"}',
|
||||
],
|
||||
[
|
||||
"handles isError and non-string errors",
|
||||
{ isError: true, error: new Error("bad input") },
|
||||
false,
|
||||
"bad input",
|
||||
],
|
||||
[
|
||||
"normalizes non-json error values",
|
||||
{ error: Symbol("bad input") },
|
||||
false,
|
||||
"Symbol(bad input)",
|
||||
],
|
||||
[
|
||||
"does not recursively treat page data as tool status",
|
||||
{ data: { success: false, error: "page field" } },
|
||||
true,
|
||||
undefined,
|
||||
],
|
||||
])("%s", (_, result, ok, error) => {
|
||||
expect(inferToolOutput(result)).toEqual({
|
||||
ok,
|
||||
result,
|
||||
error,
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -82,6 +82,39 @@ describe("V3Evaluator verifier facade", () => {
|
||||
expect(result.perCriterion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("passes final observation screenshots to the legacy verifier adapter", async () => {
|
||||
const taskSpec: TaskSpec = {
|
||||
id: "final-observation",
|
||||
instruction: "Complete the task",
|
||||
};
|
||||
const finalScreenshot = Buffer.from("final screenshot");
|
||||
const trajectory = {
|
||||
...makeTrajectory(taskSpec),
|
||||
finalObservation: {
|
||||
url: "https://example.com/done",
|
||||
screenshot: finalScreenshot,
|
||||
},
|
||||
};
|
||||
const ask = vi.fn().mockResolvedValue({
|
||||
evaluation: "YES",
|
||||
reasoning: "The final screenshot shows completion.",
|
||||
});
|
||||
const evaluator = new V3Evaluator({} as V3, {
|
||||
backend: "legacy",
|
||||
});
|
||||
Object.defineProperty(evaluator, "legacyEvaluator", {
|
||||
value: { ask },
|
||||
});
|
||||
|
||||
await evaluator.verify(trajectory);
|
||||
|
||||
expect(ask).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
screenshot: [finalScreenshot],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps legacy tool output detail until the overall reasoning budget is reached", async () => {
|
||||
const taskSpec: TaskSpec = {
|
||||
id: "reasoning-budget",
|
||||
@ -164,10 +197,6 @@ function makeEmptyTrajectory(taskSpec: TaskSpec): Trajectory {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
},
|
||||
timing: {
|
||||
startedAt: new Date(0).toISOString(),
|
||||
endedAt: new Date(0).toISOString(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ -183,7 +212,6 @@ function makeTrajectory(
|
||||
...makeEmptyTrajectory(taskSpec),
|
||||
steps: [
|
||||
{
|
||||
index: 0,
|
||||
actionName: "act",
|
||||
actionArgs: {},
|
||||
reasoning: "I completed the task.",
|
||||
@ -195,8 +223,6 @@ function makeTrajectory(
|
||||
ok: true,
|
||||
result: options.toolResult ?? "done",
|
||||
},
|
||||
startedAt: new Date(0).toISOString(),
|
||||
finishedAt: new Date(0).toISOString(),
|
||||
},
|
||||
],
|
||||
finalAnswer: options.finalAnswer,
|
||||
|
||||
@ -0,0 +1,63 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
REDACTED_INLINE_IMAGE,
|
||||
} from "../../lib/v3/verifier/evidenceNormalization.js";
|
||||
|
||||
describe("buildAgentEvidenceFromStepFinished", () => {
|
||||
it("captures primitive tool results as text evidence", () => {
|
||||
const evidence = buildAgentEvidenceFromStepFinished({
|
||||
type: "step_finished",
|
||||
actionName: "check",
|
||||
actionArgs: {},
|
||||
reasoning: "",
|
||||
toolOutput: { ok: true, result: false },
|
||||
});
|
||||
|
||||
expect(evidence.modalities).toEqual([{ type: "text", content: "false" }]);
|
||||
});
|
||||
|
||||
it("lifts inline screenshot payloads into image evidence and redacts JSON", () => {
|
||||
const inlineScreenshot =
|
||||
Buffer.from("inline screenshot").toString("base64");
|
||||
|
||||
const evidence = buildAgentEvidenceFromStepFinished({
|
||||
type: "step_finished",
|
||||
actionName: "click",
|
||||
actionArgs: { describe: "Open fare details" },
|
||||
reasoning: "",
|
||||
toolOutput: {
|
||||
ok: true,
|
||||
result: {
|
||||
output: {
|
||||
success: true,
|
||||
describe: "Open fare details",
|
||||
screenshotBase64: inlineScreenshot,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const [imageModality, jsonModality] = evidence.modalities;
|
||||
|
||||
expect(JSON.stringify(evidence)).not.toContain(inlineScreenshot);
|
||||
expect(jsonModality).toMatchObject({
|
||||
type: "json",
|
||||
content: {
|
||||
output: {
|
||||
screenshotBase64: REDACTED_INLINE_IMAGE,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(imageModality).toMatchObject({
|
||||
type: "image",
|
||||
mediaType: "image/png",
|
||||
});
|
||||
if (imageModality?.type === "image") {
|
||||
expect(imageModality.bytes).toEqual(
|
||||
Buffer.from(inlineScreenshot, "base64"),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@ -1,4 +1,4 @@
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
loadTrajectoryFromDisk,
|
||||
nextResultFilename,
|
||||
normalizeRubric,
|
||||
writeTrajectoryDir,
|
||||
} from "../../lib/v3/verifier/trajectory.js";
|
||||
|
||||
describe("verifier trajectory utilities", () => {
|
||||
@ -63,21 +64,68 @@ describe("verifier trajectory utilities", () => {
|
||||
it("loads trajectory screenshots and image modalities from disk", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
|
||||
const screenshot = Buffer.from("probe screenshot");
|
||||
const finalScreenshot = Buffer.from("final screenshot");
|
||||
const agentImage = Buffer.from("agent image");
|
||||
await writeFile(path.join(dir, "screenshot_1.png"), screenshot);
|
||||
await writeFile(path.join(dir, "final.png"), finalScreenshot);
|
||||
await mkdir(path.join(dir, "screenshots", "agent"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(dir, "screenshots", "agent", "1.png"),
|
||||
agentImage,
|
||||
);
|
||||
await writeFile(
|
||||
path.join(dir, "trajectory.json"),
|
||||
JSON.stringify({
|
||||
task: { id: "task", instruction: "Do the task" },
|
||||
status: "complete",
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
steps: [
|
||||
{
|
||||
actionName: "act",
|
||||
actionArgs: {},
|
||||
reasoning: "",
|
||||
agentEvidence: {
|
||||
modalities: [
|
||||
{
|
||||
type: "image",
|
||||
mediaType: "image/png",
|
||||
imagePath: "screenshots/agent/1.png",
|
||||
},
|
||||
],
|
||||
},
|
||||
probeEvidence: { screenshotPath: "screenshot_1.png" },
|
||||
toolOutput: { ok: true, result: null },
|
||||
},
|
||||
],
|
||||
finalObservation: {
|
||||
url: "https://example.com/done",
|
||||
screenshotPath: "final.png",
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const trajectory = await loadTrajectoryFromDisk(dir);
|
||||
const modality = trajectory.steps[0].agentEvidence.modalities[0];
|
||||
|
||||
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(screenshot);
|
||||
expect(trajectory.finalObservation?.screenshot).toEqual(finalScreenshot);
|
||||
expect(modality.type).toBe("image");
|
||||
if (modality.type === "image") {
|
||||
expect(modality.bytes).toEqual(agentImage);
|
||||
}
|
||||
});
|
||||
|
||||
it("loads legacy base64 image modalities from disk", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
|
||||
const agentImage = Buffer.from("legacy agent image");
|
||||
await writeFile(
|
||||
path.join(dir, "trajectory.json"),
|
||||
JSON.stringify({
|
||||
task: { id: "task", instruction: "Do the task" },
|
||||
status: "complete",
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
timing: {
|
||||
startedAt: new Date(0).toISOString(),
|
||||
endedAt: new Date(0).toISOString(),
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
index: 0,
|
||||
actionName: "act",
|
||||
actionArgs: {},
|
||||
reasoning: "",
|
||||
@ -90,10 +138,8 @@ describe("verifier trajectory utilities", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
probeEvidence: { screenshotPath: "screenshot_1.png" },
|
||||
probeEvidence: {},
|
||||
toolOutput: { ok: true, result: null },
|
||||
startedAt: new Date(0).toISOString(),
|
||||
finishedAt: new Date(0).toISOString(),
|
||||
},
|
||||
],
|
||||
}),
|
||||
@ -102,13 +148,76 @@ describe("verifier trajectory utilities", () => {
|
||||
const trajectory = await loadTrajectoryFromDisk(dir);
|
||||
const modality = trajectory.steps[0].agentEvidence.modalities[0];
|
||||
|
||||
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(screenshot);
|
||||
expect(modality.type).toBe("image");
|
||||
if (modality.type === "image") {
|
||||
expect(modality.bytes).toEqual(agentImage);
|
||||
}
|
||||
});
|
||||
|
||||
it("redacts inline screenshot payloads when writing trajectories", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
|
||||
const inlineScreenshot =
|
||||
Buffer.from("inline screenshot").toString("base64");
|
||||
|
||||
await writeTrajectoryDir(dir, {
|
||||
task: { id: "task", instruction: "Do the task" },
|
||||
status: "complete",
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
finalObservation: {
|
||||
url: "https://example.com/done",
|
||||
screenshot: Buffer.from("final screenshot"),
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
actionName: "click",
|
||||
actionArgs: {},
|
||||
reasoning: "",
|
||||
agentEvidence: {
|
||||
modalities: [
|
||||
{
|
||||
type: "json",
|
||||
content: {
|
||||
output: {
|
||||
success: true,
|
||||
screenshotBase64: inlineScreenshot,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
probeEvidence: {},
|
||||
toolOutput: {
|
||||
ok: true,
|
||||
result: {
|
||||
output: {
|
||||
success: true,
|
||||
screenshotBase64: inlineScreenshot,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const raw = await readFile(path.join(dir, "trajectory.json"), "utf8");
|
||||
const trajectory = JSON.parse(raw);
|
||||
|
||||
expect(raw).not.toContain(inlineScreenshot);
|
||||
expect(
|
||||
trajectory.steps[0].agentEvidence.modalities[0].content.output
|
||||
.screenshotBase64,
|
||||
).toBe("[redacted inline image payload]");
|
||||
expect(trajectory.steps[0].toolOutput.result.output.screenshotBase64).toBe(
|
||||
"[redacted inline image payload]",
|
||||
);
|
||||
expect(trajectory.finalObservation.screenshotPath).toBe(
|
||||
"screenshots/probe/final.png",
|
||||
);
|
||||
await expect(
|
||||
readFile(path.join(dir, "screenshots", "probe", "final.png")),
|
||||
).resolves.toEqual(Buffer.from("final screenshot"));
|
||||
});
|
||||
|
||||
it("rejects screenshot paths outside the trajectory directory", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "stagehand-verifier-"));
|
||||
await writeFile(
|
||||
@ -117,21 +226,14 @@ describe("verifier trajectory utilities", () => {
|
||||
task: { id: "task", instruction: "Do the task" },
|
||||
status: "complete",
|
||||
usage: { input_tokens: 0, output_tokens: 0 },
|
||||
timing: {
|
||||
startedAt: new Date(0).toISOString(),
|
||||
endedAt: new Date(0).toISOString(),
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
index: 0,
|
||||
actionName: "act",
|
||||
actionArgs: {},
|
||||
reasoning: "",
|
||||
agentEvidence: { modalities: [] },
|
||||
probeEvidence: { screenshotPath: "../../../etc/passwd" },
|
||||
toolOutput: { ok: true, result: null },
|
||||
startedAt: new Date(0).toISOString(),
|
||||
finishedAt: new Date(0).toISOString(),
|
||||
},
|
||||
],
|
||||
}),
|
||||
|
||||
202
packages/docs/sdk-api-reference-labels.js
Normal file
202
packages/docs/sdk-api-reference-labels.js
Normal file
@ -0,0 +1,202 @@
|
||||
// Re-label generated OpenAPI schema fields on SDK-flavored API reference pages.
|
||||
//
|
||||
// Mintlify renders these pages from the OpenAPI wire schema, so body fields use
|
||||
// camelCase even when the selected SDK accepts snake_case parameters. Stainless
|
||||
// already emits language-correct code samples; this small client-side patch keeps
|
||||
// the visible parameter labels aligned with the Python and Ruby SDK snippets.
|
||||
(function () {
|
||||
const SNAKE_CASE_PARAM_LABELS = {
|
||||
acceptDownloads: "accept_downloads",
|
||||
actTimeoutMs: "act_timeout_ms",
|
||||
actionDescription: "action_description",
|
||||
actionId: "action_id",
|
||||
advancedStealth: "advanced_stealth",
|
||||
agentConfig: "agent_config",
|
||||
apiKey: "api_key",
|
||||
backendNodeId: "backend_node_id",
|
||||
baseURL: "base_url",
|
||||
blockAds: "block_ads",
|
||||
browserSettings: "browser_settings",
|
||||
browserbaseAPIKey: "browserbase_api_key",
|
||||
browserbaseProjectID: "browserbase_project_id",
|
||||
browserbaseSessionCreateParams: "browserbase_session_create_params",
|
||||
browserbaseSessionID: "browserbase_session_id",
|
||||
cacheEntry: "cache_entry",
|
||||
cacheKey: "cache_key",
|
||||
captchaImageSelector: "captcha_image_selector",
|
||||
captchaInputSelector: "captcha_input_selector",
|
||||
cdpHeaders: "cdp_headers",
|
||||
cdpUrl: "cdp_url",
|
||||
cdpURL: "cdp_url",
|
||||
chromiumSandbox: "chromium_sandbox",
|
||||
clientLanguage: "client_language",
|
||||
connectTimeoutMs: "connect_timeout_ms",
|
||||
deviceScaleFactor: "device_scale_factor",
|
||||
domSettleTimeoutMs: "dom_settle_timeout_ms",
|
||||
domainPattern: "domain_pattern",
|
||||
downloadsPath: "downloads_path",
|
||||
endTime: "end_time",
|
||||
executablePath: "executable_path",
|
||||
executeOptions: "execute_options",
|
||||
executionModel: "execution_model",
|
||||
extensionId: "extension_id",
|
||||
extensionID: "extension_id",
|
||||
frameId: "frame_id",
|
||||
frameID: "frame_id",
|
||||
googleAuthOptions: "google_auth_options",
|
||||
hasTouch: "has_touch",
|
||||
highlightCursor: "highlight_cursor",
|
||||
httpVersion: "http_version",
|
||||
ignoreDefaultArgs: "ignore_default_args",
|
||||
ignoreHTTPSErrors: "ignore_https_errors",
|
||||
ignoreSelectors: "ignore_selectors",
|
||||
inputTokens: "input_tokens",
|
||||
keepAlive: "keep_alive",
|
||||
launchOptions: "launch_options",
|
||||
logSession: "log_session",
|
||||
maxHeight: "max_height",
|
||||
maxSteps: "max_steps",
|
||||
maxWidth: "max_width",
|
||||
minHeight: "min_height",
|
||||
minWidth: "min_width",
|
||||
modelAPIKey: "model_api_key",
|
||||
modelName: "model_name",
|
||||
operatingSystems: "operating_systems",
|
||||
outputTokens: "output_tokens",
|
||||
pageText: "page_text",
|
||||
pageUrl: "page_url",
|
||||
preserveUserDataDir: "preserve_user_data_dir",
|
||||
projectId: "project_id",
|
||||
projectID: "project_id",
|
||||
recordSession: "record_session",
|
||||
rememberMe: "remember_me",
|
||||
selfHeal: "self_heal",
|
||||
sessionId: "session_id",
|
||||
shouldCache: "should_cache",
|
||||
solveCaptchas: "solve_captchas",
|
||||
streamResponse: "stream_response",
|
||||
systemPrompt: "system_prompt",
|
||||
taskCompleted: "task_completed",
|
||||
timeMs: "time_ms",
|
||||
tokenUsage: "token_usage",
|
||||
toolTimeout: "tool_timeout",
|
||||
universeDomain: "universe_domain",
|
||||
useSearch: "use_search",
|
||||
userDataDir: "user_data_dir",
|
||||
userMetadata: "user_metadata",
|
||||
waitForCaptchaSolves: "wait_for_captcha_solves",
|
||||
waitUntil: "wait_until",
|
||||
xStreamResponse: "x_stream_response",
|
||||
};
|
||||
|
||||
const SDK_PARAM_LABELS = {
|
||||
python: SNAKE_CASE_PARAM_LABELS,
|
||||
ruby: SNAKE_CASE_PARAM_LABELS,
|
||||
};
|
||||
|
||||
const CONTENT_SCOPE_SELECTOR = [
|
||||
"#content-area",
|
||||
"#api-playground-input",
|
||||
"api-section",
|
||||
"field",
|
||||
].join(",");
|
||||
|
||||
const SKIP_SELECTOR = [
|
||||
"code",
|
||||
"input",
|
||||
"pre",
|
||||
"script",
|
||||
"style",
|
||||
"textarea",
|
||||
'[contenteditable="true"]',
|
||||
].join(",");
|
||||
|
||||
let rewriteTimer = null;
|
||||
let lastPath = window.location.pathname;
|
||||
|
||||
function currentSdkLanguage() {
|
||||
const match = window.location.pathname.match(
|
||||
/^\/v3\/api-reference\/([^/]+)(?:\/|$)/,
|
||||
);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function buildPattern(labels) {
|
||||
return new RegExp(
|
||||
`\\b(${Object.keys(labels).map(escapeRegExp).join("|")})\\b`,
|
||||
"g",
|
||||
);
|
||||
}
|
||||
|
||||
function shouldRewriteTextNode(node, pattern) {
|
||||
if (!node.nodeValue) return false;
|
||||
|
||||
const parent = node.parentElement;
|
||||
if (!parent) return false;
|
||||
if (parent.closest(SKIP_SELECTOR)) return false;
|
||||
if (!parent.closest(CONTENT_SCOPE_SELECTOR)) return false;
|
||||
|
||||
pattern.lastIndex = 0;
|
||||
return pattern.test(node.nodeValue);
|
||||
}
|
||||
|
||||
function rewriteLabels(root) {
|
||||
const labels = SDK_PARAM_LABELS[currentSdkLanguage()];
|
||||
if (!labels) return;
|
||||
|
||||
const pattern = buildPattern(labels);
|
||||
const walker = document.createTreeWalker(
|
||||
root || document.body,
|
||||
NodeFilter.SHOW_TEXT,
|
||||
{
|
||||
acceptNode(node) {
|
||||
return shouldRewriteTextNode(node, pattern)
|
||||
? NodeFilter.FILTER_ACCEPT
|
||||
: NodeFilter.FILTER_REJECT;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const nodes = [];
|
||||
while (walker.nextNode()) nodes.push(walker.currentNode);
|
||||
|
||||
for (const node of nodes) {
|
||||
pattern.lastIndex = 0;
|
||||
node.nodeValue = node.nodeValue.replace(pattern, (name) => labels[name]);
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleRewrite(root) {
|
||||
if (rewriteTimer) window.clearTimeout(rewriteTimer);
|
||||
rewriteTimer = window.setTimeout(() => {
|
||||
rewriteTimer = null;
|
||||
window.requestAnimationFrame(() => rewriteLabels(root || document.body));
|
||||
}, 50);
|
||||
}
|
||||
|
||||
scheduleRewrite();
|
||||
|
||||
const observer = new MutationObserver((mutations) => {
|
||||
for (const mutation of mutations) {
|
||||
for (const node of mutation.addedNodes) {
|
||||
if (node.nodeType === Node.ELEMENT_NODE) {
|
||||
scheduleRewrite(node);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
observer.observe(document.body, { childList: true, subtree: true });
|
||||
|
||||
window.setInterval(() => {
|
||||
if (window.location.pathname !== lastPath) {
|
||||
lastPath = window.location.pathname;
|
||||
scheduleRewrite();
|
||||
}
|
||||
}, 250);
|
||||
})();
|
||||
@ -669,6 +669,7 @@ const agent = stagehand.agent({
|
||||
| Anthropic | `anthropic/claude-sonnet-4-5-20250929` |
|
||||
| Anthropic | `anthropic/claude-opus-4-5-20251101` |
|
||||
| Anthropic | `anthropic/claude-opus-4-6` |
|
||||
| Anthropic | `anthropic/claude-opus-4-8` |
|
||||
| Google | `google/gemini-2.5-computer-use-preview-10-2025` |
|
||||
| Google | `google/gemini-3-flash-preview` |
|
||||
| Google | `google/gemini-3-pro-preview` |
|
||||
|
||||
@ -72,6 +72,7 @@ interface AgentInstance {
|
||||
- `"anthropic/claude-sonnet-4-5-20250929"`
|
||||
- `"anthropic/claude-opus-4-5-20251101"`
|
||||
- `"anthropic/claude-opus-4-6"`
|
||||
- `"anthropic/claude-opus-4-8"`
|
||||
- `"google/gemini-2.5-computer-use-preview-10-2025"`
|
||||
- `"google/gemini-3-flash-preview"`
|
||||
- `"google/gemini-3-pro-preview"`
|
||||
@ -867,4 +868,4 @@ The following errors may be thrown by the `agent()` method:
|
||||
- **StagehandDefaultError** - General execution error with detailed message
|
||||
- **AgentAbortError** - Thrown when agent execution is cancelled via an `AbortSignal`
|
||||
- **StreamingCallbacksInNonStreamingModeError** - Thrown when streaming-only callbacks (`onChunk`, `onFinish`, `onError`, `onAbort`) are used without `stream: true`
|
||||
- **ExperimentalNotConfiguredError** - Thrown when experimental features (callbacks, signal, messages, streaming) are used without `experimental: true` in Stagehand constructor
|
||||
- **ExperimentalNotConfiguredError** - Thrown when experimental features (callbacks, signal, messages, streaming) are used without `experimental: true` in Stagehand constructor
|
||||
|
||||
243
packages/evals/framework/trajectoryRecorder.ts
Normal file
243
packages/evals/framework/trajectoryRecorder.ts
Normal file
@ -0,0 +1,243 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildAgentEvidenceFromStepFinished,
|
||||
mergeAgentEvidence,
|
||||
redactInlineImagePayloads,
|
||||
shouldPersistTrajectory,
|
||||
writeTrajectoryDir,
|
||||
} from "@browserbasehq/stagehand";
|
||||
import type {
|
||||
AgentEvidence,
|
||||
AgentEvidenceEvent,
|
||||
AgentFinalAnswerEvent,
|
||||
AgentScreenshotEvidenceEvent,
|
||||
AgentStepFinishedEvent,
|
||||
AgentStepObservedEvent,
|
||||
ProbeEvidence,
|
||||
TaskSpec,
|
||||
Trajectory,
|
||||
TrajectoryStatus,
|
||||
TrajectoryStep,
|
||||
TrajectoryUsage,
|
||||
EvaluationResult,
|
||||
} from "@browserbasehq/stagehand";
|
||||
|
||||
export interface TrajectoryRecorderOptions {
|
||||
taskSpec: TaskSpec;
|
||||
/**
|
||||
* Root directory under which trajectory dirs are written. Each task run
|
||||
* gets a subdirectory named by runId/task.id.
|
||||
* Defaults to `<cwd>/.trajectories`.
|
||||
*/
|
||||
outputRoot?: string;
|
||||
/** Run identifier (e.g., ISO timestamp + env). Defaults to a fresh timestamp. */
|
||||
runId?: string;
|
||||
/**
|
||||
* Override the env-gated persistence default. `true` always persists,
|
||||
* `false` never does, `undefined` defers to VERIFIER_PERSIST_TRAJECTORIES.
|
||||
*/
|
||||
persist?: boolean;
|
||||
}
|
||||
|
||||
export interface TrajectoryFinishOptions {
|
||||
status: TrajectoryStatus;
|
||||
finalAnswer?: string;
|
||||
usage?: Partial<TrajectoryUsage>;
|
||||
}
|
||||
|
||||
const ZERO_USAGE: TrajectoryUsage = {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
};
|
||||
|
||||
export class TrajectoryRecorder {
|
||||
private readonly taskSpec: TaskSpec;
|
||||
private readonly runId: string;
|
||||
private readonly outputDir: string;
|
||||
private readonly persistEnabled: boolean;
|
||||
|
||||
// Steps are appended in arrival order on each step_finished event.
|
||||
private readonly steps: TrajectoryStep[] = [];
|
||||
// The most recent agent-role screenshot. It applies to every step_finished
|
||||
// until a newer agent-role screenshot replaces it — a CUA provider can pick
|
||||
// multiple actions from one screenshot, so each of those steps must carry
|
||||
// that same tier-1 frame. (It is NOT cleared on consume; it is only replaced
|
||||
// by a newer screenshot, or wiped on cancel().)
|
||||
private latestAgentScreenshot?: Buffer;
|
||||
// The most recent probe-role screenshot waits for the matching step_observed.
|
||||
private pendingProbeScreenshot?: Buffer;
|
||||
// Steps that haven't yet had a probe attached. The next step_observed fans
|
||||
// out to all of them (one probe per agent turn, N tool calls per turn).
|
||||
private stepsAwaitingProbe: number[] = [];
|
||||
private finalAnswerEvent?: AgentFinalAnswerEvent;
|
||||
private finalObservation?: ProbeEvidence;
|
||||
|
||||
private onScreenshot(e: AgentScreenshotEvidenceEvent): void {
|
||||
if (e.evidenceRole === "agent") {
|
||||
this.latestAgentScreenshot = e.screenshot;
|
||||
} else {
|
||||
this.pendingProbeScreenshot = e.screenshot;
|
||||
}
|
||||
}
|
||||
|
||||
private onStepFinished(e: AgentStepFinishedEvent): void {
|
||||
const modalities: AgentEvidence["modalities"] = [];
|
||||
if (this.latestAgentScreenshot) {
|
||||
modalities.push({
|
||||
type: "image",
|
||||
bytes: this.latestAgentScreenshot,
|
||||
mediaType: "image/png",
|
||||
});
|
||||
}
|
||||
const merged = mergeAgentEvidence(
|
||||
{ modalities },
|
||||
buildAgentEvidenceFromStepFinished(e),
|
||||
);
|
||||
|
||||
// Intentionally not cleared here: the same agent screenshot applies to
|
||||
// every step in a batched CUA turn. It's replaced when a newer agent
|
||||
// screenshot arrives (onScreenshot) or wiped on cancel().
|
||||
this.stepsAwaitingProbe.push(this.steps.length);
|
||||
this.steps.push({
|
||||
actionName: e.actionName,
|
||||
actionArgs: e.actionArgs,
|
||||
reasoning: e.reasoning,
|
||||
agentEvidence: merged,
|
||||
probeEvidence: {},
|
||||
toolOutput: {
|
||||
...e.toolOutput,
|
||||
result: redactInlineImagePayloads(e.toolOutput.result, e.actionName),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private onStepObserved(e: AgentStepObservedEvent): void {
|
||||
const probe: ProbeEvidence = { url: e.url };
|
||||
if (this.pendingProbeScreenshot)
|
||||
probe.screenshot = this.pendingProbeScreenshot;
|
||||
if (e.ariaTree !== undefined) probe.ariaTree = e.ariaTree;
|
||||
for (const idx of this.stepsAwaitingProbe) {
|
||||
this.steps[idx].probeEvidence = probe;
|
||||
}
|
||||
this.stepsAwaitingProbe = [];
|
||||
this.pendingProbeScreenshot = undefined;
|
||||
}
|
||||
|
||||
private onFinalAnswer(e: AgentFinalAnswerEvent): void {
|
||||
this.finalAnswerEvent = e;
|
||||
if (e.observation) {
|
||||
this.finalObservation = {
|
||||
url: e.observation.url,
|
||||
...(e.observation.screenshot
|
||||
? { screenshot: e.observation.screenshot }
|
||||
: {}),
|
||||
...(e.observation.ariaTree !== undefined
|
||||
? { ariaTree: e.observation.ariaTree }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
constructor(opts: TrajectoryRecorderOptions) {
|
||||
this.taskSpec = opts.taskSpec;
|
||||
this.runId = opts.runId ?? new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const root = opts.outputRoot ?? path.join(process.cwd(), ".trajectories");
|
||||
this.outputDir = path.join(root, this.runId, opts.taskSpec.id);
|
||||
this.persistEnabled = shouldPersistTrajectory(opts.persist);
|
||||
}
|
||||
|
||||
/** Ingest an evidence callback event from agent.execute(). */
|
||||
record(event: AgentEvidenceEvent): void {
|
||||
switch (event.type) {
|
||||
case "screenshot":
|
||||
this.onScreenshot(event);
|
||||
break;
|
||||
case "step_finished":
|
||||
this.onStepFinished(event);
|
||||
break;
|
||||
case "step_observed":
|
||||
this.onStepObserved(event);
|
||||
break;
|
||||
case "final_answer":
|
||||
this.onFinalAnswer(event);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detach listeners, assemble the Trajectory, and (if persistence is on)
|
||||
* write the on-disk layout. Idempotent.
|
||||
*/
|
||||
async finish(opts: TrajectoryFinishOptions): Promise<Trajectory> {
|
||||
const trajectory: Trajectory = {
|
||||
task: this.taskSpec,
|
||||
steps: this.steps,
|
||||
finalAnswer: opts.finalAnswer ?? this.finalAnswerEvent?.message,
|
||||
...(this.finalObservation
|
||||
? { finalObservation: this.finalObservation }
|
||||
: {}),
|
||||
status: opts.status,
|
||||
usage: { ...ZERO_USAGE, ...(opts.usage ?? {}) },
|
||||
};
|
||||
|
||||
if (this.persistEnabled) {
|
||||
await writeTrajectoryDir(this.outputDir, trajectory);
|
||||
}
|
||||
|
||||
return trajectory;
|
||||
}
|
||||
|
||||
/** Throw away in-memory state without writing to disk. Used on early abort. */
|
||||
cancel(): void {
|
||||
this.steps.length = 0;
|
||||
this.latestAgentScreenshot = undefined;
|
||||
this.pendingProbeScreenshot = undefined;
|
||||
this.stepsAwaitingProbe = [];
|
||||
this.finalAnswerEvent = undefined;
|
||||
this.finalObservation = undefined;
|
||||
}
|
||||
|
||||
/** Where the trajectory dir lives (whether or not it was persisted). */
|
||||
get directory(): string {
|
||||
return this.outputDir;
|
||||
}
|
||||
|
||||
/** Whether this recorder wrote the trajectory directory on finish(). */
|
||||
get persisted(): boolean {
|
||||
return this.persistEnabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist evaluator result next to the trajectory. No-op when trajectory
|
||||
* persistence is disabled.
|
||||
*/
|
||||
async persistResult(
|
||||
result: EvaluationResult,
|
||||
filename = "result.json",
|
||||
): Promise<void> {
|
||||
if (!this.persistEnabled) return;
|
||||
|
||||
const scoresDir = path.join(this.outputDir, "scores");
|
||||
await fs.mkdir(scoresDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(scoresDir, filename),
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
|
||||
const taskDataPath = path.join(this.outputDir, "task_data.json");
|
||||
let taskData: Record<string, unknown>;
|
||||
try {
|
||||
taskData = JSON.parse(await fs.readFile(taskDataPath, "utf8")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
} catch {
|
||||
taskData = { task: this.taskSpec };
|
||||
}
|
||||
await fs.writeFile(
|
||||
taskDataPath,
|
||||
JSON.stringify({ ...taskData, result }, null, 2),
|
||||
);
|
||||
}
|
||||
}
|
||||
247
packages/evals/tests/framework/trajectoryRecorder.test.ts
Normal file
247
packages/evals/tests/framework/trajectoryRecorder.test.ts
Normal file
@ -0,0 +1,247 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { TaskSpec } from "@browserbasehq/stagehand";
|
||||
|
||||
import { TrajectoryRecorder } from "../../framework/trajectoryRecorder.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (tempDirs.length > 0) {
|
||||
const dir = tempDirs.pop();
|
||||
if (dir) await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function makeTempDir(): Promise<string> {
|
||||
return fs
|
||||
.mkdtemp(path.join(os.tmpdir(), "trajectory-recorder-"))
|
||||
.then((dir) => {
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
});
|
||||
}
|
||||
|
||||
function makeTaskSpec(): TaskSpec {
|
||||
return {
|
||||
id: "recorder-task",
|
||||
instruction: "Compare economy and business fares.",
|
||||
initUrl: "https://example.com",
|
||||
};
|
||||
}
|
||||
|
||||
function recordSimpleStep(recorder: TrajectoryRecorder, screenshot: Buffer) {
|
||||
recorder.record({
|
||||
type: "screenshot",
|
||||
screenshot,
|
||||
url: "https://example.com/search",
|
||||
evidenceRole: "agent",
|
||||
});
|
||||
recorder.record({
|
||||
type: "step_finished",
|
||||
actionName: "act",
|
||||
actionArgs: { instruction: "Search fares" },
|
||||
reasoning: "Search for fares.",
|
||||
toolOutput: { ok: true, result: "done" },
|
||||
});
|
||||
recorder.record({
|
||||
type: "screenshot",
|
||||
screenshot,
|
||||
url: "https://example.com/search",
|
||||
evidenceRole: "probe",
|
||||
});
|
||||
recorder.record({
|
||||
type: "step_observed",
|
||||
url: "https://example.com/search",
|
||||
});
|
||||
}
|
||||
|
||||
function recordFinalAnswer(
|
||||
recorder: TrajectoryRecorder,
|
||||
opts: { message: string; screenshot: Buffer; ariaTree?: string },
|
||||
): void {
|
||||
recorder.record({
|
||||
type: "final_answer",
|
||||
message: opts.message,
|
||||
observation: {
|
||||
url: "https://example.com/complete",
|
||||
screenshot: opts.screenshot,
|
||||
...(opts.ariaTree !== undefined ? { ariaTree: opts.ariaTree } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("TrajectoryRecorder", () => {
|
||||
it("assembles ordered callback events into trajectory steps", async () => {
|
||||
const recorder = new TrajectoryRecorder({
|
||||
taskSpec: makeTaskSpec(),
|
||||
persist: false,
|
||||
});
|
||||
const screenshot = Buffer.from("screen-1");
|
||||
const staleScreenshot = Buffer.from("stale-screen");
|
||||
const probeScreenshot = Buffer.from("probe-screen");
|
||||
|
||||
recorder.record({
|
||||
type: "screenshot",
|
||||
screenshot: staleScreenshot,
|
||||
url: "https://example.com/stale",
|
||||
evidenceRole: "agent",
|
||||
});
|
||||
recorder.record({
|
||||
type: "screenshot",
|
||||
screenshot,
|
||||
url: "https://example.com/search",
|
||||
evidenceRole: "agent",
|
||||
});
|
||||
recorder.record({
|
||||
type: "step_finished",
|
||||
actionName: "click",
|
||||
actionArgs: { describe: "Open fares" },
|
||||
reasoning: "Open fare details.",
|
||||
toolOutput: { ok: true, result: "opened" },
|
||||
});
|
||||
recorder.record({
|
||||
type: "step_finished",
|
||||
actionName: "extract",
|
||||
actionArgs: { instruction: "Read fares" },
|
||||
reasoning: "Read visible fare cells.",
|
||||
toolOutput: {
|
||||
ok: true,
|
||||
result: { economy: "$100", business: "$250" },
|
||||
},
|
||||
});
|
||||
recorder.record({
|
||||
type: "screenshot",
|
||||
screenshot: probeScreenshot,
|
||||
url: "https://example.com/search",
|
||||
evidenceRole: "probe",
|
||||
});
|
||||
recorder.record({
|
||||
type: "step_observed",
|
||||
url: "https://example.com/search",
|
||||
ariaTree: "RootWebArea\nStaticText: Economy $100",
|
||||
});
|
||||
recordFinalAnswer(recorder, {
|
||||
message: "Business is $150 more than economy.",
|
||||
screenshot: Buffer.from("final-screen"),
|
||||
ariaTree: "RootWebArea\nStaticText: Complete",
|
||||
});
|
||||
|
||||
const trajectory = await recorder.finish({
|
||||
status: "complete",
|
||||
usage: { input_tokens: 10, output_tokens: 5 },
|
||||
});
|
||||
|
||||
expect(trajectory.steps).toHaveLength(2);
|
||||
expect(trajectory.steps[0]).toMatchObject({
|
||||
actionName: "click",
|
||||
probeEvidence: {
|
||||
url: "https://example.com/search",
|
||||
ariaTree: "RootWebArea\nStaticText: Economy $100",
|
||||
},
|
||||
});
|
||||
expect(trajectory.steps[1]).toMatchObject({
|
||||
actionName: "extract",
|
||||
toolOutput: { ok: true, result: { economy: "$100", business: "$250" } },
|
||||
});
|
||||
expect(trajectory.steps[0].probeEvidence.screenshot).toEqual(
|
||||
probeScreenshot,
|
||||
);
|
||||
expect(trajectory.steps[1].probeEvidence.screenshot).toEqual(
|
||||
probeScreenshot,
|
||||
);
|
||||
expect(trajectory.steps[0].agentEvidence.modalities).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ type: "image", bytes: screenshot, mediaType: "image/png" },
|
||||
{ type: "text", content: "Open fare details." },
|
||||
]),
|
||||
);
|
||||
// Both actions were chosen from the same agent screenshot (one screenshot,
|
||||
// two step_finished), so the second step must carry that frame too.
|
||||
expect(trajectory.steps[1].agentEvidence.modalities).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ type: "image", bytes: screenshot, mediaType: "image/png" },
|
||||
]),
|
||||
);
|
||||
expect(trajectory.finalAnswer).toBe("Business is $150 more than economy.");
|
||||
expect(trajectory.finalObservation).toMatchObject({
|
||||
url: "https://example.com/complete",
|
||||
ariaTree: "RootWebArea\nStaticText: Complete",
|
||||
});
|
||||
expect(trajectory.finalObservation?.screenshot).toEqual(
|
||||
Buffer.from("final-screen"),
|
||||
);
|
||||
});
|
||||
|
||||
it("persists trajectory files and evaluator results", async () => {
|
||||
const outputRoot = await makeTempDir();
|
||||
const recorder = new TrajectoryRecorder({
|
||||
taskSpec: makeTaskSpec(),
|
||||
outputRoot,
|
||||
runId: "run-1",
|
||||
persist: true,
|
||||
});
|
||||
const screenshot = Buffer.from("screen-1");
|
||||
|
||||
recordSimpleStep(recorder, screenshot);
|
||||
recordFinalAnswer(recorder, {
|
||||
message: "Complete.",
|
||||
screenshot: Buffer.from("final-screen"),
|
||||
});
|
||||
|
||||
await recorder.finish({ status: "complete" });
|
||||
await recorder.persistResult({
|
||||
outcomeSuccess: true,
|
||||
explanation: "The task was completed.",
|
||||
});
|
||||
|
||||
const taskDir = path.join(outputRoot, "run-1", "recorder-task");
|
||||
await expect(fs.readdir(taskDir)).resolves.toEqual(
|
||||
expect.arrayContaining([
|
||||
"core.log",
|
||||
"scores",
|
||||
"screenshots",
|
||||
"task_data.json",
|
||||
"trajectory.json",
|
||||
]),
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(taskDir, "screenshots", "probe", "1.png")),
|
||||
).resolves.toEqual(screenshot);
|
||||
await expect(
|
||||
fs.readFile(path.join(taskDir, "screenshots", "probe", "final.png")),
|
||||
).resolves.toEqual(Buffer.from("final-screen"));
|
||||
await expect(
|
||||
fs.readFile(path.join(taskDir, "screenshots", "agent", "1.png")),
|
||||
).resolves.toEqual(screenshot);
|
||||
await expect(
|
||||
fs.readFile(path.join(taskDir, "scores", "result.json"), "utf8"),
|
||||
).resolves.toContain('"outcomeSuccess": true');
|
||||
|
||||
const trajectory = JSON.parse(
|
||||
await fs.readFile(path.join(taskDir, "trajectory.json"), "utf8"),
|
||||
);
|
||||
expect(trajectory.steps[0].probeEvidence.screenshotPath).toBe(
|
||||
"screenshots/probe/1.png",
|
||||
);
|
||||
expect(trajectory.finalObservation.screenshotPath).toBe(
|
||||
"screenshots/probe/final.png",
|
||||
);
|
||||
expect(trajectory.steps[0].agentEvidence.modalities).toContainEqual({
|
||||
type: "image",
|
||||
imagePath: "screenshots/agent/1.png",
|
||||
mediaType: "image/png",
|
||||
});
|
||||
|
||||
const taskData = JSON.parse(
|
||||
await fs.readFile(path.join(taskDir, "task_data.json"), "utf8"),
|
||||
);
|
||||
expect(taskData.result).toMatchObject({
|
||||
outcomeSuccess: true,
|
||||
explanation: "The task was completed.",
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -202,9 +202,14 @@ components:
|
||||
- private_key
|
||||
- client_email
|
||||
additionalProperties: false
|
||||
GoogleAuthOptions:
|
||||
GoogleServiceAccountAuth:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
description: Use inline Google Cloud service account credentials for provider
|
||||
authentication
|
||||
type: string
|
||||
const: googleServiceAccount
|
||||
credentials:
|
||||
description: Google Cloud service account credentials
|
||||
$ref: "#/components/schemas/GoogleServiceAccountCredentials"
|
||||
@ -221,21 +226,51 @@ components:
|
||||
universeDomain:
|
||||
description: Google Cloud universe domain
|
||||
type: string
|
||||
required:
|
||||
- type
|
||||
- credentials
|
||||
additionalProperties: false
|
||||
ModelConfigObject:
|
||||
ModelAuth:
|
||||
oneOf:
|
||||
- $ref: "#/components/schemas/GoogleServiceAccountAuth"
|
||||
VertexProviderOptions:
|
||||
type: object
|
||||
properties:
|
||||
provider:
|
||||
description: AI provider for the model (or provide a baseURL endpoint instead)
|
||||
example: openai
|
||||
project:
|
||||
description: Google Cloud project ID for Vertex AI models
|
||||
example: my-gcp-project
|
||||
type: string
|
||||
enum:
|
||||
- openai
|
||||
- anthropic
|
||||
- google
|
||||
- microsoft
|
||||
- bedrock
|
||||
- vertex
|
||||
location:
|
||||
description: Google Cloud location for Vertex AI models
|
||||
example: us-central1
|
||||
type: string
|
||||
baseURL:
|
||||
description: Base URL for the Vertex AI provider
|
||||
type: string
|
||||
format: uri
|
||||
headers:
|
||||
description: Custom headers sent with every request to the Vertex AI provider
|
||||
type: object
|
||||
propertyNames:
|
||||
type: string
|
||||
additionalProperties:
|
||||
type: string
|
||||
required:
|
||||
- project
|
||||
- location
|
||||
additionalProperties: false
|
||||
ModelProviderOptions:
|
||||
type: object
|
||||
properties:
|
||||
vertex:
|
||||
description: Vertex AI provider-specific settings
|
||||
$ref: "#/components/schemas/VertexProviderOptions"
|
||||
required:
|
||||
- vertex
|
||||
additionalProperties: false
|
||||
GenericModelConfigObject:
|
||||
type: object
|
||||
properties:
|
||||
modelName:
|
||||
description: Model name string with provider prefix (e.g., 'openai/gpt-5-nano')
|
||||
example: openai/gpt-5.4-mini
|
||||
@ -256,19 +291,62 @@ components:
|
||||
type: string
|
||||
additionalProperties:
|
||||
type: string
|
||||
project:
|
||||
description: Google Cloud project ID for Vertex AI models
|
||||
example: my-gcp-project
|
||||
provider:
|
||||
description: AI provider for the model (or provide a baseURL endpoint instead)
|
||||
example: openai
|
||||
type: string
|
||||
location:
|
||||
description: Google Cloud location for Vertex AI models
|
||||
example: us-central1
|
||||
type: string
|
||||
googleAuthOptions:
|
||||
description: google-auth-library options used to authenticate Vertex AI models
|
||||
$ref: "#/components/schemas/GoogleAuthOptions"
|
||||
enum:
|
||||
- openai
|
||||
- anthropic
|
||||
- google
|
||||
- microsoft
|
||||
- bedrock
|
||||
required:
|
||||
- modelName
|
||||
additionalProperties: false
|
||||
VertexModelConfigObject:
|
||||
type: object
|
||||
properties:
|
||||
modelName:
|
||||
description: Model name string with provider prefix (e.g., 'openai/gpt-5-nano')
|
||||
example: openai/gpt-5.4-mini
|
||||
type: string
|
||||
apiKey:
|
||||
description: API key for the model provider
|
||||
example: sk-some-openai-api-key
|
||||
type: string
|
||||
baseURL:
|
||||
description: Base URL for the model provider
|
||||
example: https://api.openai.com/v1
|
||||
type: string
|
||||
format: uri
|
||||
headers:
|
||||
description: Custom headers sent with every request to the model provider
|
||||
type: object
|
||||
propertyNames:
|
||||
type: string
|
||||
additionalProperties:
|
||||
type: string
|
||||
provider:
|
||||
description: Vertex AI model provider
|
||||
type: string
|
||||
const: vertex
|
||||
auth:
|
||||
description: Vertex provider authentication configuration
|
||||
$ref: "#/components/schemas/ModelAuth"
|
||||
providerOptions:
|
||||
description: Vertex provider-specific model configuration
|
||||
$ref: "#/components/schemas/ModelProviderOptions"
|
||||
required:
|
||||
- modelName
|
||||
- provider
|
||||
- auth
|
||||
- providerOptions
|
||||
additionalProperties: false
|
||||
ModelConfigObject:
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/VertexModelConfigObject"
|
||||
- $ref: "#/components/schemas/GenericModelConfigObject"
|
||||
ModelConfig:
|
||||
$ref: "#/components/schemas/ModelConfigObject"
|
||||
Action:
|
||||
@ -1319,56 +1397,6 @@ components:
|
||||
acceptDownloads:
|
||||
type: boolean
|
||||
additionalProperties: false
|
||||
ModelConfigObjectOutput:
|
||||
type: object
|
||||
properties:
|
||||
provider:
|
||||
description: AI provider for the model (or provide a baseURL endpoint instead)
|
||||
example: openai
|
||||
type: string
|
||||
enum:
|
||||
- openai
|
||||
- anthropic
|
||||
- google
|
||||
- microsoft
|
||||
- bedrock
|
||||
- vertex
|
||||
modelName:
|
||||
description: Model name string with provider prefix (e.g., 'openai/gpt-5-nano')
|
||||
example: openai/gpt-5.4-mini
|
||||
type: string
|
||||
apiKey:
|
||||
description: API key for the model provider
|
||||
example: sk-some-openai-api-key
|
||||
type: string
|
||||
baseURL:
|
||||
description: Base URL for the model provider
|
||||
example: https://api.openai.com/v1
|
||||
type: string
|
||||
format: uri
|
||||
headers:
|
||||
description: Custom headers sent with every request to the model provider
|
||||
type: object
|
||||
propertyNames:
|
||||
type: string
|
||||
additionalProperties:
|
||||
type: string
|
||||
project:
|
||||
description: Google Cloud project ID for Vertex AI models
|
||||
example: my-gcp-project
|
||||
type: string
|
||||
location:
|
||||
description: Google Cloud location for Vertex AI models
|
||||
example: us-central1
|
||||
type: string
|
||||
googleAuthOptions:
|
||||
description: google-auth-library options used to authenticate Vertex AI models
|
||||
$ref: "#/components/schemas/GoogleAuthOptions"
|
||||
required:
|
||||
- modelName
|
||||
additionalProperties: false
|
||||
ModelConfigOutput:
|
||||
$ref: "#/components/schemas/ModelConfigObjectOutput"
|
||||
ActionOutput:
|
||||
description: Action object returned by observe and used by act
|
||||
type: object
|
||||
@ -1633,7 +1661,7 @@ components:
|
||||
description: Model configuration object or model name string (e.g.,
|
||||
'openai/gpt-5-nano')
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/ModelConfigOutput"
|
||||
- $ref: "#/components/schemas/ModelConfig"
|
||||
- type: string
|
||||
variables:
|
||||
description: Variables to substitute in the action instruction. Accepts flat
|
||||
@ -1693,7 +1721,7 @@ components:
|
||||
description: Model configuration object or model name string (e.g.,
|
||||
'openai/gpt-5-nano')
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/ModelConfigOutput"
|
||||
- $ref: "#/components/schemas/ModelConfig"
|
||||
- type: string
|
||||
timeout:
|
||||
description: Timeout in ms for the extraction
|
||||
@ -1738,7 +1766,7 @@ components:
|
||||
description: Model configuration object or model name string (e.g.,
|
||||
'openai/gpt-5-nano')
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/ModelConfigOutput"
|
||||
- $ref: "#/components/schemas/ModelConfig"
|
||||
- type: string
|
||||
variables:
|
||||
description: Variables whose names are exposed to the model so observe() returns
|
||||
@ -1801,7 +1829,7 @@ components:
|
||||
description: Model configuration object or model name string (e.g.,
|
||||
'openai/gpt-5-nano')
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/ModelConfigOutput"
|
||||
- $ref: "#/components/schemas/ModelConfig"
|
||||
- type: string
|
||||
systemPrompt:
|
||||
description: Custom system prompt for the agent
|
||||
@ -1825,7 +1853,7 @@ components:
|
||||
agent tools). If not specified, inherits from the main model
|
||||
configuration.
|
||||
anyOf:
|
||||
- $ref: "#/components/schemas/ModelConfigOutput"
|
||||
- $ref: "#/components/schemas/ModelConfig"
|
||||
- type: string
|
||||
additionalProperties: false
|
||||
AgentUsageOutput:
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
* SEA_INCLUDE_SOURCEMAPS.
|
||||
* Example: pnpm run build:sea:cjs -- --target-platform=linux --target-arch=arm64
|
||||
*/
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
@ -87,6 +87,17 @@ const includeSourcemaps = parseBoolean(
|
||||
false,
|
||||
);
|
||||
|
||||
const gitOutput = (args: string[]) =>
|
||||
execFileSync("git", args, { cwd: repoDir, encoding: "utf8" }).trim();
|
||||
const buildCommit = gitOutput(["rev-parse", "HEAD"]);
|
||||
const seaVersion = [
|
||||
`commit: ${buildCommit}`,
|
||||
`timestamp: ${gitOutput(["show", "-s", "--format=%cI", buildCommit])}`,
|
||||
].join("\n");
|
||||
const seaBuildDefine = {
|
||||
__STAGEHAND_SERVER_VERSION__: JSON.stringify(seaVersion),
|
||||
};
|
||||
|
||||
const run = (cmd: string, args: string[], opts: { cwd?: string } = {}) => {
|
||||
const result = spawnSync(cmd, args, { stdio: "inherit", ...opts });
|
||||
if (result.error) {
|
||||
@ -273,6 +284,7 @@ const buildCjsBundle = () => {
|
||||
platform: "node",
|
||||
format: "cjs",
|
||||
outfile: bundlePath,
|
||||
define: seaBuildDefine,
|
||||
logLevel: "warning",
|
||||
absWorkingDir: repoDir,
|
||||
});
|
||||
@ -295,6 +307,7 @@ const buildEsmBundle = () => {
|
||||
format: "esm",
|
||||
treeShaking: false,
|
||||
outfile: appBundlePath,
|
||||
define: seaBuildDefine,
|
||||
alias: {
|
||||
"@browserbasehq/stagehand": `${repoDir}/packages/core/dist/esm/index.js`,
|
||||
},
|
||||
@ -412,6 +425,15 @@ const { pathToFileURL } = require("node:url");
|
||||
const bundleBase64 = ${JSON.stringify(appBytes.toString("base64"))};
|
||||
const bundleLength = ${appBytes.length};
|
||||
const bundleHash = ${JSON.stringify(bundleHash)};
|
||||
const versionOutput = ${JSON.stringify(seaVersion)};
|
||||
|
||||
const argv = process.argv.slice(1);
|
||||
const normalizedArgv = argv[0]?.startsWith("--") ? argv : argv.slice(1);
|
||||
|
||||
if (normalizedArgv.includes("--version")) {
|
||||
console.log(versionOutput);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const cacheRoot =
|
||||
process.env.STAGEHAND_SEA_CACHE_DIR ||
|
||||
|
||||
@ -44,7 +44,12 @@ async function main() {
|
||||
LocalBrowserLaunchOptions: Api.LocalBrowserLaunchOptionsSchema,
|
||||
GoogleServiceAccountCredentials:
|
||||
Api.GoogleServiceAccountCredentialsSchema,
|
||||
GoogleAuthOptions: Api.GoogleAuthOptionsSchema,
|
||||
GoogleServiceAccountAuth: Api.GoogleServiceAccountAuthSchema,
|
||||
ModelAuth: Api.ModelAuthSchema,
|
||||
VertexProviderOptions: Api.VertexProviderOptionsSchema,
|
||||
ModelProviderOptions: Api.ModelProviderOptionsSchema,
|
||||
GenericModelConfigObject: Api.GenericModelConfigObjectSchema,
|
||||
VertexModelConfigObject: Api.VertexModelConfigObjectSchema,
|
||||
ModelConfigObject: Api.ModelConfigObjectSchema,
|
||||
ModelConfig: Api.ModelConfigSchema,
|
||||
Action: Api.ActionSchema,
|
||||
|
||||
@ -208,10 +208,17 @@ export class InMemorySessionStore implements SessionStore {
|
||||
|
||||
const options: V3Options = {
|
||||
env: isBrowserbase ? "BROWSERBASE" : "LOCAL",
|
||||
model: {
|
||||
modelName: params.modelName,
|
||||
apiKey: ctx.modelApiKey,
|
||||
},
|
||||
model: ctx.requestModelConfig
|
||||
? {
|
||||
...ctx.requestModelConfig,
|
||||
...(ctx.requestModelConfig.apiKey
|
||||
? {}
|
||||
: { apiKey: ctx.modelApiKey }),
|
||||
}
|
||||
: {
|
||||
modelName: params.modelName,
|
||||
apiKey: ctx.modelApiKey,
|
||||
},
|
||||
verbose: params.verbose,
|
||||
systemPrompt: params.systemPrompt,
|
||||
selfHeal: params.selfHeal,
|
||||
|
||||
@ -67,6 +67,8 @@ export interface CreateSessionParams {
|
||||
export interface RequestContext {
|
||||
/** Model API key (from x-model-api-key header) */
|
||||
modelApiKey?: string;
|
||||
/** Structured model config from the current request body */
|
||||
requestModelConfig?: Api.ModelConfig;
|
||||
/** Logger function for this request */
|
||||
logger?: (message: LogLine) => void;
|
||||
}
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import type { FastifyRequest } from "fastify";
|
||||
import { Api } from "@browserbasehq/stagehand";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { MissingHeaderError } from "../types/error.js";
|
||||
|
||||
@ -39,41 +41,134 @@ export const getOptionalHeader = (
|
||||
return headerValue;
|
||||
};
|
||||
|
||||
const emptyStringSchema = z.literal("").transform((): undefined => undefined);
|
||||
const requiredStringSchema = z.string().min(1);
|
||||
const optionalStringSchema = z
|
||||
.union([emptyStringSchema, z.string()])
|
||||
.optional();
|
||||
|
||||
const requestModelInputSchema = z
|
||||
.union([
|
||||
emptyStringSchema,
|
||||
requiredStringSchema.transform(
|
||||
(modelName): Api.ModelConfig => ({ modelName }),
|
||||
),
|
||||
Api.ModelConfigSchema,
|
||||
])
|
||||
.optional();
|
||||
const requestModelCarrierSchema = z
|
||||
.object({
|
||||
model: requestModelInputSchema,
|
||||
})
|
||||
.passthrough();
|
||||
const unparsedModelCarrierSchema = z
|
||||
.object({
|
||||
model: z.unknown().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const requestModelEnvelopeSchema = z
|
||||
.object({
|
||||
options: requestModelCarrierSchema.optional(),
|
||||
agentConfig: unparsedModelCarrierSchema.optional(),
|
||||
modelName: optionalStringSchema,
|
||||
})
|
||||
.passthrough();
|
||||
const stagehandInitModelEnvelopeSchema = z
|
||||
.object({
|
||||
options: requestModelCarrierSchema.optional(),
|
||||
agentConfig: requestModelCarrierSchema.optional(),
|
||||
modelName: optionalStringSchema,
|
||||
})
|
||||
.passthrough();
|
||||
const requestModelConfigSchema = z
|
||||
.object({
|
||||
model: Api.ModelConfigSchema.optional(),
|
||||
modelName: optionalStringSchema,
|
||||
apiKey: optionalStringSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type RequestModelConfig = z.infer<typeof requestModelConfigSchema>;
|
||||
type RequestModelConfigResult =
|
||||
| { success: true; data: RequestModelConfig }
|
||||
| { success: false; error: z.ZodError };
|
||||
|
||||
/**
|
||||
* Extracts model name from request body, supporting V3 structure.
|
||||
* - V3: body.options.model.modelName
|
||||
* Extracts request-level model config with precedence.
|
||||
*
|
||||
* Model name:
|
||||
* 1. body.options.model.modelName or body.options.model string
|
||||
* 2. Legacy body.modelName fallback
|
||||
*
|
||||
* API key:
|
||||
* 1. body.options.model.apiKey
|
||||
* 2. x-model-api-key header
|
||||
*
|
||||
* agentConfig.model is parsed separately for Stagehand initialization. Its
|
||||
* credentials are scoped to the agent main model and must not become the
|
||||
* request-level API key fallback used by action/execution models.
|
||||
*/
|
||||
export function getModelName(request: FastifyRequest): string | undefined {
|
||||
const body = request.body as Record<string, unknown> | undefined;
|
||||
const options = body?.options as Record<string, unknown> | undefined;
|
||||
const model = options?.model as Record<string, unknown> | undefined;
|
||||
|
||||
if (typeof model?.modelName === "string" && model.modelName) {
|
||||
return model.modelName;
|
||||
export function getRequestModelConfig(
|
||||
request: FastifyRequest,
|
||||
): RequestModelConfigResult {
|
||||
const bodyResult = requestModelEnvelopeSchema.safeParse(request.body ?? {});
|
||||
if (!bodyResult.success) {
|
||||
return bodyResult;
|
||||
}
|
||||
|
||||
if (typeof body?.modelName === "string" && body.modelName) {
|
||||
return body.modelName;
|
||||
const body = bodyResult.data;
|
||||
const model = body.options?.model;
|
||||
const configResult = requestModelConfigSchema.safeParse({
|
||||
model,
|
||||
modelName: model?.modelName ?? body.modelName,
|
||||
apiKey: model?.apiKey ?? getOptionalHeader(request, "x-model-api-key"),
|
||||
});
|
||||
if (!configResult.success) {
|
||||
return configResult;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
return configResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the model API key with precedence:
|
||||
* 1. Per-request body apiKey (V3: body.options.model.apiKey)
|
||||
* 2. Per-request header x-model-api-key
|
||||
* Extracts the structured model config used when creating a Stagehand instance
|
||||
* for this request. This can read agentConfig.model for agentExecute startup,
|
||||
* but it does not promote agent model credentials to the request-level API key.
|
||||
*/
|
||||
export function getModelApiKey(request: FastifyRequest): string | undefined {
|
||||
const body = request.body as Record<string, unknown> | undefined;
|
||||
const options = body?.options as Record<string, unknown> | undefined;
|
||||
const model = options?.model as Record<string, unknown> | undefined;
|
||||
|
||||
if (typeof model?.apiKey === "string" && model.apiKey) {
|
||||
return model.apiKey;
|
||||
export function getStagehandInitModelConfig(
|
||||
request: FastifyRequest,
|
||||
requestModelConfig?: RequestModelConfig,
|
||||
): RequestModelConfigResult {
|
||||
const baseConfigResult = requestModelConfig
|
||||
? requestModelConfigSchema.safeParse(requestModelConfig)
|
||||
: getRequestModelConfig(request);
|
||||
if (!baseConfigResult.success) {
|
||||
return baseConfigResult;
|
||||
}
|
||||
|
||||
return getOptionalHeader(request, "x-model-api-key");
|
||||
const baseConfig = baseConfigResult.data;
|
||||
if (baseConfig.model) {
|
||||
return baseConfigResult;
|
||||
}
|
||||
|
||||
const bodyResult = stagehandInitModelEnvelopeSchema.safeParse(
|
||||
request.body ?? {},
|
||||
);
|
||||
if (!bodyResult.success) {
|
||||
return bodyResult;
|
||||
}
|
||||
|
||||
const agentModel = bodyResult.data.agentConfig?.model;
|
||||
const configResult = requestModelConfigSchema.safeParse({
|
||||
model: agentModel,
|
||||
modelName: agentModel?.modelName ?? baseConfig.modelName,
|
||||
apiKey: baseConfig.apiKey,
|
||||
});
|
||||
if (!configResult.success) {
|
||||
return configResult;
|
||||
}
|
||||
|
||||
return configResult;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -6,10 +6,11 @@ import { z } from "zod/v4";
|
||||
|
||||
import { AppError } from "./errorHandler.js";
|
||||
import {
|
||||
getModelApiKey,
|
||||
getModelName,
|
||||
getOptionalHeader,
|
||||
getRequestModelConfig,
|
||||
getStagehandInitModelConfig,
|
||||
shouldRespondWithSSE,
|
||||
type RequestModelConfig,
|
||||
} from "./header.js";
|
||||
import { error, success } from "./response.js";
|
||||
import { getSessionStore } from "./sessionStoreManager.js";
|
||||
@ -35,6 +36,13 @@ type StreamEventName =
|
||||
| "error";
|
||||
type StreamPayloadType = "system" | "log";
|
||||
|
||||
function formatZodIssues(err: z.ZodError) {
|
||||
return err.issues.map((issue) => ({
|
||||
path: issue.path[0] ?? "unknown",
|
||||
message: issue.message,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function createStreamingResponse<TV3>({
|
||||
sessionId,
|
||||
request,
|
||||
@ -44,7 +52,6 @@ export async function createStreamingResponse<TV3>({
|
||||
operation,
|
||||
}: StreamingResponseOptions<TV3>) {
|
||||
const shouldStreamResponse = shouldRespondWithSSE(request);
|
||||
const modelApiKey = getModelApiKey(request);
|
||||
|
||||
const sessionStore = getSessionStore();
|
||||
const sessionConfig = await sessionStore.getSessionConfig(sessionId);
|
||||
@ -127,9 +134,45 @@ export async function createStreamingResponse<TV3>({
|
||||
const actionId = v4();
|
||||
|
||||
sendData("starting", "system", { status: "starting" });
|
||||
const sendZodValidationError = (err: z.ZodError) => {
|
||||
const validationIssues = formatZodIssues(err);
|
||||
if (shouldStreamResponse) {
|
||||
sendData("error", "system", {
|
||||
status: "error",
|
||||
error: validationIssues,
|
||||
});
|
||||
reply.raw.end();
|
||||
return reply;
|
||||
}
|
||||
|
||||
return reply.status(StatusCodes.BAD_REQUEST).send({
|
||||
error: validationIssues,
|
||||
});
|
||||
};
|
||||
|
||||
const requestModelConfigResult = getRequestModelConfig(request);
|
||||
if (requestModelConfigResult.success === false) {
|
||||
return sendZodValidationError(requestModelConfigResult.error);
|
||||
}
|
||||
|
||||
const requestModelConfig: RequestModelConfig = requestModelConfigResult.data;
|
||||
const stagehandInitModelConfigResult = getStagehandInitModelConfig(
|
||||
request,
|
||||
requestModelConfig,
|
||||
);
|
||||
if (stagehandInitModelConfigResult.success === false) {
|
||||
return sendZodValidationError(stagehandInitModelConfigResult.error);
|
||||
}
|
||||
|
||||
const stagehandInitModelConfig = stagehandInitModelConfigResult.data;
|
||||
const modelApiKey = requestModelConfig.apiKey;
|
||||
const parsedRequestModelConfig = stagehandInitModelConfig.model?.modelName
|
||||
? stagehandInitModelConfig.model
|
||||
: undefined;
|
||||
|
||||
const requestContext: RequestContext = {
|
||||
modelApiKey,
|
||||
requestModelConfig: parsedRequestModelConfig,
|
||||
logger: shouldStreamResponse
|
||||
? (message) => {
|
||||
sendData("running", "log", { status: "running", message });
|
||||
@ -177,7 +220,7 @@ export async function createStreamingResponse<TV3>({
|
||||
operation: operation ?? "operation",
|
||||
sessionId,
|
||||
browserType,
|
||||
modelName: getModelName(request),
|
||||
modelName: requestModelConfig.modelName,
|
||||
hasModelApiKey: Boolean(modelApiKey),
|
||||
hasBrowserbaseApiKey: Boolean(browserbaseApiKey),
|
||||
hasBrowserbaseProjectId: Boolean(browserbaseProjectId),
|
||||
|
||||
@ -8,7 +8,10 @@ import { z } from "zod/v4";
|
||||
|
||||
import { authMiddleware } from "../../../lib/auth.js";
|
||||
import { withErrorHandling } from "../../../lib/errorHandler.js";
|
||||
import { getModelApiKey, getOptionalHeader } from "../../../lib/header.js";
|
||||
import {
|
||||
getOptionalHeader,
|
||||
getRequestModelConfig,
|
||||
} from "../../../lib/header.js";
|
||||
import { error, success } from "../../../lib/response.js";
|
||||
import { getSessionStore } from "../../../lib/sessionStoreManager.js";
|
||||
import { AISDK_PROVIDERS } from "../../../types/model.js";
|
||||
@ -110,11 +113,30 @@ const startRouteHandler: RouteHandler = withErrorHandling(
|
||||
}
|
||||
|
||||
const browserType = browser?.type ?? "browserbase";
|
||||
const localBrowserLaunchOptions =
|
||||
browserType === "local" && browser?.launchOptions && !browser?.cdpUrl
|
||||
? browser.launchOptions
|
||||
: undefined;
|
||||
|
||||
let bbApiKey: string | undefined;
|
||||
let browserbaseProjectId: string | undefined;
|
||||
let browserbaseSessionId: string | undefined;
|
||||
let connectUrl: string | undefined;
|
||||
let localBrowserModelApiKey: string | undefined;
|
||||
|
||||
if (localBrowserLaunchOptions) {
|
||||
const requestModelConfigResult = getRequestModelConfig(request);
|
||||
if (requestModelConfigResult.success === false) {
|
||||
return reply.status(StatusCodes.BAD_REQUEST).send({
|
||||
error: requestModelConfigResult.error.issues.map((issue) => ({
|
||||
path: issue.path[0] ?? "unknown",
|
||||
message: issue.message,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
localBrowserModelApiKey = requestModelConfigResult.data.apiKey;
|
||||
}
|
||||
|
||||
if (browserType === "browserbase") {
|
||||
bbApiKey = getOptionalHeader(request, "x-bb-api-key");
|
||||
@ -210,12 +232,11 @@ const startRouteHandler: RouteHandler = withErrorHandling(
|
||||
// For local browsers with launchOptions (no explicit cdpUrl), eagerly
|
||||
// initialize the browser so we can return the actual CDP URL
|
||||
let finalCdpUrl = connectUrl ?? session.cdpUrl ?? "";
|
||||
if (browserType === "local" && browser?.launchOptions && !browser?.cdpUrl) {
|
||||
const modelApiKey = getModelApiKey(request);
|
||||
if (localBrowserLaunchOptions) {
|
||||
try {
|
||||
const stagehand = await sessionStore.getOrCreateStagehand(
|
||||
session.sessionId,
|
||||
{ modelApiKey },
|
||||
{ modelApiKey: localBrowserModelApiKey },
|
||||
);
|
||||
finalCdpUrl = stagehand.connectURL();
|
||||
} catch (err) {
|
||||
@ -226,12 +247,12 @@ const startRouteHandler: RouteHandler = withErrorHandling(
|
||||
browserType,
|
||||
chromePathEnv: process.env.CHROME_PATH,
|
||||
launchOptions: {
|
||||
executablePath: browser.launchOptions.executablePath,
|
||||
argsCount: browser.launchOptions.args?.length ?? 0,
|
||||
headless: browser.launchOptions.headless,
|
||||
hasUserDataDir: Boolean(browser.launchOptions.userDataDir),
|
||||
port: browser.launchOptions.port,
|
||||
connectTimeoutMs: browser.launchOptions.connectTimeoutMs,
|
||||
executablePath: localBrowserLaunchOptions.executablePath,
|
||||
argsCount: localBrowserLaunchOptions.args?.length ?? 0,
|
||||
headless: localBrowserLaunchOptions.headless,
|
||||
hasUserDataDir: Boolean(localBrowserLaunchOptions.userDataDir),
|
||||
port: localBrowserLaunchOptions.port,
|
||||
connectTimeoutMs: localBrowserLaunchOptions.connectTimeoutMs,
|
||||
},
|
||||
},
|
||||
"Failed to initialize local browser session in /v1/sessions/start",
|
||||
|
||||
@ -1,15 +1,23 @@
|
||||
import { __internalMaybeRunShutdownSupervisorFromArgv } from "@browserbasehq/stagehand";
|
||||
declare const __STAGEHAND_SERVER_VERSION__: string;
|
||||
|
||||
// if SEA binary is launched with --supervisor, it will run the shutdown supervisor only
|
||||
const argv = process.argv.slice(1);
|
||||
const normalizedArgv = argv[0]?.startsWith("--") ? argv : argv.slice(1);
|
||||
|
||||
// otherwise, start the stagehand/server
|
||||
if (!__internalMaybeRunShutdownSupervisorFromArgv(normalizedArgv)) {
|
||||
// Force flow logs on before the server module loads Stagehand core.
|
||||
process.env.BROWSERBASE_FLOW_LOGS = "1";
|
||||
void import("./server.js").catch((err) => {
|
||||
if (normalizedArgv.includes("--version")) {
|
||||
console.log(__STAGEHAND_SERVER_VERSION__);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
void import("@browserbasehq/stagehand")
|
||||
.then(async ({ __internalMaybeRunShutdownSupervisorFromArgv }) => {
|
||||
if (__internalMaybeRunShutdownSupervisorFromArgv(normalizedArgv)) {
|
||||
return;
|
||||
}
|
||||
|
||||
process.env.BROWSERBASE_FLOW_LOGS = "1";
|
||||
await import("./server.js");
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error("Failed to start server:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
120
packages/server-v3/tests/unit/requestModelConfig.test.ts
Normal file
120
packages/server-v3/tests/unit/requestModelConfig.test.ts
Normal file
@ -0,0 +1,120 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import type { FastifyRequest } from "fastify";
|
||||
|
||||
import {
|
||||
getRequestModelConfig,
|
||||
getStagehandInitModelConfig,
|
||||
type RequestModelConfig,
|
||||
} from "../../src/lib/header.js";
|
||||
|
||||
function createRequest({
|
||||
body,
|
||||
headers = {},
|
||||
}: {
|
||||
body?: Record<string, unknown>;
|
||||
headers?: Record<string, string>;
|
||||
}): FastifyRequest {
|
||||
return {
|
||||
body,
|
||||
headers,
|
||||
} as FastifyRequest;
|
||||
}
|
||||
|
||||
function assertSuccess(
|
||||
result: ReturnType<typeof getRequestModelConfig>,
|
||||
): RequestModelConfig {
|
||||
if (result.success === false) {
|
||||
throw result.error;
|
||||
}
|
||||
assert.equal(result.success, true);
|
||||
return result.data;
|
||||
}
|
||||
|
||||
describe("getRequestModelConfig", () => {
|
||||
it("preserves a Vertex model config from an action request so auth fields reach session initialization", () => {
|
||||
const model = {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
auth: {
|
||||
type: "googleServiceAccount",
|
||||
credentials: {
|
||||
client_email: "vertex@example.iam.gserviceaccount.com",
|
||||
private_key:
|
||||
"-----BEGIN PRIVATE KEY-----\\ntest\\n-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
providerOptions: {
|
||||
vertex: {
|
||||
project: "test-gcp-project",
|
||||
location: "us-central1",
|
||||
},
|
||||
},
|
||||
};
|
||||
const request = createRequest({
|
||||
body: {
|
||||
options: { model },
|
||||
},
|
||||
headers: {
|
||||
"x-model-api-key": "sk-header",
|
||||
},
|
||||
});
|
||||
|
||||
const config = assertSuccess(getRequestModelConfig(request));
|
||||
|
||||
assert.equal(config.modelName, "vertex/gemini-2.5-flash");
|
||||
assert.equal(config.apiKey, "sk-header");
|
||||
assert.deepEqual(config.model, model);
|
||||
});
|
||||
|
||||
it("does not read agentConfig.model as a request-level model config", () => {
|
||||
const request = createRequest({
|
||||
body: {
|
||||
agentConfig: { model: "google/gemini-2.5-flash" },
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(assertSuccess(getRequestModelConfig(request)), {
|
||||
model: undefined,
|
||||
modelName: undefined,
|
||||
apiKey: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes an agent string model into the session bootstrap model config", () => {
|
||||
const request = createRequest({
|
||||
body: {
|
||||
agentConfig: { model: "google/gemini-2.5-flash" },
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(assertSuccess(getStagehandInitModelConfig(request)), {
|
||||
model: { modelName: "google/gemini-2.5-flash" },
|
||||
modelName: "google/gemini-2.5-flash",
|
||||
apiKey: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a validation result instead of throwing when agent bootstrap model config is invalid", () => {
|
||||
const request = createRequest({
|
||||
body: {
|
||||
agentConfig: {
|
||||
model: {
|
||||
provider: "vertex",
|
||||
modelName: "vertex/gemini-2.5-flash",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(assertSuccess(getRequestModelConfig(request)), {
|
||||
model: undefined,
|
||||
modelName: undefined,
|
||||
apiKey: undefined,
|
||||
});
|
||||
|
||||
const result = getStagehandInitModelConfig(request);
|
||||
assert.equal(result.success, false);
|
||||
});
|
||||
});
|
||||
1280
pnpm-lock.yaml
generated
1280
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user