Compare commits

...

2 Commits

Author SHA1 Message Date
miguel
482e49bac6 fix tests and build 2026-05-21 07:45:29 -07:00
Chromie
0f85f972a2
fix[evals] remove braintrust api key requirement (#2145)
# why

# what changed

# test plan


<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Make Braintrust optional for evals. Evals run without a
BRAINTRUST_API_KEY and skip sending logs, removing prior
errors/warnings; behavior is unchanged when a key is set.

- **Bug Fixes**
- Added hasBraintrustApiKey() and made tracedSpan() a passthrough when
no key.
- Runner now passes noSendLogs: true to Eval and skips flush() without a
key; sends logs and flushes when present.
- `AISdkClientWrapped` uses the plain `ai` SDK unless `braintrust` can
wrap it (key present), avoiding “Please specify an api key” and flush
warnings.
  - Added tests covering both key-present and key-absent paths.

<sup>Written for commit 0a850a00a6ebe6a50c0a39825c6e7a8b23693403.
Summary will update on new commits. <a
href="https://cubic.dev/pr/browserbase/stagehand/pull/2145?utm_source=github">Review
in cubic</a></sup>

<!-- End of auto-generated description by cubic. -->

---------

Co-authored-by: Claude <noreply@anthropic.com>
2026-05-21 07:19:43 -07:00
6 changed files with 215 additions and 11 deletions

View File

@ -1,5 +1,9 @@
let braintrustPromise: Promise<typeof import("braintrust")> | undefined;
export function hasBraintrustApiKey(): boolean {
return Boolean(process.env.BRAINTRUST_API_KEY);
}
export function loadBraintrust(): Promise<typeof import("braintrust")> {
braintrustPromise ??= import("braintrust");
return braintrustPromise;
@ -9,6 +13,9 @@ export async function tracedSpan<T>(
fn: () => Promise<T>,
options: { name: string },
): Promise<T> {
if (!hasBraintrustApiKey()) {
return fn();
}
const { traced } = await loadBraintrust();
return traced(fn, options);
}

View File

@ -22,7 +22,11 @@ import type { Testcase, EvalInput } from "../types/evals.js";
import { generateBenchTestcases } from "./benchPlanner.js";
import { DEFAULT_BENCH_HARNESS, type Harness } from "./benchTypes.js";
import { executeBenchTask } from "./benchRunner.js";
import { loadBraintrust, tracedSpan } from "./braintrust.js";
import {
hasBraintrustApiKey,
loadBraintrust,
tracedSpan,
} from "./braintrust.js";
import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js";
import { loadTaskModuleFromPath } from "./taskLoader.js";
@ -352,6 +356,7 @@ export async function runEvals(
: [exactMatch, errorMatch];
const { Eval, flush } = await loadBraintrust();
const sendLogs = hasBraintrustApiKey();
// Aggressive abort: when the caller flips signal.reason to "aggressive",
// close every active session so any in-flight task throws on its next
@ -440,10 +445,13 @@ export async function runEvals(
{
progress: silentBraintrustProgress,
reporter: silentBraintrustReporter,
...(sendLogs ? {} : { noSendLogs: true }),
},
);
await flush();
if (sendLogs) {
await flush();
}
const summaryResults = evalResult.results.map((result) => {
const output =

View File

@ -21,16 +21,18 @@ import {
toJsonSchema,
} from "@browserbasehq/stagehand";
let wrappedAiPromise:
| Promise<ReturnType<(typeof import("braintrust"))["wrapAISDK"]>>
| undefined;
type WrappedAI = ReturnType<(typeof import("braintrust"))["wrapAISDK"]>;
async function loadWrappedAISDK(): Promise<
ReturnType<(typeof import("braintrust"))["wrapAISDK"]>
> {
wrappedAiPromise ??= import("braintrust").then(({ wrapAISDK }) =>
wrapAISDK(ai),
);
let wrappedAiPromise: Promise<WrappedAI> | undefined;
async function loadWrappedAISDK(): Promise<WrappedAI> {
wrappedAiPromise ??= (async () => {
if (!process.env.BRAINTRUST_API_KEY) {
return ai as unknown as WrappedAI;
}
const { wrapAISDK } = await import("braintrust");
return wrapAISDK(ai);
})();
return wrappedAiPromise;
}

View File

@ -0,0 +1,62 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { hasBraintrustApiKey, tracedSpan } from "../../framework/braintrust.js";
/**
* Verify that braintrust.ts helpers and AISdkClientWrapped work without
* BRAINTRUST_API_KEY. The runner-level tests live in a separate file
* (braintrust-runner-nolog.test.ts) because they need to vi.mock the
* braintrust module entirely.
*/
describe("braintrust.ts helpers without BRAINTRUST_API_KEY", () => {
const originalKey = process.env.BRAINTRUST_API_KEY;
beforeEach(() => {
delete process.env.BRAINTRUST_API_KEY;
});
afterEach(() => {
if (originalKey !== undefined) {
process.env.BRAINTRUST_API_KEY = originalKey;
} else {
delete process.env.BRAINTRUST_API_KEY;
}
});
it("hasBraintrustApiKey returns false when BRAINTRUST_API_KEY is unset", () => {
expect(hasBraintrustApiKey()).toBe(false);
});
it("hasBraintrustApiKey returns true when BRAINTRUST_API_KEY is set", () => {
process.env.BRAINTRUST_API_KEY = "test-key-123";
expect(hasBraintrustApiKey()).toBe(true);
});
it("tracedSpan calls fn directly without Braintrust when key is absent", async () => {
const fn = vi.fn(async () => 42);
const result = await tracedSpan(fn, { name: "test-span" });
expect(fn).toHaveBeenCalledOnce();
expect(result).toBe(42);
});
});
describe("AISdkClientWrapped without BRAINTRUST_API_KEY", () => {
const originalKey = process.env.BRAINTRUST_API_KEY;
beforeEach(() => {
delete process.env.BRAINTRUST_API_KEY;
});
afterEach(() => {
if (originalKey !== undefined) {
process.env.BRAINTRUST_API_KEY = originalKey;
} else {
delete process.env.BRAINTRUST_API_KEY;
}
});
it("AISdkClientWrapped class is importable without BRAINTRUST_API_KEY", async () => {
const mod = await import("../../lib/AISdkClientWrapped.js");
expect(mod.AISdkClientWrapped).toBeDefined();
});
});

View File

@ -0,0 +1,121 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
/**
* Verify that the runner passes noSendLogs to Braintrust Eval and skips
* flush() when BRAINTRUST_API_KEY is absent. This file lives separately
* because it needs to vi.mock the braintrust module at the top level.
*/
vi.mock("playwright", () => ({
chromium: {},
}));
const mockEval = vi.fn<
(
name: string,
options: unknown,
evalOptions?: unknown,
) => Promise<{
results: unknown[];
summary: { experimentName: string; scores: Record<string, unknown> };
}>
>(async () => ({
results: [],
summary: { experimentName: "test", scores: {} },
}));
const mockFlush = vi.fn(async () => {});
let mockHasKey = false;
vi.mock("../../framework/braintrust.js", () => ({
hasBraintrustApiKey: () => mockHasKey,
loadBraintrust: async () => ({
Eval: mockEval,
flush: mockFlush,
}),
tracedSpan: async <T>(fn: () => Promise<T>) => fn(),
}));
describe("runner.ts skips Braintrust logging when API key is absent", () => {
const originalKey = process.env.BRAINTRUST_API_KEY;
beforeEach(() => {
delete process.env.BRAINTRUST_API_KEY;
mockEval.mockClear();
mockFlush.mockClear();
mockHasKey = false;
});
afterEach(() => {
if (originalKey !== undefined) {
process.env.BRAINTRUST_API_KEY = originalKey;
} else {
delete process.env.BRAINTRUST_API_KEY;
}
});
it("passes noSendLogs: true to Eval when BRAINTRUST_API_KEY is unset", async () => {
mockHasKey = false;
const { runEvals } = await import("../../framework/runner.js");
const task = {
name: "test-task",
tier: "bench" as const,
primaryCategory: "extract",
categories: ["extract"],
tags: [] as string[],
filePath: "/fake.ts",
isLegacy: false,
};
await runEvals({
tasks: [task],
registry: {
tasks: [task],
byName: new Map([[task.name, task]]),
byTier: new Map([["bench", [task]]]),
byCategory: new Map([["extract", [task]]]),
},
trials: 1,
});
expect(mockEval).toHaveBeenCalledTimes(1);
const evalOptions = mockEval.mock.calls[0][2];
expect(evalOptions).toHaveProperty("noSendLogs", true);
// flush should NOT be called
expect(mockFlush).not.toHaveBeenCalled();
});
it("does NOT pass noSendLogs when BRAINTRUST_API_KEY is set", async () => {
mockHasKey = true;
const { runEvals } = await import("../../framework/runner.js");
const task = {
name: "test-task",
tier: "bench" as const,
primaryCategory: "extract",
categories: ["extract"],
tags: [] as string[],
filePath: "/fake.ts",
isLegacy: false,
};
await runEvals({
tasks: [task],
registry: {
tasks: [task],
byName: new Map([[task.name, task]]),
byTier: new Map([["bench", [task]]]),
byCategory: new Map([["extract", [task]]]),
},
trials: 1,
});
expect(mockEval).toHaveBeenCalledTimes(1);
const evalOptions = mockEval.mock.calls[0][2];
expect(evalOptions).not.toHaveProperty("noSendLogs");
// flush SHOULD be called
expect(mockFlush).toHaveBeenCalledTimes(1);
});
});

View File

@ -65,6 +65,9 @@ function makeRegistry(tasks: DiscoveredTask[]): TaskRegistry {
beforeEach(() => {
originalCi = process.env.CI;
delete process.env.CI;
// Set a dummy API key so the runner sends logs and calls flush(),
// which this test asserts.
process.env.BRAINTRUST_API_KEY = "test-key";
tracedNames.length = 0;
evalMock.mockReset();
flushMock.mockClear();
@ -79,6 +82,7 @@ afterEach(() => {
} else {
process.env.CI = originalCi;
}
delete process.env.BRAINTRUST_API_KEY;
while (tempDirs.length > 0) {
const dir = tempDirs.pop();