Compare commits
2 Commits
main
...
contrib/21
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
482e49bac6 | ||
|
|
0f85f972a2 |
@ -1,5 +1,9 @@
|
||||
let braintrustPromise: Promise<typeof import("braintrust")> | undefined;
|
||||
|
||||
export function hasBraintrustApiKey(): boolean {
|
||||
return Boolean(process.env.BRAINTRUST_API_KEY);
|
||||
}
|
||||
|
||||
export function loadBraintrust(): Promise<typeof import("braintrust")> {
|
||||
braintrustPromise ??= import("braintrust");
|
||||
return braintrustPromise;
|
||||
@ -9,6 +13,9 @@ export async function tracedSpan<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: { name: string },
|
||||
): Promise<T> {
|
||||
if (!hasBraintrustApiKey()) {
|
||||
return fn();
|
||||
}
|
||||
const { traced } = await loadBraintrust();
|
||||
return traced(fn, options);
|
||||
}
|
||||
|
||||
@ -22,7 +22,11 @@ import type { Testcase, EvalInput } from "../types/evals.js";
|
||||
import { generateBenchTestcases } from "./benchPlanner.js";
|
||||
import { DEFAULT_BENCH_HARNESS, type Harness } from "./benchTypes.js";
|
||||
import { executeBenchTask } from "./benchRunner.js";
|
||||
import { loadBraintrust, tracedSpan } from "./braintrust.js";
|
||||
import {
|
||||
hasBraintrustApiKey,
|
||||
loadBraintrust,
|
||||
tracedSpan,
|
||||
} from "./braintrust.js";
|
||||
import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js";
|
||||
import { loadTaskModuleFromPath } from "./taskLoader.js";
|
||||
|
||||
@ -352,6 +356,7 @@ export async function runEvals(
|
||||
: [exactMatch, errorMatch];
|
||||
|
||||
const { Eval, flush } = await loadBraintrust();
|
||||
const sendLogs = hasBraintrustApiKey();
|
||||
|
||||
// Aggressive abort: when the caller flips signal.reason to "aggressive",
|
||||
// close every active session so any in-flight task throws on its next
|
||||
@ -440,10 +445,13 @@ export async function runEvals(
|
||||
{
|
||||
progress: silentBraintrustProgress,
|
||||
reporter: silentBraintrustReporter,
|
||||
...(sendLogs ? {} : { noSendLogs: true }),
|
||||
},
|
||||
);
|
||||
|
||||
await flush();
|
||||
if (sendLogs) {
|
||||
await flush();
|
||||
}
|
||||
|
||||
const summaryResults = evalResult.results.map((result) => {
|
||||
const output =
|
||||
|
||||
@ -21,16 +21,18 @@ import {
|
||||
toJsonSchema,
|
||||
} from "@browserbasehq/stagehand";
|
||||
|
||||
let wrappedAiPromise:
|
||||
| Promise<ReturnType<(typeof import("braintrust"))["wrapAISDK"]>>
|
||||
| undefined;
|
||||
type WrappedAI = ReturnType<(typeof import("braintrust"))["wrapAISDK"]>;
|
||||
|
||||
async function loadWrappedAISDK(): Promise<
|
||||
ReturnType<(typeof import("braintrust"))["wrapAISDK"]>
|
||||
> {
|
||||
wrappedAiPromise ??= import("braintrust").then(({ wrapAISDK }) =>
|
||||
wrapAISDK(ai),
|
||||
);
|
||||
let wrappedAiPromise: Promise<WrappedAI> | undefined;
|
||||
|
||||
async function loadWrappedAISDK(): Promise<WrappedAI> {
|
||||
wrappedAiPromise ??= (async () => {
|
||||
if (!process.env.BRAINTRUST_API_KEY) {
|
||||
return ai as unknown as WrappedAI;
|
||||
}
|
||||
const { wrapAISDK } = await import("braintrust");
|
||||
return wrapAISDK(ai);
|
||||
})();
|
||||
return wrappedAiPromise;
|
||||
}
|
||||
|
||||
|
||||
62
packages/evals/tests/framework/braintrust-optional.test.ts
Normal file
62
packages/evals/tests/framework/braintrust-optional.test.ts
Normal file
@ -0,0 +1,62 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { hasBraintrustApiKey, tracedSpan } from "../../framework/braintrust.js";
|
||||
|
||||
/**
|
||||
* Verify that braintrust.ts helpers and AISdkClientWrapped work without
|
||||
* BRAINTRUST_API_KEY. The runner-level tests live in a separate file
|
||||
* (braintrust-runner-nolog.test.ts) because they need to vi.mock the
|
||||
* braintrust module entirely.
|
||||
*/
|
||||
|
||||
describe("braintrust.ts helpers without BRAINTRUST_API_KEY", () => {
|
||||
const originalKey = process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalKey !== undefined) {
|
||||
process.env.BRAINTRUST_API_KEY = originalKey;
|
||||
} else {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
}
|
||||
});
|
||||
|
||||
it("hasBraintrustApiKey returns false when BRAINTRUST_API_KEY is unset", () => {
|
||||
expect(hasBraintrustApiKey()).toBe(false);
|
||||
});
|
||||
|
||||
it("hasBraintrustApiKey returns true when BRAINTRUST_API_KEY is set", () => {
|
||||
process.env.BRAINTRUST_API_KEY = "test-key-123";
|
||||
expect(hasBraintrustApiKey()).toBe(true);
|
||||
});
|
||||
|
||||
it("tracedSpan calls fn directly without Braintrust when key is absent", async () => {
|
||||
const fn = vi.fn(async () => 42);
|
||||
const result = await tracedSpan(fn, { name: "test-span" });
|
||||
expect(fn).toHaveBeenCalledOnce();
|
||||
expect(result).toBe(42);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AISdkClientWrapped without BRAINTRUST_API_KEY", () => {
|
||||
const originalKey = process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalKey !== undefined) {
|
||||
process.env.BRAINTRUST_API_KEY = originalKey;
|
||||
} else {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
}
|
||||
});
|
||||
|
||||
it("AISdkClientWrapped class is importable without BRAINTRUST_API_KEY", async () => {
|
||||
const mod = await import("../../lib/AISdkClientWrapped.js");
|
||||
expect(mod.AISdkClientWrapped).toBeDefined();
|
||||
});
|
||||
});
|
||||
121
packages/evals/tests/framework/braintrust-runner-nolog.test.ts
Normal file
121
packages/evals/tests/framework/braintrust-runner-nolog.test.ts
Normal file
@ -0,0 +1,121 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
|
||||
/**
|
||||
* Verify that the runner passes noSendLogs to Braintrust Eval and skips
|
||||
* flush() when BRAINTRUST_API_KEY is absent. This file lives separately
|
||||
* because it needs to vi.mock the braintrust module at the top level.
|
||||
*/
|
||||
|
||||
vi.mock("playwright", () => ({
|
||||
chromium: {},
|
||||
}));
|
||||
|
||||
const mockEval = vi.fn<
|
||||
(
|
||||
name: string,
|
||||
options: unknown,
|
||||
evalOptions?: unknown,
|
||||
) => Promise<{
|
||||
results: unknown[];
|
||||
summary: { experimentName: string; scores: Record<string, unknown> };
|
||||
}>
|
||||
>(async () => ({
|
||||
results: [],
|
||||
summary: { experimentName: "test", scores: {} },
|
||||
}));
|
||||
const mockFlush = vi.fn(async () => {});
|
||||
let mockHasKey = false;
|
||||
|
||||
vi.mock("../../framework/braintrust.js", () => ({
|
||||
hasBraintrustApiKey: () => mockHasKey,
|
||||
loadBraintrust: async () => ({
|
||||
Eval: mockEval,
|
||||
flush: mockFlush,
|
||||
}),
|
||||
tracedSpan: async <T>(fn: () => Promise<T>) => fn(),
|
||||
}));
|
||||
|
||||
describe("runner.ts skips Braintrust logging when API key is absent", () => {
|
||||
const originalKey = process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
mockEval.mockClear();
|
||||
mockFlush.mockClear();
|
||||
mockHasKey = false;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalKey !== undefined) {
|
||||
process.env.BRAINTRUST_API_KEY = originalKey;
|
||||
} else {
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
}
|
||||
});
|
||||
|
||||
it("passes noSendLogs: true to Eval when BRAINTRUST_API_KEY is unset", async () => {
|
||||
mockHasKey = false;
|
||||
const { runEvals } = await import("../../framework/runner.js");
|
||||
|
||||
const task = {
|
||||
name: "test-task",
|
||||
tier: "bench" as const,
|
||||
primaryCategory: "extract",
|
||||
categories: ["extract"],
|
||||
tags: [] as string[],
|
||||
filePath: "/fake.ts",
|
||||
isLegacy: false,
|
||||
};
|
||||
|
||||
await runEvals({
|
||||
tasks: [task],
|
||||
registry: {
|
||||
tasks: [task],
|
||||
byName: new Map([[task.name, task]]),
|
||||
byTier: new Map([["bench", [task]]]),
|
||||
byCategory: new Map([["extract", [task]]]),
|
||||
},
|
||||
trials: 1,
|
||||
});
|
||||
|
||||
expect(mockEval).toHaveBeenCalledTimes(1);
|
||||
const evalOptions = mockEval.mock.calls[0][2];
|
||||
expect(evalOptions).toHaveProperty("noSendLogs", true);
|
||||
|
||||
// flush should NOT be called
|
||||
expect(mockFlush).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does NOT pass noSendLogs when BRAINTRUST_API_KEY is set", async () => {
|
||||
mockHasKey = true;
|
||||
const { runEvals } = await import("../../framework/runner.js");
|
||||
|
||||
const task = {
|
||||
name: "test-task",
|
||||
tier: "bench" as const,
|
||||
primaryCategory: "extract",
|
||||
categories: ["extract"],
|
||||
tags: [] as string[],
|
||||
filePath: "/fake.ts",
|
||||
isLegacy: false,
|
||||
};
|
||||
|
||||
await runEvals({
|
||||
tasks: [task],
|
||||
registry: {
|
||||
tasks: [task],
|
||||
byName: new Map([[task.name, task]]),
|
||||
byTier: new Map([["bench", [task]]]),
|
||||
byCategory: new Map([["extract", [task]]]),
|
||||
},
|
||||
trials: 1,
|
||||
});
|
||||
|
||||
expect(mockEval).toHaveBeenCalledTimes(1);
|
||||
const evalOptions = mockEval.mock.calls[0][2];
|
||||
expect(evalOptions).not.toHaveProperty("noSendLogs");
|
||||
|
||||
// flush SHOULD be called
|
||||
expect(mockFlush).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@ -65,6 +65,9 @@ function makeRegistry(tasks: DiscoveredTask[]): TaskRegistry {
|
||||
beforeEach(() => {
|
||||
originalCi = process.env.CI;
|
||||
delete process.env.CI;
|
||||
// Set a dummy API key so the runner sends logs and calls flush(),
|
||||
// which this test asserts.
|
||||
process.env.BRAINTRUST_API_KEY = "test-key";
|
||||
tracedNames.length = 0;
|
||||
evalMock.mockReset();
|
||||
flushMock.mockClear();
|
||||
@ -79,6 +82,7 @@ afterEach(() => {
|
||||
} else {
|
||||
process.env.CI = originalCi;
|
||||
}
|
||||
delete process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
while (tempDirs.length > 0) {
|
||||
const dir = tempDirs.pop();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user