This repository has been archived on 2026-11-17. You can view files and clone it, but cannot push or open issues or pull requests.
gitgram-ai/aitimemachine/tests/training-export-e2e.test.ts
caelum0x 49b183818c Unify gitgram + aitimemachine into the monorepo (de-submodule) and merge vendored engines
Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as
opaque submodule gitlinks on GitHub) into fully-tracked first-party content.

AITM now merges its 7 vendored repos as first-party engines:
- memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts
- git: vendored C git (bundled binary) + gitnexus (in-process code intelligence)
- agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess)

Includes the full session's product build: Gitgram web/mobile/backend features,
AITM website/CLI/MCP, the unified memory layer, engine build tooling
(scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md),
and extensive test suites.

Build artifacts, node_modules, external/, and gemini-cli are gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 14:14:09 +03:00

384 lines
14 KiB
TypeScript

import * as fs from "fs";
import * as os from "os";
import * as path from "path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { PrivacyRedactor } from "../src/export/privacy-redactor.js";
import {
runRedactionBenchmark,
type RedactionBenchmarkSample,
} from "../src/export/redaction-benchmark.js";
import { saveConsent } from "../src/auth/consent-policy.js";
import { TrainingExporter } from "../src/export/training-exporter.js";
describe("training export pipeline end-to-end", () => {
let projectDir: string;
beforeEach(() => {
projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "aitm-export-e2e-"));
fs.mkdirSync(path.join(projectDir, ".aitimemachine"), { recursive: true });
// Grant full consent for training export
saveConsent(projectDir, {
policy: {
operational_capture: true,
training_export: true,
},
});
// Create a permissive license so compliance does not block
fs.writeFileSync(
path.join(projectDir, "package.json"),
JSON.stringify({ name: "test-project", version: "1.0.0", license: "MIT" }),
"utf-8"
);
});
afterEach(() => {
fs.rmSync(projectDir, { recursive: true, force: true });
});
it("exports training data with redaction and records metrics end-to-end", async () => {
const outputPath = path.join(projectDir, "training.jsonl");
const exporter = new TrainingExporter();
const result = await exporter.exportTrainingData({
projectPath: projectDir,
outputPath,
redact: true,
evaluateRedaction: true,
});
// Verify export succeeded
expect(result.success).toBe(true);
expect(fs.existsSync(outputPath)).toBe(true);
// Verify redaction quality metrics are present
expect(result.stats.redactionQuality).toBeDefined();
const rq = result.stats.redactionQuality!;
expect(rq.precision).toBeGreaterThanOrEqual(0);
expect(rq.precision).toBeLessThanOrEqual(1);
expect(rq.recall).toBeGreaterThanOrEqual(0);
expect(rq.recall).toBeLessThanOrEqual(1);
expect(rq.f1).toBeGreaterThanOrEqual(0);
expect(rq.f1).toBeLessThanOrEqual(1);
expect(rq.sampleCount).toBeGreaterThan(0);
// Verify the redaction benchmark report JSON was written
expect(rq.reportPath).toBeDefined();
expect(fs.existsSync(rq.reportPath!)).toBe(true);
const reportContent = JSON.parse(fs.readFileSync(rq.reportPath!, "utf-8"));
expect(reportContent.report.metrics.precision).toBe(rq.precision);
expect(reportContent.report.metrics.recall).toBe(rq.recall);
expect(reportContent.report.sampleCount).toBe(rq.sampleCount);
});
it("exports without redaction does not include quality metrics", async () => {
const outputPath = path.join(projectDir, "training-no-redact.jsonl");
const exporter = new TrainingExporter();
const result = await exporter.exportTrainingData({
projectPath: projectDir,
outputPath,
redact: false,
evaluateRedaction: false,
});
expect(result.success).toBe(true);
expect(result.stats.redactionQuality).toBeUndefined();
});
it("produces a valid trace with exportId and consent snapshot", async () => {
const outputPath = path.join(projectDir, "training-trace.jsonl");
const exporter = new TrainingExporter();
const result = await exporter.exportTrainingData({
projectPath: projectDir,
outputPath,
redact: true,
});
expect(result.trace.exportId).toMatch(/^exp_/);
expect(result.trace.consentSnapshotId).toMatch(/^consent_/);
expect(typeof result.trace.auditId).toBe("string");
});
});
describe("PrivacyRedactor edge cases", () => {
let redactor: PrivacyRedactor;
beforeEach(() => {
redactor = new PrivacyRedactor();
});
it("handles empty strings", () => {
expect(redactor.redact("")).toBe("");
expect(redactor.detect("")).toEqual([]);
});
it("handles text with no PII", () => {
const clean = "This is a perfectly clean string with no secrets.";
expect(redactor.redact(clean)).toBe(clean);
expect(redactor.detect(clean)).toEqual([]);
});
it("detects and redacts OpenAI API keys", () => {
const text = "export OPENAI_API_KEY=sk-abcdefghijklmnopqrstuvwxyz123456";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456");
expect(redacted).toContain("[REDACTED_API_KEY]");
const matches = redactor.detect(text);
expect(matches.length).toBeGreaterThanOrEqual(1);
expect(matches.some((m) => m.rule === "OpenAI API key")).toBe(true);
});
it("detects and redacts Anthropic API keys", () => {
const text = "key is sk-ant-api03-abcdefghijklmnopqrstuvwx";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("sk-ant-api03");
expect(redacted).toContain("[REDACTED_API_KEY]");
});
it("detects and redacts GitHub Personal Access Tokens", () => {
const text = "GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuvwxyz1234";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("ghp_1234567890abcdefghijklmnopqrstuvwxyz1234");
// The generic token= rule fires after the specific ghp_ rule, producing [REDACTED]
expect(redacted).toContain("[REDACTED]");
});
it("detects and redacts GitHub OAuth tokens", () => {
const text = "token: gho_abcdefghijklmnopqrstuvwxyz0123456789ABCD";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("gho_abcdefghijklmnopqrstuvwxyz0123456789ABCD");
// The generic token= rule fires after the specific gho_ rule, producing [REDACTED]
expect(redacted).toContain("[REDACTED]");
});
it("detects and redacts Slack tokens", () => {
const text = "SLACK_TOKEN=xoxb-1234567890-abcdef";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("xoxb-1234567890-abcdef");
// The generic token= rule fires after the specific xoxb- rule, producing [REDACTED]
expect(redacted).toContain("[REDACTED]");
});
it("detects and redacts AWS Access Keys", () => {
const text = "AWS_ACCESS_KEY_ID=AKIA1234567890ABCDEF";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("AKIA1234567890ABCDEF");
expect(redacted).toContain("[REDACTED_AWS_KEY]");
});
it("detects and redacts email addresses", () => {
const text = "Contact user@example.com for support";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("user@example.com");
expect(redacted).toContain("[REDACTED_EMAIL]");
});
it("detects and redacts IP addresses", () => {
const text = "Server at 192.168.1.100 is responding";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("192.168.1.100");
expect(redacted).toContain("[REDACTED_IP]");
});
it("detects and redacts credit card numbers", () => {
const text = "Card: 4111 1111 1111 1111 was charged";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("4111 1111 1111 1111");
expect(redacted).toContain("[REDACTED_CC]");
});
it("detects and redacts credit card numbers with dashes", () => {
const text = "CC number 4111-1111-1111-1111 end";
const redacted = redactor.redact(text);
expect(redacted).not.toContain("4111-1111-1111-1111");
expect(redacted).toContain("[REDACTED_CC]");
});
it("detects and redacts JWT tokens", () => {
const jwt =
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
const text = `Authorization: Bearer ${jwt}`;
const redacted = redactor.redact(text);
expect(redacted).not.toContain("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9");
const matches = redactor.detect(text);
expect(matches.some((m) => m.rule === "JWT Token")).toBe(true);
});
it("detects and redacts SSH private keys", () => {
const text = `Here is a key:
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn/ygWyF4PbnGcY5unA67hiQ==
-----END RSA PRIVATE KEY-----
done`;
const redacted = redactor.redact(text);
expect(redacted).not.toContain("MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn");
expect(redacted).toContain("[REDACTED_PRIVATE_KEY]");
const matches = redactor.detect(text);
expect(matches.some((m) => m.rule === "Private Key")).toBe(true);
});
it("detects and redacts OPENSSH private keys", () => {
const text = `-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAA==
-----END OPENSSH PRIVATE KEY-----`;
const redacted = redactor.redact(text);
expect(redacted).not.toContain("b3BlbnNzaC1rZXktdjEAAAAA");
expect(redacted).toContain("[REDACTED_PRIVATE_KEY]");
});
it("detects password assignments", () => {
const text = 'const dbPassword = "super_secret_123";';
const redacted = redactor.redact(text);
// password= assignment rule should fire
expect(redacted).toContain("[REDACTED]");
expect(redacted).not.toContain("super_secret_123");
});
it("detects multiple PII types in the same text", () => {
const text = `
Send email to admin@corp.com from 10.0.0.1.
API key: sk-1234567890abcdefghijklmnopqrstuv
Token: ghp_abcdefghijklmnopqrstuvwxyz0123456789ABCD
`;
const matches = redactor.detect(text);
const rules = new Set(matches.map((m) => m.rule));
expect(rules.has("Email")).toBe(true);
expect(rules.has("IP Address")).toBe(true);
expect(rules.has("OpenAI API key")).toBe(true);
expect(rules.has("GitHub PAT")).toBe(true);
const redacted = redactor.redact(text);
expect(redacted).not.toContain("admin@corp.com");
expect(redacted).not.toContain("10.0.0.1");
expect(redacted).not.toContain("sk-1234567890abcdefghijklmnopqrstuv");
expect(redacted).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz0123456789ABCD");
});
it("redactObject redacts sensitive keys regardless of value format", () => {
const obj = {
username: "alice",
password: "hunter2",
apiKey: "sk-test123456789012345678901234",
data: { token: "abc123", nested: { secret: "xyz" } },
};
const redacted = redactor.redactObject(obj);
expect(redacted.password).toBe("[REDACTED]");
expect(redacted.apiKey).toBe("[REDACTED]");
expect(redacted.data.token).toBe("[REDACTED]");
expect(redacted.data.nested.secret).toBe("[REDACTED]");
// Non-sensitive keys preserved
expect(redacted.username).toBe("alice");
});
it("redactObject handles null and undefined gracefully", () => {
expect(redactor.redactObject(null)).toBeNull();
expect(redactor.redactObject(undefined)).toBeUndefined();
});
it("redactObject handles arrays", () => {
const arr = ["user@example.com", "clean text", "sk-abcdefghijklmnopqrstuvwxyz123456"];
const redacted = redactor.redactObject(arr);
expect(redacted[0]).toContain("[REDACTED_EMAIL]");
expect(redacted[1]).toBe("clean text");
expect(redacted[2]).toContain("[REDACTED_API_KEY]");
});
it("addPattern allows custom patterns to be added", () => {
redactor.addPattern(/CUSTOM_SECRET_[A-Z]+/g, "[CUSTOM_REDACTED]", "Custom Secret");
const text = "value is CUSTOM_SECRET_FOOBAR end";
const redacted = redactor.redact(text);
expect(redacted).toContain("[CUSTOM_REDACTED]");
expect(redacted).not.toContain("CUSTOM_SECRET_FOOBAR");
});
it("getRedactionStats returns counts per rule", () => {
const text = "user@a.com and admin@b.com and 10.0.0.1";
const stats = redactor.getRedactionStats(text);
const emailStat = stats.find((s) => s.rule === "Email");
expect(emailStat).toBeDefined();
expect(emailStat!.count).toBe(2);
const ipStat = stats.find((s) => s.rule === "IP Address");
expect(ipStat).toBeDefined();
expect(ipStat!.count).toBe(1);
});
});
describe("redaction benchmark with custom samples", () => {
it("handles empty expected secrets (control sample)", () => {
const redactor = new PrivacyRedactor();
const samples: RedactionBenchmarkSample[] = [
{
id: "no-secrets",
text: "Just some regular code with variables and functions",
expectedSecrets: [],
},
];
const report = runRedactionBenchmark(redactor, samples);
expect(report.sampleCount).toBe(1);
// No true positives, no false negatives expected
expect(report.metrics.falseNegatives).toBe(0);
});
it("correctly measures false negatives for unrecognized patterns", () => {
const redactor = new PrivacyRedactor();
const samples: RedactionBenchmarkSample[] = [
{
id: "custom-secret",
text: "MyCustomSecret=XYZ_ULTRA_SECRET_VALUE_42",
expectedSecrets: ["XYZ_ULTRA_SECRET_VALUE_42"],
},
];
const report = runRedactionBenchmark(redactor, samples);
// The redactor does not have a rule for this made-up format
expect(report.metrics.falseNegatives).toBeGreaterThanOrEqual(1);
expect(report.metrics.recall).toBeLessThan(1);
});
it("correctly measures true positives for known patterns", () => {
const redactor = new PrivacyRedactor();
const samples: RedactionBenchmarkSample[] = [
{
id: "known-email",
text: "Contact support@example.com",
expectedSecrets: ["support@example.com"],
},
];
const report = runRedactionBenchmark(redactor, samples);
expect(report.metrics.truePositives).toBe(1);
expect(report.metrics.falseNegatives).toBe(0);
expect(report.metrics.recall).toBe(1);
});
it("handles samples with mixed hits and misses", () => {
const redactor = new PrivacyRedactor();
const samples: RedactionBenchmarkSample[] = [
{
id: "mixed",
text: "email: admin@test.org and custom_value=NOTDETECTABLE",
expectedSecrets: ["admin@test.org", "NOTDETECTABLE"],
},
];
const report = runRedactionBenchmark(redactor, samples);
// Email should be detected (TP=1), NOTDETECTABLE should be missed (FN=1)
expect(report.metrics.truePositives).toBeGreaterThanOrEqual(1);
expect(report.metrics.falseNegatives).toBeGreaterThanOrEqual(1);
expect(report.metrics.precision).toBeGreaterThan(0);
expect(report.metrics.recall).toBeLessThan(1);
});
it("evaluate returns zero metrics for empty samples array", () => {
const redactor = new PrivacyRedactor();
const metrics = redactor.evaluate([]);
expect(metrics.truePositives).toBe(0);
expect(metrics.falsePositives).toBe(0);
expect(metrics.falseNegatives).toBe(0);
expect(metrics.precision).toBe(0);
expect(metrics.recall).toBe(0);
expect(metrics.f1).toBe(0);
});
});