Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as opaque submodule gitlinks on GitHub) into fully-tracked first-party content. AITM now merges its 7 vendored repos as first-party engines: - memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts - git: vendored C git (bundled binary) + gitnexus (in-process code intelligence) - agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess) Includes the full session's product build: Gitgram web/mobile/backend features, AITM website/CLI/MCP, the unified memory layer, engine build tooling (scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md), and extensive test suites. Build artifacts, node_modules, external/, and gemini-cli are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
384 lines
14 KiB
TypeScript
384 lines
14 KiB
TypeScript
import * as fs from "fs";
|
|
import * as os from "os";
|
|
import * as path from "path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import { PrivacyRedactor } from "../src/export/privacy-redactor.js";
|
|
import {
|
|
runRedactionBenchmark,
|
|
type RedactionBenchmarkSample,
|
|
} from "../src/export/redaction-benchmark.js";
|
|
import { saveConsent } from "../src/auth/consent-policy.js";
|
|
import { TrainingExporter } from "../src/export/training-exporter.js";
|
|
|
|
describe("training export pipeline end-to-end", () => {
|
|
let projectDir: string;
|
|
|
|
beforeEach(() => {
|
|
projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "aitm-export-e2e-"));
|
|
fs.mkdirSync(path.join(projectDir, ".aitimemachine"), { recursive: true });
|
|
// Grant full consent for training export
|
|
saveConsent(projectDir, {
|
|
policy: {
|
|
operational_capture: true,
|
|
training_export: true,
|
|
},
|
|
});
|
|
// Create a permissive license so compliance does not block
|
|
fs.writeFileSync(
|
|
path.join(projectDir, "package.json"),
|
|
JSON.stringify({ name: "test-project", version: "1.0.0", license: "MIT" }),
|
|
"utf-8"
|
|
);
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("exports training data with redaction and records metrics end-to-end", async () => {
|
|
const outputPath = path.join(projectDir, "training.jsonl");
|
|
const exporter = new TrainingExporter();
|
|
|
|
const result = await exporter.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath,
|
|
redact: true,
|
|
evaluateRedaction: true,
|
|
});
|
|
|
|
// Verify export succeeded
|
|
expect(result.success).toBe(true);
|
|
expect(fs.existsSync(outputPath)).toBe(true);
|
|
|
|
// Verify redaction quality metrics are present
|
|
expect(result.stats.redactionQuality).toBeDefined();
|
|
const rq = result.stats.redactionQuality!;
|
|
expect(rq.precision).toBeGreaterThanOrEqual(0);
|
|
expect(rq.precision).toBeLessThanOrEqual(1);
|
|
expect(rq.recall).toBeGreaterThanOrEqual(0);
|
|
expect(rq.recall).toBeLessThanOrEqual(1);
|
|
expect(rq.f1).toBeGreaterThanOrEqual(0);
|
|
expect(rq.f1).toBeLessThanOrEqual(1);
|
|
expect(rq.sampleCount).toBeGreaterThan(0);
|
|
|
|
// Verify the redaction benchmark report JSON was written
|
|
expect(rq.reportPath).toBeDefined();
|
|
expect(fs.existsSync(rq.reportPath!)).toBe(true);
|
|
const reportContent = JSON.parse(fs.readFileSync(rq.reportPath!, "utf-8"));
|
|
expect(reportContent.report.metrics.precision).toBe(rq.precision);
|
|
expect(reportContent.report.metrics.recall).toBe(rq.recall);
|
|
expect(reportContent.report.sampleCount).toBe(rq.sampleCount);
|
|
});
|
|
|
|
it("exports without redaction does not include quality metrics", async () => {
|
|
const outputPath = path.join(projectDir, "training-no-redact.jsonl");
|
|
const exporter = new TrainingExporter();
|
|
|
|
const result = await exporter.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath,
|
|
redact: false,
|
|
evaluateRedaction: false,
|
|
});
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.stats.redactionQuality).toBeUndefined();
|
|
});
|
|
|
|
it("produces a valid trace with exportId and consent snapshot", async () => {
|
|
const outputPath = path.join(projectDir, "training-trace.jsonl");
|
|
const exporter = new TrainingExporter();
|
|
|
|
const result = await exporter.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath,
|
|
redact: true,
|
|
});
|
|
|
|
expect(result.trace.exportId).toMatch(/^exp_/);
|
|
expect(result.trace.consentSnapshotId).toMatch(/^consent_/);
|
|
expect(typeof result.trace.auditId).toBe("string");
|
|
});
|
|
});
|
|
|
|
describe("PrivacyRedactor edge cases", () => {
|
|
let redactor: PrivacyRedactor;
|
|
|
|
beforeEach(() => {
|
|
redactor = new PrivacyRedactor();
|
|
});
|
|
|
|
it("handles empty strings", () => {
|
|
expect(redactor.redact("")).toBe("");
|
|
expect(redactor.detect("")).toEqual([]);
|
|
});
|
|
|
|
it("handles text with no PII", () => {
|
|
const clean = "This is a perfectly clean string with no secrets.";
|
|
expect(redactor.redact(clean)).toBe(clean);
|
|
expect(redactor.detect(clean)).toEqual([]);
|
|
});
|
|
|
|
it("detects and redacts OpenAI API keys", () => {
|
|
const text = "export OPENAI_API_KEY=sk-abcdefghijklmnopqrstuvwxyz123456";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("sk-abcdefghijklmnopqrstuvwxyz123456");
|
|
expect(redacted).toContain("[REDACTED_API_KEY]");
|
|
|
|
const matches = redactor.detect(text);
|
|
expect(matches.length).toBeGreaterThanOrEqual(1);
|
|
expect(matches.some((m) => m.rule === "OpenAI API key")).toBe(true);
|
|
});
|
|
|
|
it("detects and redacts Anthropic API keys", () => {
|
|
const text = "key is sk-ant-api03-abcdefghijklmnopqrstuvwx";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("sk-ant-api03");
|
|
expect(redacted).toContain("[REDACTED_API_KEY]");
|
|
});
|
|
|
|
it("detects and redacts GitHub Personal Access Tokens", () => {
|
|
const text = "GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuvwxyz1234";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("ghp_1234567890abcdefghijklmnopqrstuvwxyz1234");
|
|
// The generic token= rule fires after the specific ghp_ rule, producing [REDACTED]
|
|
expect(redacted).toContain("[REDACTED]");
|
|
});
|
|
|
|
it("detects and redacts GitHub OAuth tokens", () => {
|
|
const text = "token: gho_abcdefghijklmnopqrstuvwxyz0123456789ABCD";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("gho_abcdefghijklmnopqrstuvwxyz0123456789ABCD");
|
|
// The generic token= rule fires after the specific gho_ rule, producing [REDACTED]
|
|
expect(redacted).toContain("[REDACTED]");
|
|
});
|
|
|
|
it("detects and redacts Slack tokens", () => {
|
|
const text = "SLACK_TOKEN=xoxb-1234567890-abcdef";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("xoxb-1234567890-abcdef");
|
|
// The generic token= rule fires after the specific xoxb- rule, producing [REDACTED]
|
|
expect(redacted).toContain("[REDACTED]");
|
|
});
|
|
|
|
it("detects and redacts AWS Access Keys", () => {
|
|
const text = "AWS_ACCESS_KEY_ID=AKIA1234567890ABCDEF";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("AKIA1234567890ABCDEF");
|
|
expect(redacted).toContain("[REDACTED_AWS_KEY]");
|
|
});
|
|
|
|
it("detects and redacts email addresses", () => {
|
|
const text = "Contact user@example.com for support";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("user@example.com");
|
|
expect(redacted).toContain("[REDACTED_EMAIL]");
|
|
});
|
|
|
|
it("detects and redacts IP addresses", () => {
|
|
const text = "Server at 192.168.1.100 is responding";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("192.168.1.100");
|
|
expect(redacted).toContain("[REDACTED_IP]");
|
|
});
|
|
|
|
it("detects and redacts credit card numbers", () => {
|
|
const text = "Card: 4111 1111 1111 1111 was charged";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("4111 1111 1111 1111");
|
|
expect(redacted).toContain("[REDACTED_CC]");
|
|
});
|
|
|
|
it("detects and redacts credit card numbers with dashes", () => {
|
|
const text = "CC number 4111-1111-1111-1111 end";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("4111-1111-1111-1111");
|
|
expect(redacted).toContain("[REDACTED_CC]");
|
|
});
|
|
|
|
it("detects and redacts JWT tokens", () => {
|
|
const jwt =
|
|
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
|
|
const text = `Authorization: Bearer ${jwt}`;
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9");
|
|
|
|
const matches = redactor.detect(text);
|
|
expect(matches.some((m) => m.rule === "JWT Token")).toBe(true);
|
|
});
|
|
|
|
it("detects and redacts SSH private keys", () => {
|
|
const text = `Here is a key:
|
|
-----BEGIN RSA PRIVATE KEY-----
|
|
MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn/ygWyF4PbnGcY5unA67hiQ==
|
|
-----END RSA PRIVATE KEY-----
|
|
done`;
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn");
|
|
expect(redacted).toContain("[REDACTED_PRIVATE_KEY]");
|
|
|
|
const matches = redactor.detect(text);
|
|
expect(matches.some((m) => m.rule === "Private Key")).toBe(true);
|
|
});
|
|
|
|
it("detects and redacts OPENSSH private keys", () => {
|
|
const text = `-----BEGIN OPENSSH PRIVATE KEY-----
|
|
b3BlbnNzaC1rZXktdjEAAAAA==
|
|
-----END OPENSSH PRIVATE KEY-----`;
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("b3BlbnNzaC1rZXktdjEAAAAA");
|
|
expect(redacted).toContain("[REDACTED_PRIVATE_KEY]");
|
|
});
|
|
|
|
it("detects password assignments", () => {
|
|
const text = 'const dbPassword = "super_secret_123";';
|
|
const redacted = redactor.redact(text);
|
|
// password= assignment rule should fire
|
|
expect(redacted).toContain("[REDACTED]");
|
|
expect(redacted).not.toContain("super_secret_123");
|
|
});
|
|
|
|
it("detects multiple PII types in the same text", () => {
|
|
const text = `
|
|
Send email to admin@corp.com from 10.0.0.1.
|
|
API key: sk-1234567890abcdefghijklmnopqrstuv
|
|
Token: ghp_abcdefghijklmnopqrstuvwxyz0123456789ABCD
|
|
`;
|
|
const matches = redactor.detect(text);
|
|
const rules = new Set(matches.map((m) => m.rule));
|
|
expect(rules.has("Email")).toBe(true);
|
|
expect(rules.has("IP Address")).toBe(true);
|
|
expect(rules.has("OpenAI API key")).toBe(true);
|
|
expect(rules.has("GitHub PAT")).toBe(true);
|
|
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).not.toContain("admin@corp.com");
|
|
expect(redacted).not.toContain("10.0.0.1");
|
|
expect(redacted).not.toContain("sk-1234567890abcdefghijklmnopqrstuv");
|
|
expect(redacted).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz0123456789ABCD");
|
|
});
|
|
|
|
it("redactObject redacts sensitive keys regardless of value format", () => {
|
|
const obj = {
|
|
username: "alice",
|
|
password: "hunter2",
|
|
apiKey: "sk-test123456789012345678901234",
|
|
data: { token: "abc123", nested: { secret: "xyz" } },
|
|
};
|
|
const redacted = redactor.redactObject(obj);
|
|
expect(redacted.password).toBe("[REDACTED]");
|
|
expect(redacted.apiKey).toBe("[REDACTED]");
|
|
expect(redacted.data.token).toBe("[REDACTED]");
|
|
expect(redacted.data.nested.secret).toBe("[REDACTED]");
|
|
// Non-sensitive keys preserved
|
|
expect(redacted.username).toBe("alice");
|
|
});
|
|
|
|
it("redactObject handles null and undefined gracefully", () => {
|
|
expect(redactor.redactObject(null)).toBeNull();
|
|
expect(redactor.redactObject(undefined)).toBeUndefined();
|
|
});
|
|
|
|
it("redactObject handles arrays", () => {
|
|
const arr = ["user@example.com", "clean text", "sk-abcdefghijklmnopqrstuvwxyz123456"];
|
|
const redacted = redactor.redactObject(arr);
|
|
expect(redacted[0]).toContain("[REDACTED_EMAIL]");
|
|
expect(redacted[1]).toBe("clean text");
|
|
expect(redacted[2]).toContain("[REDACTED_API_KEY]");
|
|
});
|
|
|
|
it("addPattern allows custom patterns to be added", () => {
|
|
redactor.addPattern(/CUSTOM_SECRET_[A-Z]+/g, "[CUSTOM_REDACTED]", "Custom Secret");
|
|
const text = "value is CUSTOM_SECRET_FOOBAR end";
|
|
const redacted = redactor.redact(text);
|
|
expect(redacted).toContain("[CUSTOM_REDACTED]");
|
|
expect(redacted).not.toContain("CUSTOM_SECRET_FOOBAR");
|
|
});
|
|
|
|
it("getRedactionStats returns counts per rule", () => {
|
|
const text = "user@a.com and admin@b.com and 10.0.0.1";
|
|
const stats = redactor.getRedactionStats(text);
|
|
const emailStat = stats.find((s) => s.rule === "Email");
|
|
expect(emailStat).toBeDefined();
|
|
expect(emailStat!.count).toBe(2);
|
|
const ipStat = stats.find((s) => s.rule === "IP Address");
|
|
expect(ipStat).toBeDefined();
|
|
expect(ipStat!.count).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("redaction benchmark with custom samples", () => {
|
|
it("handles empty expected secrets (control sample)", () => {
|
|
const redactor = new PrivacyRedactor();
|
|
const samples: RedactionBenchmarkSample[] = [
|
|
{
|
|
id: "no-secrets",
|
|
text: "Just some regular code with variables and functions",
|
|
expectedSecrets: [],
|
|
},
|
|
];
|
|
const report = runRedactionBenchmark(redactor, samples);
|
|
expect(report.sampleCount).toBe(1);
|
|
// No true positives, no false negatives expected
|
|
expect(report.metrics.falseNegatives).toBe(0);
|
|
});
|
|
|
|
it("correctly measures false negatives for unrecognized patterns", () => {
|
|
const redactor = new PrivacyRedactor();
|
|
const samples: RedactionBenchmarkSample[] = [
|
|
{
|
|
id: "custom-secret",
|
|
text: "MyCustomSecret=XYZ_ULTRA_SECRET_VALUE_42",
|
|
expectedSecrets: ["XYZ_ULTRA_SECRET_VALUE_42"],
|
|
},
|
|
];
|
|
const report = runRedactionBenchmark(redactor, samples);
|
|
// The redactor does not have a rule for this made-up format
|
|
expect(report.metrics.falseNegatives).toBeGreaterThanOrEqual(1);
|
|
expect(report.metrics.recall).toBeLessThan(1);
|
|
});
|
|
|
|
it("correctly measures true positives for known patterns", () => {
|
|
const redactor = new PrivacyRedactor();
|
|
const samples: RedactionBenchmarkSample[] = [
|
|
{
|
|
id: "known-email",
|
|
text: "Contact support@example.com",
|
|
expectedSecrets: ["support@example.com"],
|
|
},
|
|
];
|
|
const report = runRedactionBenchmark(redactor, samples);
|
|
expect(report.metrics.truePositives).toBe(1);
|
|
expect(report.metrics.falseNegatives).toBe(0);
|
|
expect(report.metrics.recall).toBe(1);
|
|
});
|
|
|
|
it("handles samples with mixed hits and misses", () => {
|
|
const redactor = new PrivacyRedactor();
|
|
const samples: RedactionBenchmarkSample[] = [
|
|
{
|
|
id: "mixed",
|
|
text: "email: admin@test.org and custom_value=NOTDETECTABLE",
|
|
expectedSecrets: ["admin@test.org", "NOTDETECTABLE"],
|
|
},
|
|
];
|
|
const report = runRedactionBenchmark(redactor, samples);
|
|
// Email should be detected (TP=1), NOTDETECTABLE should be missed (FN=1)
|
|
expect(report.metrics.truePositives).toBeGreaterThanOrEqual(1);
|
|
expect(report.metrics.falseNegatives).toBeGreaterThanOrEqual(1);
|
|
expect(report.metrics.precision).toBeGreaterThan(0);
|
|
expect(report.metrics.recall).toBeLessThan(1);
|
|
});
|
|
|
|
it("evaluate returns zero metrics for empty samples array", () => {
|
|
const redactor = new PrivacyRedactor();
|
|
const metrics = redactor.evaluate([]);
|
|
expect(metrics.truePositives).toBe(0);
|
|
expect(metrics.falsePositives).toBe(0);
|
|
expect(metrics.falseNegatives).toBe(0);
|
|
expect(metrics.precision).toBe(0);
|
|
expect(metrics.recall).toBe(0);
|
|
expect(metrics.f1).toBe(0);
|
|
});
|
|
});
|