Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as opaque submodule gitlinks on GitHub) into fully-tracked first-party content. AITM now merges its 7 vendored repos as first-party engines: - memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts - git: vendored C git (bundled binary) + gitnexus (in-process code intelligence) - agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess) Includes the full session's product build: Gitgram web/mobile/backend features, AITM website/CLI/MCP, the unified memory layer, engine build tooling (scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md), and extensive test suites. Build artifacts, node_modules, external/, and gemini-cli are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
392 lines
13 KiB
TypeScript
392 lines
13 KiB
TypeScript
/**
|
|
* #23 - Export traceability guarantees: comprehensive validation
|
|
*
|
|
* Verifies 100% of exports produce valid traceability linkage
|
|
* (no null/empty consent/audit IDs) and that audit log entries are
|
|
* created for every export operation (start + completion).
|
|
*/
|
|
import * as fs from "fs";
|
|
import * as os from "os";
|
|
import * as path from "path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import { saveConsent } from "../src/auth/consent-policy.js";
|
|
import { getAuditLogs } from "../src/auth/audit.js";
|
|
import {
|
|
beginExportTrace,
|
|
completeExportTrace,
|
|
type ExportTrace,
|
|
} from "../src/export/traceability.js";
|
|
import { TrainingExporter } from "../src/export/training-exporter.js";
|
|
import { DataExporter } from "../src/export/data-exporter.js";
|
|
import { ChangesExporter } from "../src/export/changes-exporter.js";
|
|
import { SessionExporter } from "../src/export/session-exporter.js";
|
|
import {
|
|
exportAnalytics,
|
|
exportTrainingData as exportTrainingDataHelper,
|
|
} from "../src/export/index.js";
|
|
|
|
/**
|
|
* Validates that an ExportTrace has fully populated, non-null linkage fields.
|
|
*/
|
|
function assertValidTrace(trace: ExportTrace, label: string): void {
|
|
expect(trace.exportId, `${label}: exportId must be a non-empty string`).toBeTruthy();
|
|
expect(
|
|
trace.exportId.startsWith("exp_"),
|
|
`${label}: exportId must start with 'exp_'`
|
|
).toBe(true);
|
|
|
|
expect(
|
|
trace.consentSnapshotId,
|
|
`${label}: consentSnapshotId must be a non-empty string`
|
|
).toBeTruthy();
|
|
expect(
|
|
trace.consentSnapshotId.startsWith("consent_"),
|
|
`${label}: consentSnapshotId must start with 'consent_'`
|
|
).toBe(true);
|
|
|
|
expect(trace.auditId, `${label}: auditId must be a non-empty string`).toBeTruthy();
|
|
expect(
|
|
typeof trace.auditId === "string" && trace.auditId.length > 0,
|
|
`${label}: auditId must be a valid UUID string`
|
|
).toBe(true);
|
|
}
|
|
|
|
describe("#23 export traceability: 100% valid linkage for all exports", () => {
|
|
let projectDir: string;
|
|
let outputDir: string;
|
|
let originalCwd: string;
|
|
|
|
beforeEach(() => {
|
|
projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "aitm-trace-complete-"));
|
|
outputDir = path.join(projectDir, "output");
|
|
fs.mkdirSync(path.join(projectDir, ".aitimemachine"), { recursive: true });
|
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(projectDir, "index.ts"),
|
|
'export const x = 1;\n',
|
|
"utf-8"
|
|
);
|
|
// Grant all consent so exports succeed and we can inspect trace linkage
|
|
saveConsent(projectDir, {
|
|
policy: { operational_capture: true, training_export: true },
|
|
});
|
|
originalCwd = process.cwd();
|
|
});
|
|
|
|
afterEach(() => {
|
|
process.chdir(originalCwd);
|
|
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("beginExportTrace produces non-null exportId, consentSnapshotId, startAuditId", () => {
|
|
const trace = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_operational",
|
|
source: "test.traceability-complete.begin",
|
|
resource: projectDir,
|
|
});
|
|
|
|
expect(trace.exportId).toBeTruthy();
|
|
expect(trace.exportId.startsWith("exp_")).toBe(true);
|
|
expect(trace.consentSnapshotId).toBeTruthy();
|
|
expect(trace.consentSnapshotId.startsWith("consent_")).toBe(true);
|
|
expect(trace.startAuditId).toBeTruthy();
|
|
expect(typeof trace.startAuditId).toBe("string");
|
|
expect(trace.startAuditId.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("completeExportTrace produces non-null auditId different from startAuditId", () => {
|
|
const trace = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_operational",
|
|
source: "test.traceability-complete.complete",
|
|
resource: projectDir,
|
|
});
|
|
|
|
const result = completeExportTrace(trace, {
|
|
success: true,
|
|
outputPath: path.join(outputDir, "test.json"),
|
|
stats: { records: 0 },
|
|
});
|
|
|
|
assertValidTrace(result, "completeExportTrace");
|
|
// The completion auditId must differ from the start auditId
|
|
expect(result.auditId).not.toBe(trace.startAuditId);
|
|
});
|
|
|
|
it("TrainingExporter.exportTrainingData returns valid trace linkage", async () => {
|
|
const exporter = new TrainingExporter();
|
|
const result = await exporter.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "training.jsonl"),
|
|
});
|
|
|
|
expect(result.success).toBe(true);
|
|
assertValidTrace(result.trace, "TrainingExporter");
|
|
});
|
|
|
|
it("DataExporter.exportAll returns valid trace linkage", async () => {
|
|
const exporter = new DataExporter(projectDir);
|
|
const result = await exporter.exportAll({
|
|
projectPath: projectDir,
|
|
format: "json",
|
|
outputPath: path.join(outputDir, "data.json"),
|
|
});
|
|
|
|
expect(result.success).toBe(true);
|
|
assertValidTrace(result.trace, "DataExporter");
|
|
});
|
|
|
|
it("ChangesExporter.exportChanges returns valid trace linkage", async () => {
|
|
const exporter = new ChangesExporter(projectDir);
|
|
const result = await exporter.exportChanges({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "changes.json"),
|
|
});
|
|
|
|
expect(result.success).toBe(true);
|
|
assertValidTrace(result.trace, "ChangesExporter");
|
|
});
|
|
|
|
it("SessionExporter.exportSessions returns valid trace linkage", async () => {
|
|
const exporter = new SessionExporter();
|
|
const result = await exporter.exportSessions({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "sessions.json"),
|
|
});
|
|
|
|
expect(result.success).toBe(true);
|
|
assertValidTrace(result.trace, "SessionExporter");
|
|
});
|
|
|
|
it("exportAnalytics helper returns valid trace via audit logs", () => {
|
|
process.chdir(projectDir);
|
|
|
|
// exportAnalytics does not return a trace object, but internally uses
|
|
// beginExportTrace / completeExportTrace which create audit entries.
|
|
// Verify that the audit logs have export.trace.start and export.trace.success entries.
|
|
const beforeLogs = getAuditLogs(10000);
|
|
const beforeCount = beforeLogs.filter(
|
|
(e) => e.action === "export.trace.start" || e.action === "export.trace.success"
|
|
).length;
|
|
|
|
exportAnalytics([], [], path.join(outputDir, "analytics.json"), "json");
|
|
|
|
const afterLogs = getAuditLogs(10000);
|
|
const afterTraceEntries = afterLogs.filter(
|
|
(e) => e.action === "export.trace.start" || e.action === "export.trace.success"
|
|
);
|
|
|
|
// At least 2 new audit entries should exist (start + success)
|
|
expect(afterTraceEntries.length).toBeGreaterThanOrEqual(beforeCount + 2);
|
|
});
|
|
|
|
it("exportTrainingData helper produces audit trail entries", () => {
|
|
process.chdir(projectDir);
|
|
|
|
const beforeLogs = getAuditLogs(10000);
|
|
const beforeStartCount = beforeLogs.filter(
|
|
(e) => e.action === "export.trace.start"
|
|
).length;
|
|
|
|
exportTrainingDataHelper(
|
|
[
|
|
{ role: "user", content: "hello" },
|
|
{ role: "assistant", content: "hi there" },
|
|
],
|
|
path.join(outputDir, "train-helper.jsonl"),
|
|
"generic"
|
|
);
|
|
|
|
const afterLogs = getAuditLogs(10000);
|
|
const afterStartEntries = afterLogs.filter(
|
|
(e) => e.action === "export.trace.start"
|
|
);
|
|
|
|
expect(afterStartEntries.length).toBeGreaterThan(beforeStartCount);
|
|
});
|
|
|
|
it("failed export still gets a trace.failure audit entry", () => {
|
|
const trace = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_operational",
|
|
source: "test.traceability-complete.failure",
|
|
resource: projectDir,
|
|
});
|
|
|
|
const result = completeExportTrace(trace, {
|
|
success: false,
|
|
error: new Error("simulated failure"),
|
|
});
|
|
|
|
assertValidTrace(result, "failed-export");
|
|
|
|
// Check that audit log contains the failure entry
|
|
const logs = getAuditLogs(10000);
|
|
const failureEntry = logs.find(
|
|
(e) =>
|
|
e.action === "export.trace.failure" &&
|
|
e.details?.exportId === trace.exportId
|
|
);
|
|
expect(failureEntry).toBeDefined();
|
|
expect(failureEntry!.success).toBe(false);
|
|
expect(failureEntry!.details?.linkedStartAuditId).toBe(trace.startAuditId);
|
|
});
|
|
});
|
|
|
|
describe("#23 export traceability: audit log entries for every export", () => {
|
|
let projectDir: string;
|
|
let outputDir: string;
|
|
let originalCwd: string;
|
|
|
|
beforeEach(() => {
|
|
projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "aitm-trace-audit-"));
|
|
outputDir = path.join(projectDir, "output");
|
|
fs.mkdirSync(path.join(projectDir, ".aitimemachine"), { recursive: true });
|
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(projectDir, "lib.ts"),
|
|
'export const lib = true;\n',
|
|
"utf-8"
|
|
);
|
|
saveConsent(projectDir, {
|
|
policy: { operational_capture: true, training_export: true },
|
|
});
|
|
originalCwd = process.cwd();
|
|
});
|
|
|
|
afterEach(() => {
|
|
process.chdir(originalCwd);
|
|
fs.rmSync(projectDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("each export creates exactly one start and one completion audit entry", async () => {
|
|
process.chdir(projectDir);
|
|
const logsBefore = getAuditLogs(10000);
|
|
const startCountBefore = logsBefore.filter(
|
|
(e) => e.action === "export.trace.start"
|
|
).length;
|
|
const successCountBefore = logsBefore.filter(
|
|
(e) => e.action === "export.trace.success"
|
|
).length;
|
|
|
|
// Run 4 different exports
|
|
const exporter1 = new DataExporter(projectDir);
|
|
await exporter1.exportAll({
|
|
projectPath: projectDir,
|
|
format: "json",
|
|
outputPath: path.join(outputDir, "d1.json"),
|
|
});
|
|
|
|
const exporter2 = new ChangesExporter(projectDir);
|
|
await exporter2.exportChanges({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "c1.json"),
|
|
});
|
|
|
|
const exporter3 = new SessionExporter();
|
|
await exporter3.exportSessions({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "s1.json"),
|
|
});
|
|
|
|
const exporter4 = new TrainingExporter();
|
|
await exporter4.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "t1.jsonl"),
|
|
});
|
|
|
|
const logsAfter = getAuditLogs(10000);
|
|
const startCountAfter = logsAfter.filter(
|
|
(e) => e.action === "export.trace.start"
|
|
).length;
|
|
const successCountAfter = logsAfter.filter(
|
|
(e) => e.action === "export.trace.success"
|
|
).length;
|
|
|
|
// 4 exports -> at least 4 new start entries and 4 new success entries
|
|
expect(startCountAfter - startCountBefore).toBeGreaterThanOrEqual(4);
|
|
expect(successCountAfter - successCountBefore).toBeGreaterThanOrEqual(4);
|
|
});
|
|
|
|
it("audit entries contain exportId linking start and completion", async () => {
|
|
process.chdir(projectDir);
|
|
const exporter = new TrainingExporter();
|
|
const result = await exporter.exportTrainingData({
|
|
projectPath: projectDir,
|
|
outputPath: path.join(outputDir, "linked.jsonl"),
|
|
});
|
|
|
|
const logs = getAuditLogs(10000);
|
|
|
|
// Find the start entry for this specific export
|
|
const startEntry = logs.find(
|
|
(e) =>
|
|
e.action === "export.trace.start" &&
|
|
e.details?.exportId === result.trace.exportId
|
|
);
|
|
expect(startEntry).toBeDefined();
|
|
expect(startEntry!.success).toBe(true);
|
|
expect(startEntry!.details?.consentSnapshotId).toBe(result.trace.consentSnapshotId);
|
|
|
|
// Find the completion entry for this specific export
|
|
const completionEntry = logs.find(
|
|
(e) =>
|
|
e.action === "export.trace.success" &&
|
|
e.details?.exportId === result.trace.exportId
|
|
);
|
|
expect(completionEntry).toBeDefined();
|
|
expect(completionEntry!.success).toBe(true);
|
|
expect(completionEntry!.details?.linkedStartAuditId).toBe(startEntry!.id);
|
|
expect(completionEntry!.details?.consentSnapshotId).toBe(
|
|
result.trace.consentSnapshotId
|
|
);
|
|
});
|
|
|
|
it("consent snapshot IDs are deterministic for same consent state", () => {
|
|
const trace1 = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_operational",
|
|
source: "test.deterministic.1",
|
|
});
|
|
|
|
const trace2 = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_operational",
|
|
source: "test.deterministic.2",
|
|
});
|
|
|
|
// Same consent state + same action = same consentSnapshotId
|
|
expect(trace1.consentSnapshotId).toBe(trace2.consentSnapshotId);
|
|
|
|
// But exportIds must be unique
|
|
expect(trace1.exportId).not.toBe(trace2.exportId);
|
|
});
|
|
|
|
it("consent snapshot ID changes when consent policy is updated", () => {
|
|
const trace1 = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_training",
|
|
source: "test.consent-change.before",
|
|
});
|
|
|
|
// Change consent
|
|
saveConsent(projectDir, {
|
|
policy: { operational_capture: true, training_export: false },
|
|
});
|
|
// Re-enable for the next trace to succeed
|
|
saveConsent(projectDir, {
|
|
policy: { operational_capture: true, training_export: true },
|
|
});
|
|
|
|
const trace2 = beginExportTrace({
|
|
projectPath: projectDir,
|
|
action: "export_training",
|
|
source: "test.consent-change.after",
|
|
});
|
|
|
|
// updatedAt changed, so consentSnapshotId must differ
|
|
expect(trace1.consentSnapshotId).not.toBe(trace2.consentSnapshotId);
|
|
});
|
|
});
|