This repository has been archived on 2026-11-17. You can view files and clone it, but cannot push or open issues or pull requests.
gitgram-ai/aitimemachine/gitnexus/test/unit/parse-impl-worker-lazy-cache.test.ts
caelum0x 49b183818c Unify gitgram + aitimemachine into the monorepo (de-submodule) and merge vendored engines
Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as
opaque submodule gitlinks on GitHub) into fully-tracked first-party content.

AITM now merges its 7 vendored repos as first-party engines:
- memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts
- git: vendored C git (bundled binary) + gitnexus (in-process code intelligence)
- agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess)

Includes the full session's product build: Gitgram web/mobile/backend features,
AITM website/CLI/MCP, the unified memory layer, engine build tooling
(scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md),
and extensive test suites.

Build artifacts, node_modules, external/, and gemini-cli are gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 14:14:09 +03:00

322 lines
11 KiB
TypeScript

/**
* Regression coverage for native-worker startup on warm parse-cache runs.
*
* A cache-hit chunk must replay cached worker output without spawning the
* parse-worker. Spawning workers on a warm cache hit still loads tree-sitter
* native bindings at top level, which was the root trigger for intermittent
* `libc++abi ... Napi::Error` crashes in linked local builds.
*/
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js';
import { computeChunkHash, fileContentHash } from '../../src/storage/parse-cache.js';
import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js';
const emptyWorkerResult = (filePath: string, name: string): ParseWorkerResult => ({
nodes: [
{
id: `Function:${filePath}:${name}`,
label: 'Function',
properties: {
name,
filePath,
startLine: 1,
endLine: 1,
language: 'typescript',
},
},
],
relationships: [],
symbols: [],
imports: [],
calls: [],
assignments: [],
heritage: [],
routes: [],
fetchCalls: [],
fetchWrapperDefs: [],
decoratorRoutes: [],
routerIncludes: [],
routerImports: [],
toolDefs: [],
ormQueries: [],
constructorBindings: [],
fileScopeBindings: [],
parsedFiles: [],
skippedLanguages: {},
fileCount: 1,
});
const writeReadyWorker = (workerPath: string, markerPath: string): void => {
fs.writeFileSync(
workerPath,
`
const fs = require('node:fs');
const { parentPort } = require('node:worker_threads');
fs.writeFileSync(${JSON.stringify(markerPath)}, 'spawned');
parentPort.postMessage({ type: 'ready' });
parentPort.on('message', () => {});
`,
);
};
const writeResultWorker = (workerPath: string, markerPath: string): void => {
fs.writeFileSync(
workerPath,
`
const fs = require('node:fs');
const { parentPort } = require('node:worker_threads');
const decoder = new TextDecoder('utf-8');
fs.writeFileSync(${JSON.stringify(markerPath)}, 'spawned');
parentPort.postMessage({ type: 'ready' });
const accumulated = {
nodes: [], relationships: [], symbols: [], imports: [], calls: [], assignments: [], heritage: [],
routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], routerImports: [], toolDefs: [], ormQueries: [], constructorBindings: [],
fileScopeBindings: [], parsedFiles: [], skippedLanguages: {}, fileCount: 0,
};
parentPort.on('message', (msg) => {
if (msg && msg.type === 'sub-batch') {
for (const file of msg.files) {
const filePath = file.path;
const name = filePath.split('/').pop().replace(/\\.ts$/, '');
accumulated.nodes.push({
id: 'Function:' + filePath + ':' + name,
label: 'Function',
properties: { name, filePath, startLine: 1, endLine: 1, language: 'typescript' },
});
accumulated.fileCount++;
// Decode to exercise the same transfer-list shape as production.
if (file.content && typeof file.content !== 'string') decoder.decode(file.content);
}
parentPort.postMessage({ type: 'progress', filesProcessed: accumulated.fileCount });
parentPort.postMessage({ type: 'sub-batch-done' });
return;
}
if (msg && msg.type === 'flush') parentPort.postMessage({ type: 'result', data: accumulated });
});
`,
);
};
const writeExitBeforeReadyWorker = (workerPath: string): void => {
fs.writeFileSync(workerPath, `process.exit(1);\n`);
};
describe('parse-impl worker pool lazy startup', () => {
let tempDir = '';
let repoDir = '';
beforeEach(() => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'parse-impl-worker-lazy-cache-'));
repoDir = path.join(tempDir, 'repo');
fs.mkdirSync(repoDir, { recursive: true });
});
afterEach(() => {
if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true });
});
it('does not spawn a parse worker when every chunk is served from parse cache', async () => {
const rel = 'src/cached.ts';
const content = 'export function cached() { return 1; }\n';
const full = path.join(repoDir, rel);
fs.mkdirSync(path.dirname(full), { recursive: true });
fs.writeFileSync(full, content);
const chunkHash = computeChunkHash([{ filePath: rel, contentHash: fileContentHash(content) }]);
const parseCache = {
version: 'test',
entries: new Map<string, ParseWorkerResult[]>([
[chunkHash, [emptyWorkerResult(rel, 'cached')]],
]),
usedKeys: new Set<string>(),
};
const markerPath = path.join(tempDir, 'worker-spawned.marker');
const workerPath = path.join(tempDir, 'ready-worker.js');
writeReadyWorker(workerPath, markerPath);
const graph = createKnowledgeGraph();
await runChunkedParseAndResolve(
graph,
[{ path: rel, size: fs.statSync(full).size }],
[rel],
1,
repoDir,
Date.now(),
() => {},
{
workerUrlForTest: pathToFileURL(workerPath),
workerPoolSize: 1,
parseCache,
},
);
expect(fs.existsSync(markerPath)).toBe(false);
expect(parseCache.usedKeys.has(chunkHash)).toBe(true);
expect(Array.from(graph.nodes.values()).some((n) => n.properties.name === 'cached')).toBe(true);
});
it('spawns the parse worker lazily on the first cache miss and stores raw results', async () => {
const rel = 'src/miss.ts';
const content = 'export function miss() { return 1; }\n';
const full = path.join(repoDir, rel);
fs.mkdirSync(path.dirname(full), { recursive: true });
fs.writeFileSync(full, content);
const markerPath = path.join(tempDir, 'worker-spawned.marker');
const workerPath = path.join(tempDir, 'result-worker.js');
writeResultWorker(workerPath, markerPath);
const parseCache = {
version: 'test',
entries: new Map<string, ParseWorkerResult[]>(),
usedKeys: new Set<string>(),
};
const chunkHash = computeChunkHash([{ filePath: rel, contentHash: fileContentHash(content) }]);
const graph = createKnowledgeGraph();
await runChunkedParseAndResolve(
graph,
[{ path: rel, size: fs.statSync(full).size }],
[rel],
1,
repoDir,
Date.now(),
() => {},
{
workerUrlForTest: pathToFileURL(workerPath),
workerPoolSize: 1,
parseCache,
},
);
expect(fs.existsSync(markerPath)).toBe(true);
expect(parseCache.entries.has(chunkHash)).toBe(true);
expect(Array.from(graph.nodes.values()).some((n) => n.properties.name === 'miss')).toBe(true);
});
it('fails fast (no silent fallback) when the pool cannot start its workers (#1741)', async () => {
const rel = 'src/fatal.ts';
const content = 'export function fatal() { return 1; }\n';
const full = path.join(repoDir, rel);
fs.mkdirSync(path.dirname(full), { recursive: true });
fs.writeFileSync(full, content);
const workerPath = path.join(tempDir, 'exit-before-ready-fatal-worker.js');
writeExitBeforeReadyWorker(workerPath);
const parseCache = {
version: 'test',
entries: new Map<string, ParseWorkerResult[]>(),
usedKeys: new Set<string>(),
};
const graph = createKnowledgeGraph();
await expect(
runChunkedParseAndResolve(
graph,
[{ path: rel, size: fs.statSync(full).size }],
[rel],
1,
repoDir,
Date.now(),
() => {},
{
workerUrlForTest: pathToFileURL(workerPath),
workerPoolSize: 1,
// No flag: a total worker-startup failure always fails fast now.
parseCache,
},
),
).rejects.toThrow(/Worker pool failed to start/i);
// The fatal path did not silently parse sequentially behind the user's back.
expect(Array.from(graph.nodes.values()).some((n) => n.properties.name === 'fatal')).toBe(false);
});
it('throws when GITNEXUS_WORKER_POOL_SIZE=0 and no --workers flag (sequential parsing removed)', async () => {
const saved = process.env.GITNEXUS_WORKER_POOL_SIZE;
process.env.GITNEXUS_WORKER_POOL_SIZE = '0';
try {
const rel = 'src/env0.ts';
const content = 'export function env0() { return 1; }\n';
const full = path.join(repoDir, rel);
fs.mkdirSync(path.dirname(full), { recursive: true });
fs.writeFileSync(full, content);
// A ready-worker double with a spawn marker — it must NOT be spawned,
// because the disabled-channel validation throws before any pool is built.
const markerPath = path.join(tempDir, 'env0-worker.marker');
const workerPath = path.join(tempDir, 'env0-ready-worker.js');
writeReadyWorker(workerPath, markerPath);
const graph = createKnowledgeGraph();
await expect(
runChunkedParseAndResolve(
graph,
[{ path: rel, size: fs.statSync(full).size }],
[rel],
1,
repoDir,
Date.now(),
() => {},
{
workerUrlForTest: pathToFileURL(workerPath),
// No workerPoolSize option — the ambient env=0 is the only signal.
},
),
).rejects.toThrow(/GITNEXUS_WORKER_POOL_SIZE=0/);
// Sequential parsing was removed: env=0 is a hard error, not a silent
// sequential run. The validation throws before any pool is constructed.
expect(fs.existsSync(markerPath)).toBe(false);
} finally {
if (saved === undefined) delete process.env.GITNEXUS_WORKER_POOL_SIZE;
else process.env.GITNEXUS_WORKER_POOL_SIZE = saved;
}
});
it('an explicit --workers wins over an ambient GITNEXUS_WORKER_POOL_SIZE=0 (#1741)', async () => {
const saved = process.env.GITNEXUS_WORKER_POOL_SIZE;
process.env.GITNEXUS_WORKER_POOL_SIZE = '0';
try {
const rel = 'src/precedence.ts';
const content = 'export function precedence() { return 1; }\n';
const full = path.join(repoDir, rel);
fs.mkdirSync(path.dirname(full), { recursive: true });
fs.writeFileSync(full, content);
const markerPath = path.join(tempDir, 'precedence-worker.marker');
const workerPath = path.join(tempDir, 'precedence-result-worker.js');
writeResultWorker(workerPath, markerPath);
const graph = createKnowledgeGraph();
const result = await runChunkedParseAndResolve(
graph,
[{ path: rel, size: fs.statSync(full).size }],
[rel],
1,
repoDir,
Date.now(),
() => {},
{
workerUrlForTest: pathToFileURL(workerPath),
workerPoolSize: 1, // explicit --workers 1 must win over ambient env=0
},
);
expect(result.usedWorkerPool).toBe(true); // explicit flag wins; env=0 ignored
expect(fs.existsSync(markerPath)).toBe(true); // worker was spawned
} finally {
if (saved === undefined) delete process.env.GITNEXUS_WORKER_POOL_SIZE;
else process.env.GITNEXUS_WORKER_POOL_SIZE = saved;
}
});
});