Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as opaque submodule gitlinks on GitHub) into fully-tracked first-party content. AITM now merges its 7 vendored repos as first-party engines: - memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts - git: vendored C git (bundled binary) + gitnexus (in-process code intelligence) - agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess) Includes the full session's product build: Gitgram web/mobile/backend features, AITM website/CLI/MCP, the unified memory layer, engine build tooling (scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md), and extensive test suites. Build artifacts, node_modules, external/, and gemini-cli are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
159 lines
7.0 KiB
TypeScript
159 lines
7.0 KiB
TypeScript
/**
|
|
* Regression test for CRLF-encoded markdown heading extraction.
|
|
*
|
|
* Files with CRLF line endings (Windows-authored markdown) previously
|
|
* produced zero Section nodes because `split('\n')` left a trailing `\r`
|
|
* on each line, and the heading regex `/^(#{1,6})\s+(.+)$/` (anchored
|
|
* with `$`) failed to match `## Heading\r` because `$` only matches at
|
|
* end-of-string while `.+` does not consume the trailing `\r`.
|
|
*
|
|
* Fix: split on `/\r\n|\r|\n/` so all line-ending conventions are
|
|
* normalized at split time. See markdown-processor.ts line 39.
|
|
*/
|
|
|
|
import { describe, it, expect } from 'vitest';
|
|
import { processMarkdown } from '../../src/core/ingestion/markdown-processor.js';
|
|
import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
|
|
import { generateId } from '../../src/lib/utils.js';
|
|
import type { GraphNode } from 'gitnexus-shared';
|
|
import type { KnowledgeGraph } from '../../src/core/graph/types.js';
|
|
|
|
function getMarkdownSections(graph: KnowledgeGraph, filePath: string): GraphNode[] {
|
|
return [...graph.iterNodes()]
|
|
.filter((n) => n.label === 'Section' && n.properties.filePath === filePath)
|
|
.sort(
|
|
(a, b) =>
|
|
((a.properties.startLine as number | undefined) ?? 0) -
|
|
((b.properties.startLine as number | undefined) ?? 0),
|
|
);
|
|
}
|
|
|
|
function expectContainsEdge(graph: KnowledgeGraph, sourceId: string, targetId: string) {
|
|
const found = [...graph.iterRelationshipsByType('CONTAINS')].some(
|
|
(r) => r.sourceId === sourceId && r.targetId === targetId,
|
|
);
|
|
expect(found).toBe(true);
|
|
}
|
|
|
|
function setupGraphWithFile(filePath: string) {
|
|
const graph = createKnowledgeGraph();
|
|
const fileNode: GraphNode = {
|
|
id: generateId('File', filePath),
|
|
label: 'File',
|
|
properties: { name: filePath, filePath },
|
|
};
|
|
graph.addNode(fileNode);
|
|
return graph;
|
|
}
|
|
|
|
describe('markdown-processor CRLF tolerance', () => {
|
|
it('extracts headings from LF-encoded markdown (baseline)', () => {
|
|
const filePath = 'lf.md';
|
|
const graph = setupGraphWithFile(filePath);
|
|
const content = '# Title\nbody line 1\n## Sub\nbody line 2\n### SubSub\nmore\n';
|
|
|
|
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
|
|
|
expect(stats.sections).toBe(3);
|
|
const sections = getMarkdownSections(graph, filePath);
|
|
expect(sections.map((s) => s.properties.name)).toEqual(['Title', 'Sub', 'SubSub']);
|
|
expect(sections.map((s) => s.properties.level)).toEqual([1, 2, 3]);
|
|
expect(sections.map((s) => s.properties.startLine)).toEqual([1, 3, 5]);
|
|
expect(sections.map((s) => s.properties.endLine)).toEqual([7, 7, 7]);
|
|
for (const s of sections) {
|
|
expect(String(s.properties.name)).not.toMatch(/\r/);
|
|
}
|
|
const fileId = generateId('File', filePath);
|
|
expectContainsEdge(graph, fileId, sections[0]!.id);
|
|
expectContainsEdge(graph, sections[0]!.id, sections[1]!.id);
|
|
expectContainsEdge(graph, sections[1]!.id, sections[2]!.id);
|
|
});
|
|
|
|
it('extracts headings from CRLF-encoded markdown (the regression)', () => {
|
|
const filePath = 'crlf.md';
|
|
const graph = setupGraphWithFile(filePath);
|
|
const content = '# Title\r\nbody line 1\r\n## Sub\r\nbody line 2\r\n### SubSub\r\nmore\r\n';
|
|
|
|
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
|
|
|
// Pre-fix: this returned 0 because `## Sub\r` failed the heading regex.
|
|
expect(stats.sections).toBe(3);
|
|
const sections = getMarkdownSections(graph, filePath);
|
|
expect(sections.map((s) => s.properties.name)).toEqual(['Title', 'Sub', 'SubSub']);
|
|
expect(sections.map((s) => s.properties.level)).toEqual([1, 2, 3]);
|
|
expect(sections.map((s) => s.properties.startLine)).toEqual([1, 3, 5]);
|
|
expect(sections.map((s) => s.properties.endLine)).toEqual([7, 7, 7]);
|
|
for (const s of sections) {
|
|
expect(String(s.properties.name)).not.toMatch(/\r/);
|
|
}
|
|
const fileId = generateId('File', filePath);
|
|
expectContainsEdge(graph, fileId, sections[0]!.id);
|
|
expectContainsEdge(graph, sections[0]!.id, sections[1]!.id);
|
|
expectContainsEdge(graph, sections[1]!.id, sections[2]!.id);
|
|
});
|
|
|
|
it('extracts headings from CR-only-encoded markdown (old Mac OS Classic)', () => {
|
|
const filePath = 'cr.md';
|
|
const graph = setupGraphWithFile(filePath);
|
|
const content = '# Title\rbody line 1\r## Sub\rbody line 2\r';
|
|
|
|
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
|
|
|
expect(stats.sections).toBe(2);
|
|
const sections = getMarkdownSections(graph, filePath);
|
|
expect(sections.map((s) => s.properties.name)).toEqual(['Title', 'Sub']);
|
|
expect(sections.map((s) => s.properties.level)).toEqual([1, 2]);
|
|
expect(sections.map((s) => s.properties.startLine)).toEqual([1, 3]);
|
|
expect(sections.map((s) => s.properties.endLine)).toEqual([5, 5]);
|
|
for (const s of sections) {
|
|
expect(String(s.properties.name)).not.toMatch(/\r/);
|
|
}
|
|
const fileId = generateId('File', filePath);
|
|
expectContainsEdge(graph, fileId, sections[0]!.id);
|
|
expectContainsEdge(graph, sections[0]!.id, sections[1]!.id);
|
|
});
|
|
|
|
it('extracts headings from mixed CRLF + LF markdown', () => {
|
|
const filePath = 'mixed.md';
|
|
const graph = setupGraphWithFile(filePath);
|
|
const content = '# LF Title\nbody\r\n## CRLF Sub\r\nmore\n### Trailing LF\nend\n';
|
|
|
|
const stats = processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
|
|
|
expect(stats.sections).toBe(3);
|
|
const sections = getMarkdownSections(graph, filePath);
|
|
expect(sections.map((s) => s.properties.name)).toEqual(['LF Title', 'CRLF Sub', 'Trailing LF']);
|
|
expect(sections.map((s) => s.properties.level)).toEqual([1, 2, 3]);
|
|
expect(sections.map((s) => s.properties.startLine)).toEqual([1, 3, 5]);
|
|
expect(sections.map((s) => s.properties.endLine)).toEqual([7, 7, 7]);
|
|
for (const s of sections) {
|
|
expect(String(s.properties.name)).not.toMatch(/\r/);
|
|
}
|
|
const fileId = generateId('File', filePath);
|
|
expectContainsEdge(graph, fileId, sections[0]!.id);
|
|
expectContainsEdge(graph, sections[0]!.id, sections[1]!.id);
|
|
expectContainsEdge(graph, sections[1]!.id, sections[2]!.id);
|
|
});
|
|
|
|
it('reports correct startLine and endLine for CRLF content', () => {
|
|
const filePath = 'crlf-lines.md';
|
|
const graph = setupGraphWithFile(filePath);
|
|
// Lines 1, 3, 5 are headings (1-indexed)
|
|
const content = '# T\r\nbody\r\n## Sub\r\nmore\r\n### SubSub\r\ntail\r\n';
|
|
|
|
processMarkdown(graph, [{ path: filePath, content }], new Set([filePath]));
|
|
|
|
const sections = getMarkdownSections(graph, filePath);
|
|
const titleSection = sections.find((s) => s.properties.name === 'T');
|
|
const subSection = sections.find((s) => s.properties.name === 'Sub');
|
|
const subSubSection = sections.find((s) => s.properties.name === 'SubSub');
|
|
|
|
expect(titleSection?.properties.startLine).toBe(1);
|
|
expect(titleSection?.properties.endLine).toBe(7);
|
|
expect(subSection?.properties.startLine).toBe(3);
|
|
expect(subSection?.properties.endLine).toBe(7);
|
|
expect(subSubSection?.properties.startLine).toBe(5);
|
|
expect(subSubSection?.properties.endLine).toBe(7);
|
|
});
|
|
});
|