This repository has been archived on 2026-11-17. You can view files and clone it, but cannot push or open issues or pull requests.
gitgram-ai/gitgram/backend/tests/unit/talentGraphRag.test.ts

304 lines
13 KiB
TypeScript

/**
* Unit tests for the talent GraphRAG serving layer.
*
* Pure (no Prisma, no network): the LLM call and the view loader are both
* injected. These tests pin the three load-bearing guarantees of the layer:
* 1. Retrieval correctness — the k-hop subgraph contains the right nodes at the
* right depth, takes no wrong hops, and always terminates.
* 2. Deterministic serialization — identical subgraph in, byte-identical text.
* 3. Grounding — the answer's evidence only ever references edges that were
* actually retrieved; hallucinated citations are dropped.
*/
import { validateEdge, type TalentEdge } from '../../src/services/talentGraphOntology';
import {
buildSubgraphFromEdges,
buildTalentSubgraph,
} from '../../src/services/talentSubgraphRetrieval';
import { serializeTalentSubgraphForLLM } from '../../src/services/talentSubgraphSerializer';
import { parseRagResponse } from '../../src/services/talentGraphRagPrompt';
import {
explainTalentMatch,
type ExplainTalentMatchDeps,
} from '../../src/services/talentGraphRag';
import type { TalentGraphView } from '../../src/services/talentGraphEdges';
// --- helpers ---------------------------------------------------------------
function edge(
type: string,
from: { id: string; type: string },
to: { id: string; type: string },
confidence = 1,
source = 'repository_claim',
): TalentEdge {
const result = validateEdge({
type,
from,
to,
provenance: { source, observedAt: new Date('2026-01-01T00:00:00Z'), confidence, sourceUrl: null },
});
if (!result.valid) throw new Error(`test edge invalid: ${result.rejections.map((r) => r.code).join(',')}`);
return result.edge;
}
const DEV = (id: string) => ({ id, type: 'developer' as const });
const REPO = (id: string) => ({ id, type: 'repo' as const });
/** A multi-hop edge set: dev-1 is the root; repo-2 sits two hops away. */
function multiHopEdges(): TalentEdge[] {
return [
edge('contributed_to', DEV('dev-1'), REPO('repo-1')),
edge('has_skill', DEV('dev-1'), { id: 'react', type: 'skill' }),
edge('member_of', DEV('dev-1'), { id: 'org-1', type: 'org' }),
edge('collaborated_with', DEV('dev-1'), DEV('dev-2'), 0.5, 'github_sync'),
// dev-2's own contribution — reachable only at depth 2 from dev-1
edge('contributed_to', DEV('dev-2'), REPO('repo-2')),
];
}
// --- retrieval correctness -------------------------------------------------
describe('buildSubgraphFromEdges (k-hop retrieval)', () => {
it('depth 0 returns only the root, no edges', () => {
const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 0);
expect(sub.nodes).toHaveLength(1);
expect(sub.nodes[0]).toMatchObject({ id: 'dev-1', type: 'developer', distance: 0 });
expect(sub.edges).toHaveLength(0);
});
it('depth 1 includes direct neighbors but takes no wrong hops', () => {
const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 1);
const nodeIds = sub.nodes.map((n) => `${n.type}:${n.id}`).sort();
expect(nodeIds).toEqual([
'developer:dev-1',
'developer:dev-2',
'org:org-1',
'repo:repo-1',
'skill:react',
]);
// repo-2 is two hops away — it must NOT appear, and neither must its edge.
expect(nodeIds).not.toContain('repo:repo-2');
expect(sub.edges.some((e) => e.to.id === 'repo-2' || e.from.id === 'repo-2')).toBe(false);
// every edge kept has both endpoints inside the node set
const present = new Set(nodeIds);
for (const e of sub.edges) {
expect(present.has(`${e.from.type}:${e.from.id}`)).toBe(true);
expect(present.has(`${e.to.type}:${e.to.id}`)).toBe(true);
}
});
it('depth 2 expands to the second hop and includes the bridging edge', () => {
const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2);
const repo2 = sub.nodes.find((n) => n.id === 'repo-2');
expect(repo2).toBeDefined();
expect(repo2?.distance).toBe(2);
expect(sub.edges.some((e) => e.from.id === 'dev-2' && e.to.id === 'repo-2')).toBe(true);
});
it('terminates on cycles and caps runaway depth', () => {
const cyclic = [
edge('collaborated_with', DEV('dev-1'), DEV('dev-2'), 0.5, 'github_sync'),
edge('collaborated_with', DEV('dev-2'), DEV('dev-3'), 0.5, 'github_sync'),
edge('collaborated_with', DEV('dev-3'), DEV('dev-1'), 0.5, 'github_sync'),
];
const sub = buildSubgraphFromEdges(cyclic, DEV('dev-1'), 1000);
// three developers, each visited once; depth is capped, not 1000
expect(sub.nodes).toHaveLength(3);
expect(sub.depth).toBeLessThanOrEqual(6);
});
it('rejects a negative or non-integer depth at the boundary', () => {
expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), -1)).toThrow();
expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 1.5)).toThrow();
});
it('rejects an empty root id', () => {
expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV(' '), 1)).toThrow();
});
});
// --- view-based retrieval --------------------------------------------------
const VIEW: TalentGraphView = {
profile: { id: 'dev-1' },
projects: [
{
id: 'repo-1',
url: 'https://github.com/acme/repo-1',
topics: ['reactjs', 'k8s'],
verificationLevel: 'verified',
lastVerifiedAt: new Date('2026-01-01T00:00:00Z'),
evidence: [{ kind: 'repository_claim', sourceUrl: 'https://github.com/acme/repo-1' }],
},
],
collaborators: [
{ id: 'dev-2', sharedProjects: [{ id: 'repo-1', url: 'https://github.com/acme/repo-1' }] },
],
};
describe('buildTalentSubgraph (view-based)', () => {
it('retrieves the developer neighborhood from an assembled view', () => {
const sub = buildTalentSubgraph(VIEW, 'dev-1', 1);
const types = sub.nodes.map((n) => n.type);
expect(types).toContain('developer'); // root + collaborator
expect(types).toContain('repo');
expect(types).toContain('skill');
// canonicalized skills present
expect(sub.nodes.some((n) => n.type === 'skill' && n.id === 'react')).toBe(true);
expect(sub.nodes.some((n) => n.type === 'skill' && n.id === 'kubernetes')).toBe(true);
// every edge carries provenance
for (const e of sub.edges) {
expect(e.provenance.confidence).toBeGreaterThan(0);
expect(e.provenance.observedAt).toBeInstanceOf(Date);
expect(typeof e.provenance.source).toBe('string');
}
});
it('rejects a rootDeveloperId that does not match the view profile', () => {
expect(() => buildTalentSubgraph(VIEW, 'someone-else', 1)).toThrow();
});
});
// --- deterministic serialization ------------------------------------------
describe('serializeTalentSubgraphForLLM', () => {
it('is deterministic: the same subgraph produces byte-identical text', () => {
// Serialization is a pure function of the subgraph, so a single retrieved
// subgraph must serialize identically every time (cacheable prompts). We
// build once and serialize twice; edge order is fixed by the retrieval layer.
const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2);
expect(serializeTalentSubgraphForLLM(sub)).toBe(serializeTalentSubgraphForLLM(sub));
});
it('orders edges canonically regardless of input edge order', () => {
const forward = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2);
const shuffled = buildSubgraphFromEdges([...multiHopEdges()].reverse(), DEV('dev-1'), 2);
expect(serializeTalentSubgraphForLLM(forward)).toBe(serializeTalentSubgraphForLLM(shuffled));
});
it('exposes edge citation keys and provenance in the context', () => {
const sub = buildTalentSubgraph(VIEW, 'dev-1', 2);
const text = serializeTalentSubgraphForLLM(sub);
expect(text).toContain('ROOT: developer:dev-1');
expect(text).toContain('EDGES (facts — cite by [key]):');
for (const e of sub.edges) {
expect(text).toContain(`[${e.key}]`);
}
expect(text).toMatch(/source=/);
expect(text).toMatch(/confidence=/);
});
it('renders an empty subgraph without crashing', () => {
const empty = buildSubgraphFromEdges([], DEV('dev-1'), 1);
const text = serializeTalentSubgraphForLLM(empty);
expect(text).toContain('ROOT: developer:dev-1');
expect(text).toContain('(none)');
});
});
// --- response parsing ------------------------------------------------------
describe('parseRagResponse', () => {
it('parses a clean JSON object', () => {
const parsed = parseRagResponse('{"answer":"ok","citedEdgeKeys":["k1","k2"]}');
expect(parsed.answer).toBe('ok');
expect(parsed.citedEdgeKeys).toEqual(['k1', 'k2']);
});
it('tolerates a ```json code fence', () => {
const parsed = parseRagResponse('```json\n{"answer":"x","citedEdgeKeys":[]}\n```');
expect(parsed.answer).toBe('x');
});
it('throws on non-JSON, wrong shape, or non-string citations', () => {
expect(() => parseRagResponse('not json')).toThrow();
expect(() => parseRagResponse('[]')).toThrow();
expect(() => parseRagResponse('{"answer":1,"citedEdgeKeys":[]}')).toThrow();
expect(() => parseRagResponse('{"answer":"x","citedEdgeKeys":[1]}')).toThrow();
expect(() => parseRagResponse('')).toThrow();
});
});
// --- grounding (the core guarantee) ---------------------------------------
describe('explainTalentMatch (grounding)', () => {
const loadView: ExplainTalentMatchDeps['loadView'] = async () => VIEW;
it('returns an answer whose evidence only references retrieved edges', async () => {
// Discover the real edge keys the subgraph will contain.
const sub = buildTalentSubgraph(VIEW, 'dev-1', 2);
const realKey = sub.edges[0].key;
const hallucinatedKey = 'collaborated_with|developer:ghost|developer:phantom';
const ask: ExplainTalentMatchDeps['ask'] = async (req) => {
// sanity: the grounding contract and the serialized context were passed
expect(req.system).toMatch(/ONLY using the facts/i);
expect(req.context).toContain('ROOT: developer:dev-1');
return JSON.stringify({
answer: 'dev-1 matches because of contributed repositories and skills.',
citedEdgeKeys: [realKey, hallucinatedKey],
});
};
const result = await explainTalentMatch('dev-1', 'Why does this developer match React work?', {
ask,
loadView,
depth: 2,
});
const subgraphKeys = new Set(sub.edges.map((e) => e.key));
// INVARIANT: every cited piece of evidence exists in the retrieved subgraph
for (const ev of result.evidence) {
expect(subgraphKeys.has(ev.key)).toBe(true);
}
expect(result.evidence.map((e) => e.key)).toContain(realKey);
// the hallucinated citation was dropped, not returned as evidence
expect(result.evidence.map((e) => e.key)).not.toContain(hallucinatedKey);
expect(result.droppedCitations).toContain(hallucinatedKey);
// evidence carries provenance for auditability
expect(result.evidence[0].source).toBeDefined();
expect(result.evidence[0].confidence).toBeGreaterThan(0);
expect(result.retrieval.edgeCount).toBe(sub.edges.length);
});
it('de-duplicates repeated citations and preserves the answer text', async () => {
const sub = buildTalentSubgraph(VIEW, 'dev-1', 2);
const key = sub.edges[0].key;
const ask: ExplainTalentMatchDeps['ask'] = async () =>
JSON.stringify({ answer: 'grounded answer', citedEdgeKeys: [key, key, key] });
const result = await explainTalentMatch('dev-1', 'why match', { ask, loadView, depth: 2 });
expect(result.answer).toBe('grounded answer');
expect(result.evidence.filter((e) => e.key === key)).toHaveLength(1);
expect(result.droppedCitations).toHaveLength(0);
});
it('returns no evidence when the model cites nothing', async () => {
const ask: ExplainTalentMatchDeps['ask'] = async () =>
JSON.stringify({ answer: 'insufficient information in the subgraph', citedEdgeKeys: [] });
const result = await explainTalentMatch('dev-1', 'why match', { ask, loadView });
expect(result.evidence).toHaveLength(0);
expect(result.answer).toMatch(/insufficient/);
});
it('validates inputs and surfaces a missing view', async () => {
const ask: ExplainTalentMatchDeps['ask'] = async () => '{"answer":"x","citedEdgeKeys":[]}';
await expect(explainTalentMatch('', 'q', { ask, loadView })).rejects.toThrow(/rootDeveloperId/);
await expect(explainTalentMatch('dev-1', ' ', { ask, loadView })).rejects.toThrow(/query/);
await expect(
explainTalentMatch('dev-1', 'q', { ask, loadView: async () => null }),
).rejects.toThrow(/No talent graph view/);
});
it('requires an injected ask function', async () => {
await expect(
explainTalentMatch('dev-1', 'q', { loadView } as unknown as ExplainTalentMatchDeps),
).rejects.toThrow(/deps.ask/);
});
it('propagates a malformed LLM reply as an explicit error', async () => {
const ask: ExplainTalentMatchDeps['ask'] = async () => 'garbage not json';
await expect(explainTalentMatch('dev-1', 'q', { ask, loadView })).rejects.toThrow(/JSON/);
});
});