diff --git a/gitgram/backend/src/controllers/talentGraphController.ts b/gitgram/backend/src/controllers/talentGraphController.ts index 58315e92..97cc9578 100644 --- a/gitgram/backend/src/controllers/talentGraphController.ts +++ b/gitgram/backend/src/controllers/talentGraphController.ts @@ -6,6 +6,30 @@ import { analyzeNeedCoverage } from '../services/talentGapAnalysisService'; import prisma from '../utils/prisma'; import { normalizePublicHttpUrl } from '../utils/publicHttpUrl'; import { runTalentMutation, type TalentMutationResult } from '../services/talentMutationService'; +import { explainTalentMatch, defaultRagAsk } from '../services/talentGraphRag'; +import type { TalentGraphView } from '../services/talentGraphEdges'; + +/** + * Run a grounded GraphRAG explanation and translate the module's plain errors + * into the API's typed errors. The LLM seam (`ask`) is injected here so the + * controller stays a thin adapter over the testable service; until a real client + * is wired, `defaultRagAsk` surfaces a clear 503 rather than a fake answer. + */ +async function runGroundedExplain( + developerId: string, + query: unknown, + loadView?: (id: string) => Promise, +) { + const cleanedQuery = boundedText(query, 'query', 2000); + try { + return await explainTalentMatch(developerId, cleanedQuery, { ask: defaultRagAsk, loadView }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes('No LLM client')) throw Errors.serviceUnavailable('Talent GraphRAG'); + if (message.includes('No talent graph view')) throw Errors.userNotFound(developerId); + throw Errors.badRequest(message); + } +} const PILOT_STATUSES = ['PLANNED', 'ACTIVE', 'REVIEW', 'COMPLETED', 'CANCELLED'] as const; type PilotStatus = (typeof PILOT_STATUSES)[number]; @@ -93,6 +117,33 @@ export const talentGraphController = { res.json({ data: graph, meta: { proofSchemaVersion: 1 } }); }), + // GraphRAG serving: grounded "why/how does this developer match?" over the + // authenticated user's own assembled talent graph. Non-breaking — a new route + // that reuses getTalentGraph via the RAG service's default view loader. + explainMine: asyncHandler(async (req: AuthRequest, res: Response) => { + const result = await runGroundedExplain(req.userId!, req.body?.query); + res.setHeader('Cache-Control', 'no-store, private'); + res.json({ data: result }); + }), + + // Grounded explanation over a public developer's privacy-safe dossier. Reuses + // getTalentGraph in publicView mode as the injected view loader. + explainPublic: asyncHandler(async (req: AuthRequest, res: Response) => { + const user = await prisma.user.findUnique({ + where: { username: req.params.username }, + select: { id: true, isPrivate: true }, + }); + if (!user || user.isPrivate) throw Errors.userNotFound(req.params.username); + + const result = await runGroundedExplain( + user.id, + req.body?.query, + async (id) => (await getTalentGraph(id, { publicView: true })) as TalentGraphView | null, + ); + res.setHeader('Cache-Control', 'no-store'); + res.json({ data: result }); + }), + listOpportunities: asyncHandler(async (req: AuthRequest, res: Response) => { const opportunities = await prisma.talentOpportunity.findMany({ where: { status: { in: ['OPEN', 'MATCHING', 'PILOT'] } }, diff --git a/gitgram/backend/src/routes/talentGraph.ts b/gitgram/backend/src/routes/talentGraph.ts index 3dabaaed..41e9c92b 100644 --- a/gitgram/backend/src/routes/talentGraph.ts +++ b/gitgram/backend/src/routes/talentGraph.ts @@ -27,7 +27,9 @@ function requireMutationOrigin(req: Request, _res: Response, next: NextFunction) } router.get('/me', authenticate, talentGraphController.getMine); +router.post('/me/explain', authenticate, requireMutationOrigin, interactionLimiter, talentGraphController.explainMine); router.get('/public/:username', talentGraphController.getPublic); +router.post('/public/:username/explain', optionalAuth, requireMutationOrigin, interactionLimiter, talentGraphController.explainPublic); router.get('/opportunities', optionalAuth, talentGraphController.listOpportunities); router.post('/opportunities', authenticate, requireMutationOrigin, interactionLimiter, talentGraphController.createOpportunity); router.get('/opportunities/:id/matches', optionalAuth, talentGraphController.getOpportunityMatches); diff --git a/gitgram/backend/src/services/talentGraphRag.ts b/gitgram/backend/src/services/talentGraphRag.ts new file mode 100644 index 00000000..6a69bec5 --- /dev/null +++ b/gitgram/backend/src/services/talentGraphRag.ts @@ -0,0 +1,212 @@ +/** + * Talent GraphRAG serving layer — grounded "why does this developer match?" query. + * + * This is the top of the GraphRAG vertical slice. It composes the pieces that + * were each built to be pure and testable: + * load view → buildTalentSubgraph (retrieve) → serializeTalentSubgraphForLLM + * → ask the LLM under a grounding contract → enforce grounding on the reply. + * + * Grounding is enforced on OUR side, not trusted from the model: the LLM is asked + * to cite edges by their canonical key, and we intersect its citations with the + * keys that actually exist in the retrieved subgraph. Any cited key that is not + * in the subgraph is a hallucinated reference and is dropped; the returned + * `evidence` therefore only ever references facts that were truly retrieved. + * + * The LLM call is injected via `deps.ask` so the retrieval + serialization + + * grounding logic is fully unit-testable without a live API. `deps.loadView` is + * likewise injectable; a default wires it to the existing `getTalentGraph` + * dossier assembler, keeping the seam while reusing the read path. + */ +import logger from '../utils/logger'; +import { getTalentGraph } from './talentGraphService'; +import type { TalentGraphView } from './talentGraphEdges'; +import type { + EvidenceSource, + NodeRef, + TalentEdge, + TalentEdgeType, +} from './talentGraphOntology'; +import { + buildTalentSubgraph, + type TalentSubgraph, +} from './talentSubgraphRetrieval'; +import { serializeTalentSubgraphForLLM } from './talentSubgraphSerializer'; +import { + buildRagPrompt, + parseRagResponse, + type RagAsk, +} from './talentGraphRagPrompt'; + +const DEFAULT_DEPTH = 2; +const MAX_QUERY_LENGTH = 2000; + +/** A single cited fact returned to the caller — a projection of a subgraph edge. */ +export interface EdgeReference { + readonly key: string; + readonly type: TalentEdgeType; + readonly from: NodeRef; + readonly to: NodeRef; + readonly source: EvidenceSource; + readonly confidence: number; + readonly observedAt: string; + readonly sourceUrl: string | null; +} + +/** The grounded answer plus the evidence it is allowed to stand on. */ +export interface ExplainTalentMatchResult { + readonly developerId: string; + readonly query: string; + readonly answer: string; + readonly evidence: readonly EdgeReference[]; + readonly retrieval: { + readonly depth: number; + readonly nodeCount: number; + readonly edgeCount: number; + }; + /** + * Keys the model cited that were NOT in the subgraph. Always empty in a + * well-behaved run; surfaced (not hidden) so grounding violations are + * observable rather than silently absorbed. + */ + readonly droppedCitations: readonly string[]; +} + +/** Injectable dependencies for {@link explainTalentMatch}. */ +export interface ExplainTalentMatchDeps { + /** The LLM seam. Required — there is no implicit network call. */ + readonly ask: RagAsk; + /** How to obtain the assembled view. Defaults to {@link getTalentGraph}. */ + readonly loadView?: (developerId: string) => Promise; + /** k-hop retrieval depth. Defaults to {@link DEFAULT_DEPTH}. */ + readonly depth?: number; +} + +/** Default view loader: reuse the existing dossier assembler on the read path. */ +async function defaultLoadView(developerId: string): Promise { + return (await getTalentGraph(developerId)) as TalentGraphView | null; +} + +function toEdgeReference(edge: TalentEdge): EdgeReference { + return { + key: edge.key, + type: edge.type, + from: edge.from, + to: edge.to, + source: edge.provenance.source, + confidence: edge.provenance.confidence, + observedAt: edge.provenance.observedAt.toISOString(), + sourceUrl: edge.provenance.sourceUrl, + }; +} + +/** + * Enforce grounding: partition the model's cited keys into those present in the + * subgraph (kept as evidence, de-duplicated, in subgraph order) and those that + * are not (dropped as hallucinations). This is the guardrail that guarantees the + * returned evidence only references retrieved facts. + */ +function groundCitations( + subgraph: TalentSubgraph, + citedKeys: readonly string[], +): { evidence: EdgeReference[]; dropped: string[] } { + const edgesByKey = new Map(subgraph.edges.map((edge) => [edge.key, edge])); + const seen = new Set(); + const dropped: string[] = []; + + for (const key of citedKeys) { + if (!edgesByKey.has(key)) { + if (!dropped.includes(key)) dropped.push(key); + } + } + + // Preserve deterministic subgraph edge order for cited, in-subgraph keys. + const citedSet = new Set(citedKeys); + const evidence: EdgeReference[] = []; + for (const edge of subgraph.edges) { + if (citedSet.has(edge.key) && !seen.has(edge.key)) { + seen.add(edge.key); + evidence.push(toEdgeReference(edge)); + } + } + + return { evidence, dropped }; +} + +function normalizeQuery(query: string): string { + const cleaned = typeof query === 'string' ? query.trim() : ''; + if (!cleaned) throw new Error('A non-empty query is required to explain a talent match'); + if (cleaned.length > MAX_QUERY_LENGTH) { + throw new Error(`Query must be at most ${MAX_QUERY_LENGTH} characters`); + } + return cleaned; +} + +/** + * Explain, grounded in the retrieved subgraph, why/how a developer matches a + * query. Retrieves the k-hop neighborhood, serializes it, asks the injected LLM + * under the grounding contract, then enforces that the returned evidence only + * references retrieved edges. + * + * @throws if inputs are invalid, the developer has no assembled view, or the LLM + * reply cannot be parsed. Errors are explicit — never silently swallowed. + */ +export async function explainTalentMatch( + rootDeveloperId: string, + query: string, + deps: ExplainTalentMatchDeps, +): Promise { + const developerId = typeof rootDeveloperId === 'string' ? rootDeveloperId.trim() : ''; + if (!developerId) throw new Error('explainTalentMatch requires a non-empty rootDeveloperId'); + if (!deps || typeof deps.ask !== 'function') { + throw new Error('explainTalentMatch requires deps.ask (the injectable LLM call)'); + } + const cleanedQuery = normalizeQuery(query); + const depth = deps.depth ?? DEFAULT_DEPTH; + const loadView = deps.loadView ?? defaultLoadView; + + const view = await loadView(developerId); + if (!view) throw new Error(`No talent graph view found for developer "${developerId}"`); + + const subgraph = buildTalentSubgraph(view, developerId, depth); + const context = serializeTalentSubgraphForLLM(subgraph); + const prompt = buildRagPrompt(cleanedQuery, context); + + const raw = await deps.ask(prompt); + const parsed = parseRagResponse(raw); + const { evidence, dropped } = groundCitations(subgraph, parsed.citedEdgeKeys); + + if (dropped.length > 0) { + logger.warn('talent-graphrag dropped hallucinated citations not present in the subgraph', { + event: 'talent_graph.rag.hallucinated_citation', + developerId, + droppedCount: dropped.length, + dropped, + }); + } + + return { + developerId, + query: cleanedQuery, + answer: parsed.answer, + evidence, + retrieval: { + depth: subgraph.depth, + nodeCount: subgraph.nodes.length, + edgeCount: subgraph.edges.length, + }, + droppedCitations: dropped, + }; +} + +/** + * Default LLM seam. There is no LLM client wired into gitgram-ai yet, so this + * default fails loudly rather than pretending to answer — production wiring + * should replace it with a real client (e.g. via `deps.ask`) while keeping this + * exact signature. Kept as a named export so the controller can depend on the + * seam and tests can substitute their own `ask`. + */ +export const defaultRagAsk: RagAsk = async () => { + throw new Error( + 'No LLM client is configured for talent GraphRAG. Provide deps.ask to explainTalentMatch.', + ); +}; diff --git a/gitgram/backend/src/services/talentGraphRagPrompt.ts b/gitgram/backend/src/services/talentGraphRagPrompt.ts new file mode 100644 index 00000000..294515c3 --- /dev/null +++ b/gitgram/backend/src/services/talentGraphRagPrompt.ts @@ -0,0 +1,97 @@ +/** + * Grounded prompt construction + response parsing for talent GraphRAG. + * + * This file owns the *grounding contract* between our retrieval layer and the + * LLM: the system instruction that forbids outside knowledge, the user prompt + * that carries the serialized subgraph and the query, and the strict parser that + * turns the model's reply back into a typed answer + cited edge keys. Keeping the + * contract in one small module makes the grounding rules auditable and lets the + * parser be unit-tested without any live API. + */ + +/** The instruction that pins the model to the retrieved subgraph only. */ +export const GROUNDING_SYSTEM_PROMPT = [ + 'You are a talent-graph analyst. You answer ONLY using the facts in the', + 'provided subgraph context. Never use outside knowledge, and never invent', + 'developers, repositories, skills, organizations, or relationships that are', + 'not present in the EDGES list.', + '', + 'Every claim in your answer must be supported by one or more edges, cited by', + 'their [key]. If the subgraph does not contain enough information to answer,', + 'say so plainly and cite no edges.', + '', + 'Respond with a single JSON object and nothing else, of the exact shape:', + '{"answer": string, "citedEdgeKeys": string[]}', + 'where each citedEdgeKeys entry is an edge [key] copied verbatim from EDGES.', +].join('\n'); + +/** A request handed to an injected LLM. `context` is the serialized subgraph. */ +export interface RagAskRequest { + readonly system: string; + readonly query: string; + readonly context: string; +} + +/** The injectable LLM seam: raw prompt in, raw model text out. */ +export type RagAsk = (request: RagAskRequest) => Promise; + +/** Parsed, still-untrusted model reply before grounding is enforced. */ +export interface ParsedRagResponse { + readonly answer: string; + readonly citedEdgeKeys: readonly string[]; +} + +/** + * Build the user-facing prompt body. The query is included verbatim but clearly + * fenced from the context so a hostile query cannot masquerade as graph facts. + */ +export function buildRagPrompt(query: string, context: string): RagAskRequest { + const cleanedQuery = typeof query === 'string' ? query.trim() : ''; + if (!cleanedQuery) throw new Error('A non-empty query is required to explain a talent match'); + return { + system: GROUNDING_SYSTEM_PROMPT, + query: cleanedQuery, + context, + }; +} + +/** + * Parse a raw LLM reply into a typed response. Tolerant of a leading/trailing + * code fence but otherwise strict: the payload must be a JSON object with a + * string `answer` and an array of string `citedEdgeKeys`. Anything else is a + * hard error — we never guess at a malformed model reply. Grounding (checking the + * cited keys actually exist in the subgraph) happens in the caller. + */ +export function parseRagResponse(raw: string): ParsedRagResponse { + if (typeof raw !== 'string' || raw.trim() === '') { + throw new Error('LLM returned an empty response'); + } + const stripped = raw + .trim() + .replace(/^```(?:json)?\s*/i, '') + .replace(/\s*```$/i, '') + .trim(); + + let parsed: unknown; + try { + parsed = JSON.parse(stripped); + } catch { + throw new Error('LLM response was not valid JSON'); + } + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('LLM response must be a JSON object'); + } + const record = parsed as Record; + if (typeof record.answer !== 'string') { + throw new Error('LLM response is missing a string "answer"'); + } + if (!Array.isArray(record.citedEdgeKeys) || record.citedEdgeKeys.some((key) => typeof key !== 'string')) { + throw new Error('LLM response "citedEdgeKeys" must be an array of strings'); + } + + return { + answer: record.answer.trim(), + citedEdgeKeys: record.citedEdgeKeys as string[], + }; +} diff --git a/gitgram/backend/src/services/talentSubgraphRetrieval.ts b/gitgram/backend/src/services/talentSubgraphRetrieval.ts new file mode 100644 index 00000000..7345b309 --- /dev/null +++ b/gitgram/backend/src/services/talentSubgraphRetrieval.ts @@ -0,0 +1,177 @@ +/** + * k-hop subgraph retrieval for the talent GraphRAG serving layer. + * + * GraphRAG serves *structure*, not prose: before an LLM can ground an answer we + * must first retrieve the bounded neighborhood of facts that are relevant to a + * developer. This module is the retrieval half. It REUSES the existing graph + * pipeline rather than rebuilding it: + * assembled view → deriveTalentEdges → validateEdges → k-hop subgraph + * so every fact that reaches the LLM has already been vetted against the + * ontology's domain/range and carries provenance (source, confidence, observedAt). + * + * Two entry points: + * - `buildSubgraphFromEdges` — the pure BFS core over an already-validated edge + * set. Traversal is undirected (a neighborhood, not a reachability query) so a + * developer's repos, skills, orgs and collaborators are all gathered, but never + * beyond `depth` hops. This is the unit-testable heart of retrieval. + * - `buildTalentSubgraph` — the view-based wrapper that runs the extract→validate + * pipeline before delegating to the BFS core. + * + * Pure and dependency-light: no Prisma, no I/O, immutable output. + */ +import { + deriveTalentEdges, + type TalentGraphView, +} from './talentGraphEdges'; +import { + validateEdges, + type NodeRef, + type TalentEdge, + type TalentNodeType, +} from './talentGraphOntology'; + +/** A node that appears in a retrieved subgraph, tagged with its BFS distance. */ +export interface SubgraphNode { + readonly id: string; + readonly type: TalentNodeType; + /** Hops from the root developer. The root itself is 0. */ + readonly distance: number; +} + +/** A bounded, validated neighborhood of the talent graph around one developer. */ +export interface TalentSubgraph { + readonly root: NodeRef; + readonly depth: number; + readonly nodes: readonly SubgraphNode[]; + readonly edges: readonly TalentEdge[]; +} + +const MAX_DEPTH = 6; + +function nodeKey(type: TalentNodeType, id: string): string { + return `${type}:${id}`; +} + +/** + * Validate the requested traversal depth at the boundary. Retrieval must always + * terminate, so depth is a non-negative integer capped at MAX_DEPTH. Never trust + * a caller-supplied bound. + */ +function normalizeDepth(depth: number): number { + if (typeof depth !== 'number' || !Number.isInteger(depth) || depth < 0) { + throw new RangeError('Subgraph depth must be a non-negative integer'); + } + return Math.min(depth, MAX_DEPTH); +} + +/** + * Retrieve the k-hop neighborhood around `root` from a set of already-validated + * edges. Undirected BFS assigns each reachable node its shortest hop distance; + * nodes farther than `depth` are excluded, and an edge is kept only when BOTH of + * its endpoints made it into the node set. This is what stops "wrong hops": + * facts hanging off a node one step beyond the frontier never enter the context. + * + * The root is always present (distance 0) even when it has no edges, so callers + * always get a well-formed subgraph. Termination is guaranteed: each node is + * enqueued at most once. + */ +export function buildSubgraphFromEdges( + edges: readonly TalentEdge[], + root: NodeRef, + depth: number, +): TalentSubgraph { + if (!root || typeof root.id !== 'string' || root.id.trim() === '') { + throw new Error('Subgraph retrieval requires a root node with a non-empty id'); + } + const maxDepth = normalizeDepth(depth); + const rootRef: NodeRef = { id: root.id, type: root.type }; + + // Adjacency list over an undirected view of the (directed) edge set. + const adjacency = new Map(); + const addAdjacency = (from: NodeRef, to: NodeRef) => { + const key = nodeKey(from.type, from.id); + const list = adjacency.get(key) ?? []; + list.push({ neighbor: to }); + adjacency.set(key, list); + }; + for (const edge of edges) { + addAdjacency(edge.from, edge.to); + addAdjacency(edge.to, edge.from); + } + + // BFS from the root, recording the shortest distance to every node. + const distances = new Map(); + const rootKey = nodeKey(rootRef.type, rootRef.id); + distances.set(rootKey, { id: rootRef.id, type: rootRef.type, distance: 0 }); + let frontier: NodeRef[] = [rootRef]; + + for (let hop = 1; hop <= maxDepth && frontier.length > 0; hop += 1) { + const next: NodeRef[] = []; + for (const node of frontier) { + for (const { neighbor } of adjacency.get(nodeKey(node.type, node.id)) ?? []) { + const key = nodeKey(neighbor.type, neighbor.id); + if (distances.has(key)) continue; + distances.set(key, { id: neighbor.id, type: neighbor.type, distance: hop }); + next.push(neighbor); + } + } + frontier = next; + } + + // Keep only edges whose BOTH endpoints are inside the retrieved node set. + const included = (node: NodeRef) => distances.has(nodeKey(node.type, node.id)); + const keptEdges = edges.filter((edge) => included(edge.from) && included(edge.to)); + + const nodes = Array.from(distances.values()); + return { + root: rootRef, + depth: maxDepth, + nodes: sortNodes(nodes), + edges: sortEdges(keptEdges), + }; +} + +/** + * View-based retrieval entry point. Runs the existing extract→validate pipeline + * (`deriveTalentEdges` + `validateEdges`) so only ontology-valid, provenance- + * carrying edges are traversed, then returns the k-hop subgraph around the given + * developer. `rootDeveloperId` is validated against the view's own profile to keep + * the seam honest — you can only build a subgraph for the developer the view + * describes. + */ +export function buildTalentSubgraph( + view: TalentGraphView, + rootDeveloperId: string, + depth: number, +): TalentSubgraph { + if (!view || typeof view !== 'object') { + throw new Error('buildTalentSubgraph requires an assembled talent graph view'); + } + const developerId = typeof rootDeveloperId === 'string' ? rootDeveloperId.trim() : ''; + if (!developerId) { + throw new Error('buildTalentSubgraph requires a non-empty rootDeveloperId'); + } + if (view.profile?.id !== developerId) { + throw new Error( + `rootDeveloperId "${developerId}" does not match the view profile "${view.profile?.id ?? ''}"`, + ); + } + + const { accepted } = validateEdges(deriveTalentEdges(view)); + return buildSubgraphFromEdges(accepted, { id: developerId, type: 'developer' }, depth); +} + +/** Deterministic node ordering: by distance, then type, then id. */ +function sortNodes(nodes: readonly SubgraphNode[]): SubgraphNode[] { + return [...nodes].sort( + (a, b) => + a.distance - b.distance || + a.type.localeCompare(b.type) || + a.id.localeCompare(b.id), + ); +} + +/** Deterministic edge ordering by canonical key. */ +function sortEdges(edges: readonly TalentEdge[]): TalentEdge[] { + return [...edges].sort((a, b) => a.key.localeCompare(b.key)); +} diff --git a/gitgram/backend/src/services/talentSubgraphSerializer.ts b/gitgram/backend/src/services/talentSubgraphSerializer.ts new file mode 100644 index 00000000..aec07b7e --- /dev/null +++ b/gitgram/backend/src/services/talentSubgraphSerializer.ts @@ -0,0 +1,101 @@ +/** + * Deterministic serialization of a retrieved talent subgraph into LLM-ready + * context. + * + * The grounding contract of GraphRAG is only as strong as the context window is + * faithful: the LLM may answer ONLY from what we hand it, so the serialized block + * must (a) be deterministic — identical subgraph in, byte-identical text out, so + * prompts are cacheable and tests are stable — and (b) expose the citation handle + * (each edge's canonical `key`) the model is required to cite, alongside the + * provenance (source, confidence, observedAt) that makes each fact auditable. + * + * Nodes are grouped by ontology type in a fixed order; edges are rendered as + * subject–predicate–object triples with their provenance and citation key. No + * prose, no I/O — pure formatting. + */ +import { + TALENT_NODE_TYPES, + type TalentEdge, + type TalentNodeType, +} from './talentGraphOntology'; +import type { SubgraphNode, TalentSubgraph } from './talentSubgraphRetrieval'; + +/** Human-facing labels for each node type, kept stable for prompt determinism. */ +const NODE_TYPE_LABELS: Readonly> = { + developer: 'Developers', + repo: 'Repositories', + org: 'Organizations', + skill: 'Skills', + contribution: 'Contributions', +}; + +/** Verb phrasing for each edge type so triples read naturally but stay stable. */ +const EDGE_TYPE_PHRASES: Readonly> = { + contributed_to: 'contributed to', + member_of: 'is a member of', + has_skill: 'has skill', + collaborated_with: 'collaborated with', +}; + +function formatConfidence(value: number): string { + return value.toFixed(2); +} + +function formatNodeLine(node: SubgraphNode): string { + return ` - ${node.type}:${node.id} (depth ${node.distance})`; +} + +function formatEdgeLine(edge: TalentEdge): string { + const phrase = EDGE_TYPE_PHRASES[edge.type] ?? edge.type; + const provenance = + `source=${edge.provenance.source}` + + `, confidence=${formatConfidence(edge.provenance.confidence)}` + + `, observedAt=${edge.provenance.observedAt.toISOString()}` + + `, sourceUrl=${edge.provenance.sourceUrl ?? 'none'}`; + return ( + ` - [${edge.key}] ${edge.from.type}:${edge.from.id} ` + + `${phrase} ${edge.to.type}:${edge.to.id} {${provenance}}` + ); +} + +/** + * Render the subgraph as a deterministic text block for the LLM context window. + * Structure: + * ROOT: : (depth ) + * NODES + * + * EDGES (facts — cite by [key]) + * + * + * The input is assumed already ordered by the retrieval layer; grouping here is a + * stable re-projection, so the output is a pure function of the subgraph. + */ +export function serializeTalentSubgraphForLLM(subgraph: TalentSubgraph): string { + if (!subgraph || typeof subgraph !== 'object') { + throw new Error('serializeTalentSubgraphForLLM requires a subgraph'); + } + + const lines: string[] = []; + lines.push(`ROOT: ${subgraph.root.type}:${subgraph.root.id} (depth ${subgraph.depth})`); + + lines.push('NODES'); + if (subgraph.nodes.length === 0) { + lines.push(' (none)'); + } else { + for (const type of TALENT_NODE_TYPES) { + const ofType = subgraph.nodes.filter((node) => node.type === type); + if (ofType.length === 0) continue; + lines.push(` ${NODE_TYPE_LABELS[type]}:`); + for (const node of ofType) lines.push(` ${formatNodeLine(node)}`); + } + } + + lines.push('EDGES (facts — cite by [key]):'); + if (subgraph.edges.length === 0) { + lines.push(' (none)'); + } else { + for (const edge of subgraph.edges) lines.push(formatEdgeLine(edge)); + } + + return lines.join('\n'); +} diff --git a/gitgram/backend/tests/unit/talentGraphRag.test.ts b/gitgram/backend/tests/unit/talentGraphRag.test.ts new file mode 100644 index 00000000..c1680031 --- /dev/null +++ b/gitgram/backend/tests/unit/talentGraphRag.test.ts @@ -0,0 +1,303 @@ +/** + * Unit tests for the talent GraphRAG serving layer. + * + * Pure (no Prisma, no network): the LLM call and the view loader are both + * injected. These tests pin the three load-bearing guarantees of the layer: + * 1. Retrieval correctness — the k-hop subgraph contains the right nodes at the + * right depth, takes no wrong hops, and always terminates. + * 2. Deterministic serialization — identical subgraph in, byte-identical text. + * 3. Grounding — the answer's evidence only ever references edges that were + * actually retrieved; hallucinated citations are dropped. + */ +import { validateEdge, type TalentEdge } from '../../src/services/talentGraphOntology'; +import { + buildSubgraphFromEdges, + buildTalentSubgraph, +} from '../../src/services/talentSubgraphRetrieval'; +import { serializeTalentSubgraphForLLM } from '../../src/services/talentSubgraphSerializer'; +import { parseRagResponse } from '../../src/services/talentGraphRagPrompt'; +import { + explainTalentMatch, + type ExplainTalentMatchDeps, +} from '../../src/services/talentGraphRag'; +import type { TalentGraphView } from '../../src/services/talentGraphEdges'; + +// --- helpers --------------------------------------------------------------- + +function edge( + type: string, + from: { id: string; type: string }, + to: { id: string; type: string }, + confidence = 1, + source = 'repository_claim', +): TalentEdge { + const result = validateEdge({ + type, + from, + to, + provenance: { source, observedAt: new Date('2026-01-01T00:00:00Z'), confidence, sourceUrl: null }, + }); + if (!result.valid) throw new Error(`test edge invalid: ${result.rejections.map((r) => r.code).join(',')}`); + return result.edge; +} + +const DEV = (id: string) => ({ id, type: 'developer' as const }); +const REPO = (id: string) => ({ id, type: 'repo' as const }); + +/** A multi-hop edge set: dev-1 is the root; repo-2 sits two hops away. */ +function multiHopEdges(): TalentEdge[] { + return [ + edge('contributed_to', DEV('dev-1'), REPO('repo-1')), + edge('has_skill', DEV('dev-1'), { id: 'react', type: 'skill' }), + edge('member_of', DEV('dev-1'), { id: 'org-1', type: 'org' }), + edge('collaborated_with', DEV('dev-1'), DEV('dev-2'), 0.5, 'github_sync'), + // dev-2's own contribution — reachable only at depth 2 from dev-1 + edge('contributed_to', DEV('dev-2'), REPO('repo-2')), + ]; +} + +// --- retrieval correctness ------------------------------------------------- + +describe('buildSubgraphFromEdges (k-hop retrieval)', () => { + it('depth 0 returns only the root, no edges', () => { + const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 0); + expect(sub.nodes).toHaveLength(1); + expect(sub.nodes[0]).toMatchObject({ id: 'dev-1', type: 'developer', distance: 0 }); + expect(sub.edges).toHaveLength(0); + }); + + it('depth 1 includes direct neighbors but takes no wrong hops', () => { + const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 1); + const nodeIds = sub.nodes.map((n) => `${n.type}:${n.id}`).sort(); + expect(nodeIds).toEqual([ + 'developer:dev-1', + 'developer:dev-2', + 'org:org-1', + 'repo:repo-1', + 'skill:react', + ]); + // repo-2 is two hops away — it must NOT appear, and neither must its edge. + expect(nodeIds).not.toContain('repo:repo-2'); + expect(sub.edges.some((e) => e.to.id === 'repo-2' || e.from.id === 'repo-2')).toBe(false); + // every edge kept has both endpoints inside the node set + const present = new Set(nodeIds); + for (const e of sub.edges) { + expect(present.has(`${e.from.type}:${e.from.id}`)).toBe(true); + expect(present.has(`${e.to.type}:${e.to.id}`)).toBe(true); + } + }); + + it('depth 2 expands to the second hop and includes the bridging edge', () => { + const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2); + const repo2 = sub.nodes.find((n) => n.id === 'repo-2'); + expect(repo2).toBeDefined(); + expect(repo2?.distance).toBe(2); + expect(sub.edges.some((e) => e.from.id === 'dev-2' && e.to.id === 'repo-2')).toBe(true); + }); + + it('terminates on cycles and caps runaway depth', () => { + const cyclic = [ + edge('collaborated_with', DEV('dev-1'), DEV('dev-2'), 0.5, 'github_sync'), + edge('collaborated_with', DEV('dev-2'), DEV('dev-3'), 0.5, 'github_sync'), + edge('collaborated_with', DEV('dev-3'), DEV('dev-1'), 0.5, 'github_sync'), + ]; + const sub = buildSubgraphFromEdges(cyclic, DEV('dev-1'), 1000); + // three developers, each visited once; depth is capped, not 1000 + expect(sub.nodes).toHaveLength(3); + expect(sub.depth).toBeLessThanOrEqual(6); + }); + + it('rejects a negative or non-integer depth at the boundary', () => { + expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), -1)).toThrow(); + expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 1.5)).toThrow(); + }); + + it('rejects an empty root id', () => { + expect(() => buildSubgraphFromEdges(multiHopEdges(), DEV(' '), 1)).toThrow(); + }); +}); + +// --- view-based retrieval -------------------------------------------------- + +const VIEW: TalentGraphView = { + profile: { id: 'dev-1' }, + projects: [ + { + id: 'repo-1', + url: 'https://github.com/acme/repo-1', + topics: ['reactjs', 'k8s'], + verificationLevel: 'verified', + lastVerifiedAt: new Date('2026-01-01T00:00:00Z'), + evidence: [{ kind: 'repository_claim', sourceUrl: 'https://github.com/acme/repo-1' }], + }, + ], + collaborators: [ + { id: 'dev-2', sharedProjects: [{ id: 'repo-1', url: 'https://github.com/acme/repo-1' }] }, + ], +}; + +describe('buildTalentSubgraph (view-based)', () => { + it('retrieves the developer neighborhood from an assembled view', () => { + const sub = buildTalentSubgraph(VIEW, 'dev-1', 1); + const types = sub.nodes.map((n) => n.type); + expect(types).toContain('developer'); // root + collaborator + expect(types).toContain('repo'); + expect(types).toContain('skill'); + // canonicalized skills present + expect(sub.nodes.some((n) => n.type === 'skill' && n.id === 'react')).toBe(true); + expect(sub.nodes.some((n) => n.type === 'skill' && n.id === 'kubernetes')).toBe(true); + // every edge carries provenance + for (const e of sub.edges) { + expect(e.provenance.confidence).toBeGreaterThan(0); + expect(e.provenance.observedAt).toBeInstanceOf(Date); + expect(typeof e.provenance.source).toBe('string'); + } + }); + + it('rejects a rootDeveloperId that does not match the view profile', () => { + expect(() => buildTalentSubgraph(VIEW, 'someone-else', 1)).toThrow(); + }); +}); + +// --- deterministic serialization ------------------------------------------ + +describe('serializeTalentSubgraphForLLM', () => { + it('is deterministic: the same subgraph produces byte-identical text', () => { + // Serialization is a pure function of the subgraph, so a single retrieved + // subgraph must serialize identically every time (cacheable prompts). We + // build once and serialize twice; edge order is fixed by the retrieval layer. + const sub = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2); + expect(serializeTalentSubgraphForLLM(sub)).toBe(serializeTalentSubgraphForLLM(sub)); + }); + + it('orders edges canonically regardless of input edge order', () => { + const forward = buildSubgraphFromEdges(multiHopEdges(), DEV('dev-1'), 2); + const shuffled = buildSubgraphFromEdges([...multiHopEdges()].reverse(), DEV('dev-1'), 2); + expect(serializeTalentSubgraphForLLM(forward)).toBe(serializeTalentSubgraphForLLM(shuffled)); + }); + + it('exposes edge citation keys and provenance in the context', () => { + const sub = buildTalentSubgraph(VIEW, 'dev-1', 2); + const text = serializeTalentSubgraphForLLM(sub); + expect(text).toContain('ROOT: developer:dev-1'); + expect(text).toContain('EDGES (facts — cite by [key]):'); + for (const e of sub.edges) { + expect(text).toContain(`[${e.key}]`); + } + expect(text).toMatch(/source=/); + expect(text).toMatch(/confidence=/); + }); + + it('renders an empty subgraph without crashing', () => { + const empty = buildSubgraphFromEdges([], DEV('dev-1'), 1); + const text = serializeTalentSubgraphForLLM(empty); + expect(text).toContain('ROOT: developer:dev-1'); + expect(text).toContain('(none)'); + }); +}); + +// --- response parsing ------------------------------------------------------ + +describe('parseRagResponse', () => { + it('parses a clean JSON object', () => { + const parsed = parseRagResponse('{"answer":"ok","citedEdgeKeys":["k1","k2"]}'); + expect(parsed.answer).toBe('ok'); + expect(parsed.citedEdgeKeys).toEqual(['k1', 'k2']); + }); + + it('tolerates a ```json code fence', () => { + const parsed = parseRagResponse('```json\n{"answer":"x","citedEdgeKeys":[]}\n```'); + expect(parsed.answer).toBe('x'); + }); + + it('throws on non-JSON, wrong shape, or non-string citations', () => { + expect(() => parseRagResponse('not json')).toThrow(); + expect(() => parseRagResponse('[]')).toThrow(); + expect(() => parseRagResponse('{"answer":1,"citedEdgeKeys":[]}')).toThrow(); + expect(() => parseRagResponse('{"answer":"x","citedEdgeKeys":[1]}')).toThrow(); + expect(() => parseRagResponse('')).toThrow(); + }); +}); + +// --- grounding (the core guarantee) --------------------------------------- + +describe('explainTalentMatch (grounding)', () => { + const loadView: ExplainTalentMatchDeps['loadView'] = async () => VIEW; + + it('returns an answer whose evidence only references retrieved edges', async () => { + // Discover the real edge keys the subgraph will contain. + const sub = buildTalentSubgraph(VIEW, 'dev-1', 2); + const realKey = sub.edges[0].key; + const hallucinatedKey = 'collaborated_with|developer:ghost|developer:phantom'; + + const ask: ExplainTalentMatchDeps['ask'] = async (req) => { + // sanity: the grounding contract and the serialized context were passed + expect(req.system).toMatch(/ONLY using the facts/i); + expect(req.context).toContain('ROOT: developer:dev-1'); + return JSON.stringify({ + answer: 'dev-1 matches because of contributed repositories and skills.', + citedEdgeKeys: [realKey, hallucinatedKey], + }); + }; + + const result = await explainTalentMatch('dev-1', 'Why does this developer match React work?', { + ask, + loadView, + depth: 2, + }); + + const subgraphKeys = new Set(sub.edges.map((e) => e.key)); + // INVARIANT: every cited piece of evidence exists in the retrieved subgraph + for (const ev of result.evidence) { + expect(subgraphKeys.has(ev.key)).toBe(true); + } + expect(result.evidence.map((e) => e.key)).toContain(realKey); + // the hallucinated citation was dropped, not returned as evidence + expect(result.evidence.map((e) => e.key)).not.toContain(hallucinatedKey); + expect(result.droppedCitations).toContain(hallucinatedKey); + // evidence carries provenance for auditability + expect(result.evidence[0].source).toBeDefined(); + expect(result.evidence[0].confidence).toBeGreaterThan(0); + expect(result.retrieval.edgeCount).toBe(sub.edges.length); + }); + + it('de-duplicates repeated citations and preserves the answer text', async () => { + const sub = buildTalentSubgraph(VIEW, 'dev-1', 2); + const key = sub.edges[0].key; + const ask: ExplainTalentMatchDeps['ask'] = async () => + JSON.stringify({ answer: 'grounded answer', citedEdgeKeys: [key, key, key] }); + + const result = await explainTalentMatch('dev-1', 'why match', { ask, loadView, depth: 2 }); + expect(result.answer).toBe('grounded answer'); + expect(result.evidence.filter((e) => e.key === key)).toHaveLength(1); + expect(result.droppedCitations).toHaveLength(0); + }); + + it('returns no evidence when the model cites nothing', async () => { + const ask: ExplainTalentMatchDeps['ask'] = async () => + JSON.stringify({ answer: 'insufficient information in the subgraph', citedEdgeKeys: [] }); + const result = await explainTalentMatch('dev-1', 'why match', { ask, loadView }); + expect(result.evidence).toHaveLength(0); + expect(result.answer).toMatch(/insufficient/); + }); + + it('validates inputs and surfaces a missing view', async () => { + const ask: ExplainTalentMatchDeps['ask'] = async () => '{"answer":"x","citedEdgeKeys":[]}'; + await expect(explainTalentMatch('', 'q', { ask, loadView })).rejects.toThrow(/rootDeveloperId/); + await expect(explainTalentMatch('dev-1', ' ', { ask, loadView })).rejects.toThrow(/query/); + await expect( + explainTalentMatch('dev-1', 'q', { ask, loadView: async () => null }), + ).rejects.toThrow(/No talent graph view/); + }); + + it('requires an injected ask function', async () => { + await expect( + explainTalentMatch('dev-1', 'q', { loadView } as unknown as ExplainTalentMatchDeps), + ).rejects.toThrow(/deps.ask/); + }); + + it('propagates a malformed LLM reply as an explicit error', async () => { + const ask: ExplainTalentMatchDeps['ask'] = async () => 'garbage not json'; + await expect(explainTalentMatch('dev-1', 'q', { ask, loadView })).rejects.toThrow(/JSON/); + }); +});