diff --git a/gitgram/backend/.env.example b/gitgram/backend/.env.example index 48aca007..062cde48 100644 --- a/gitgram/backend/.env.example +++ b/gitgram/backend/.env.example @@ -159,3 +159,13 @@ APP_URL=http://localhost:3000 # BACKUP_DIR= # Comma-separated local feature flags # FEATURE_FLAGS= + +# ── Talent GraphRAG (explainMine / explainPublic) ─────────────────────────── +# Railyard Relay (OpenAI-compatible LLM gateway), tried first when both are set. +# RELAY_URL= +# RELAY_API_KEY= +# Direct fallback when Relay isn't configured or a Relay call fails. +# OPENROUTER_API_KEY= +# Model overrides (optional; sane defaults are used otherwise). +# GITGRAM_RAG_RELAY_MODEL=claude-haiku-4-5-20251001 +# GITGRAM_RAG_DIRECT_MODEL=anthropic/claude-haiku-4.5 diff --git a/gitgram/backend/src/controllers/talentGraphController.ts b/gitgram/backend/src/controllers/talentGraphController.ts index 97cc9578..8f9234db 100644 --- a/gitgram/backend/src/controllers/talentGraphController.ts +++ b/gitgram/backend/src/controllers/talentGraphController.ts @@ -6,14 +6,17 @@ import { analyzeNeedCoverage } from '../services/talentGapAnalysisService'; import prisma from '../utils/prisma'; import { normalizePublicHttpUrl } from '../utils/publicHttpUrl'; import { runTalentMutation, type TalentMutationResult } from '../services/talentMutationService'; -import { explainTalentMatch, defaultRagAsk } from '../services/talentGraphRag'; +import { explainTalentMatch } from '../services/talentGraphRag'; +import { liveRagAsk } from '../services/talentGraphLlmClient'; import type { TalentGraphView } from '../services/talentGraphEdges'; /** * Run a grounded GraphRAG explanation and translate the module's plain errors * into the API's typed errors. The LLM seam (`ask`) is injected here so the - * controller stays a thin adapter over the testable service; until a real client - * is wired, `defaultRagAsk` surfaces a clear 503 rather than a fake answer. + * controller stays a thin adapter over the testable service. `liveRagAsk` + * tries Railyard Relay first, falls back to a direct OpenRouter call, and + * throws the same "No LLM client" message (surfaced as a 503) when neither + * is configured. */ async function runGroundedExplain( developerId: string, @@ -22,7 +25,7 @@ async function runGroundedExplain( ) { const cleanedQuery = boundedText(query, 'query', 2000); try { - return await explainTalentMatch(developerId, cleanedQuery, { ask: defaultRagAsk, loadView }); + return await explainTalentMatch(developerId, cleanedQuery, { ask: liveRagAsk, loadView }); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (message.includes('No LLM client')) throw Errors.serviceUnavailable('Talent GraphRAG'); diff --git a/gitgram/backend/src/services/talentGraphLlmClient.ts b/gitgram/backend/src/services/talentGraphLlmClient.ts new file mode 100644 index 00000000..cec1964d --- /dev/null +++ b/gitgram/backend/src/services/talentGraphLlmClient.ts @@ -0,0 +1,115 @@ +/** + * Real LLM seam for talent GraphRAG's `explainTalentMatch`. + * + * Relay-first: Railyard Relay (our TensorZero fork) serves Anthropic models + * through an OpenAI-compatible endpoint, billed to our own Anthropic credits + * with a per-key spend cap. Falls back to a direct OpenRouter call on any + * Relay failure (network, non-2xx, bad JSON) or when Relay isn't configured. + * Both paths speak the same OpenAI-compatible chat/completions shape, so one + * request builder and one response parser cover both. + * + * Env-gated, not hardcoded: `RELAY_URL`+`RELAY_API_KEY` enable the Relay path, + * `OPENROUTER_API_KEY` enables the direct fallback. Neither configured -> + * throws the same "No LLM client is configured" message `defaultRagAsk` does, + * so callers can't silently get a different error shape. + */ +import logger from '../utils/logger'; +import type { RagAsk, RagAskRequest } from './talentGraphRagPrompt'; + +const RELAY_CHAT_PATH = '/openai/v1/chat/completions'; +const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions'; +const RELAY_MODEL_PREFIX = 'tensorzero::model_name::anthropic::'; +const DEFAULT_RELAY_MODEL = 'claude-haiku-4-5-20251001'; +const DEFAULT_DIRECT_MODEL = 'anthropic/claude-haiku-4.5'; +const DEFAULT_TIMEOUT_MS = 60_000; +const MAX_TOKENS = 1024; + +interface ChatCompletionConfig { + readonly endpoint: string; + readonly apiKey: string; + readonly model: string; +} + +function toChatBody(request: RagAskRequest, model: string): Record { + return { + model, + max_tokens: MAX_TOKENS, + temperature: 0, + messages: [ + { role: 'system', content: request.system }, + { role: 'user', content: `${request.context}\n\nQUESTION: ${request.query}` }, + ], + }; +} + +async function callChatCompletions(config: ChatCompletionConfig, body: Record): Promise { + const response = await fetch(config.endpoint, { + method: 'POST', + headers: { + authorization: `Bearer ${config.apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS), + }); + if (!response.ok) { + const detail = (await response.text().catch(() => '')).slice(0, 200); + throw new Error(`LLM endpoint ${config.endpoint} answered ${response.status}: ${detail}`); + } + const json = (await response.json()) as { + choices?: { message?: { content?: unknown } }[]; + }; + const text = json.choices?.[0]?.message?.content; + if (typeof text !== 'string' || !text.trim()) { + throw new Error('LLM endpoint returned no message content'); + } + return text; +} + +function relayConfig(): ChatCompletionConfig | null { + const url = process.env.RELAY_URL?.trim(); + const apiKey = process.env.RELAY_API_KEY?.trim(); + if (!url || !apiKey) return null; + const model = process.env.GITGRAM_RAG_RELAY_MODEL?.trim() || DEFAULT_RELAY_MODEL; + return { + endpoint: new URL(RELAY_CHAT_PATH, url).toString(), + apiKey, + model: `${RELAY_MODEL_PREFIX}${model}`, + }; +} + +function directConfig(): ChatCompletionConfig | null { + const apiKey = process.env.OPENROUTER_API_KEY?.trim(); + if (!apiKey) return null; + const model = process.env.GITGRAM_RAG_DIRECT_MODEL?.trim() || DEFAULT_DIRECT_MODEL; + return { endpoint: OPENROUTER_URL, apiKey, model }; +} + +/** + * Real LLM client: Relay first, falls back to a direct OpenRouter call on any + * Relay failure or when Relay isn't configured. Exactly the `RagAsk` seam + * `explainTalentMatch` expects — raw prompt in, raw model text out; grounding + * enforcement happens in the caller, not here. + */ +export const liveRagAsk: RagAsk = async (request) => { + const relay = relayConfig(); + const direct = directConfig(); + if (!relay && !direct) { + throw new Error( + 'No LLM client is configured for talent GraphRAG. Set RELAY_URL/RELAY_API_KEY or OPENROUTER_API_KEY.', + ); + } + if (relay) { + try { + return await callChatCompletions(relay, toChatBody(request, relay.model)); + } catch (error) { + const reason = error instanceof Error ? error.message.slice(0, 160) : String(error); + if (!direct) throw error; + logger.warn('talent-graphrag relay call failed, falling back to direct provider', { + event: 'talent_graph.rag.relay_fallback', + reason, + }); + } + } + return callChatCompletions(direct as ChatCompletionConfig, toChatBody(request, (direct as ChatCompletionConfig).model)); +};