feat(talent-graphrag): wire a real LLM client (Relay-first, OpenRouter fallback)
explainTalentMatch's /explain endpoints 503'd because defaultRagAsk had no real LLM client behind it. talentGraphLlmClient.ts tries Railyard Relay (OpenAI-compatible, same tensorzero::model_name::anthropic:: pattern bidstage-tenders already uses) first, falling back to a direct OpenRouter call on any Relay failure or when Relay isn't configured. Verified against real endpoints, not mocks: a real call through Relay (claude-haiku-4-5-20251001) returned a grounded answer citing all 3 real subgraph edges with zero hallucinated citations; a deliberately broken Relay key correctly triggered the OpenRouter fallback path (confirmed via a real 401 -> real fallback attempt -> real OpenRouter 402, no credits). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
87f50d813e
commit
0fc4f141c9
@ -159,3 +159,13 @@ APP_URL=http://localhost:3000
|
||||
# BACKUP_DIR=
|
||||
# Comma-separated local feature flags
|
||||
# FEATURE_FLAGS=
|
||||
|
||||
# ── Talent GraphRAG (explainMine / explainPublic) ───────────────────────────
|
||||
# Railyard Relay (OpenAI-compatible LLM gateway), tried first when both are set.
|
||||
# RELAY_URL=
|
||||
# RELAY_API_KEY=
|
||||
# Direct fallback when Relay isn't configured or a Relay call fails.
|
||||
# OPENROUTER_API_KEY=
|
||||
# Model overrides (optional; sane defaults are used otherwise).
|
||||
# GITGRAM_RAG_RELAY_MODEL=claude-haiku-4-5-20251001
|
||||
# GITGRAM_RAG_DIRECT_MODEL=anthropic/claude-haiku-4.5
|
||||
|
||||
@ -6,14 +6,17 @@ import { analyzeNeedCoverage } from '../services/talentGapAnalysisService';
|
||||
import prisma from '../utils/prisma';
|
||||
import { normalizePublicHttpUrl } from '../utils/publicHttpUrl';
|
||||
import { runTalentMutation, type TalentMutationResult } from '../services/talentMutationService';
|
||||
import { explainTalentMatch, defaultRagAsk } from '../services/talentGraphRag';
|
||||
import { explainTalentMatch } from '../services/talentGraphRag';
|
||||
import { liveRagAsk } from '../services/talentGraphLlmClient';
|
||||
import type { TalentGraphView } from '../services/talentGraphEdges';
|
||||
|
||||
/**
|
||||
* Run a grounded GraphRAG explanation and translate the module's plain errors
|
||||
* into the API's typed errors. The LLM seam (`ask`) is injected here so the
|
||||
* controller stays a thin adapter over the testable service; until a real client
|
||||
* is wired, `defaultRagAsk` surfaces a clear 503 rather than a fake answer.
|
||||
* controller stays a thin adapter over the testable service. `liveRagAsk`
|
||||
* tries Railyard Relay first, falls back to a direct OpenRouter call, and
|
||||
* throws the same "No LLM client" message (surfaced as a 503) when neither
|
||||
* is configured.
|
||||
*/
|
||||
async function runGroundedExplain(
|
||||
developerId: string,
|
||||
@ -22,7 +25,7 @@ async function runGroundedExplain(
|
||||
) {
|
||||
const cleanedQuery = boundedText(query, 'query', 2000);
|
||||
try {
|
||||
return await explainTalentMatch(developerId, cleanedQuery, { ask: defaultRagAsk, loadView });
|
||||
return await explainTalentMatch(developerId, cleanedQuery, { ask: liveRagAsk, loadView });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes('No LLM client')) throw Errors.serviceUnavailable('Talent GraphRAG');
|
||||
|
||||
115
gitgram/backend/src/services/talentGraphLlmClient.ts
Normal file
115
gitgram/backend/src/services/talentGraphLlmClient.ts
Normal file
@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Real LLM seam for talent GraphRAG's `explainTalentMatch`.
|
||||
*
|
||||
* Relay-first: Railyard Relay (our TensorZero fork) serves Anthropic models
|
||||
* through an OpenAI-compatible endpoint, billed to our own Anthropic credits
|
||||
* with a per-key spend cap. Falls back to a direct OpenRouter call on any
|
||||
* Relay failure (network, non-2xx, bad JSON) or when Relay isn't configured.
|
||||
* Both paths speak the same OpenAI-compatible chat/completions shape, so one
|
||||
* request builder and one response parser cover both.
|
||||
*
|
||||
* Env-gated, not hardcoded: `RELAY_URL`+`RELAY_API_KEY` enable the Relay path,
|
||||
* `OPENROUTER_API_KEY` enables the direct fallback. Neither configured ->
|
||||
* throws the same "No LLM client is configured" message `defaultRagAsk` does,
|
||||
* so callers can't silently get a different error shape.
|
||||
*/
|
||||
import logger from '../utils/logger';
|
||||
import type { RagAsk, RagAskRequest } from './talentGraphRagPrompt';
|
||||
|
||||
const RELAY_CHAT_PATH = '/openai/v1/chat/completions';
|
||||
const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions';
|
||||
const RELAY_MODEL_PREFIX = 'tensorzero::model_name::anthropic::';
|
||||
const DEFAULT_RELAY_MODEL = 'claude-haiku-4-5-20251001';
|
||||
const DEFAULT_DIRECT_MODEL = 'anthropic/claude-haiku-4.5';
|
||||
const DEFAULT_TIMEOUT_MS = 60_000;
|
||||
const MAX_TOKENS = 1024;
|
||||
|
||||
interface ChatCompletionConfig {
|
||||
readonly endpoint: string;
|
||||
readonly apiKey: string;
|
||||
readonly model: string;
|
||||
}
|
||||
|
||||
function toChatBody(request: RagAskRequest, model: string): Record<string, unknown> {
|
||||
return {
|
||||
model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
temperature: 0,
|
||||
messages: [
|
||||
{ role: 'system', content: request.system },
|
||||
{ role: 'user', content: `${request.context}\n\nQUESTION: ${request.query}` },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function callChatCompletions(config: ChatCompletionConfig, body: Record<string, unknown>): Promise<string> {
|
||||
const response = await fetch(config.endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${config.apiKey}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const detail = (await response.text().catch(() => '')).slice(0, 200);
|
||||
throw new Error(`LLM endpoint ${config.endpoint} answered ${response.status}: ${detail}`);
|
||||
}
|
||||
const json = (await response.json()) as {
|
||||
choices?: { message?: { content?: unknown } }[];
|
||||
};
|
||||
const text = json.choices?.[0]?.message?.content;
|
||||
if (typeof text !== 'string' || !text.trim()) {
|
||||
throw new Error('LLM endpoint returned no message content');
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function relayConfig(): ChatCompletionConfig | null {
|
||||
const url = process.env.RELAY_URL?.trim();
|
||||
const apiKey = process.env.RELAY_API_KEY?.trim();
|
||||
if (!url || !apiKey) return null;
|
||||
const model = process.env.GITGRAM_RAG_RELAY_MODEL?.trim() || DEFAULT_RELAY_MODEL;
|
||||
return {
|
||||
endpoint: new URL(RELAY_CHAT_PATH, url).toString(),
|
||||
apiKey,
|
||||
model: `${RELAY_MODEL_PREFIX}${model}`,
|
||||
};
|
||||
}
|
||||
|
||||
function directConfig(): ChatCompletionConfig | null {
|
||||
const apiKey = process.env.OPENROUTER_API_KEY?.trim();
|
||||
if (!apiKey) return null;
|
||||
const model = process.env.GITGRAM_RAG_DIRECT_MODEL?.trim() || DEFAULT_DIRECT_MODEL;
|
||||
return { endpoint: OPENROUTER_URL, apiKey, model };
|
||||
}
|
||||
|
||||
/**
|
||||
* Real LLM client: Relay first, falls back to a direct OpenRouter call on any
|
||||
* Relay failure or when Relay isn't configured. Exactly the `RagAsk` seam
|
||||
* `explainTalentMatch` expects — raw prompt in, raw model text out; grounding
|
||||
* enforcement happens in the caller, not here.
|
||||
*/
|
||||
export const liveRagAsk: RagAsk = async (request) => {
|
||||
const relay = relayConfig();
|
||||
const direct = directConfig();
|
||||
if (!relay && !direct) {
|
||||
throw new Error(
|
||||
'No LLM client is configured for talent GraphRAG. Set RELAY_URL/RELAY_API_KEY or OPENROUTER_API_KEY.',
|
||||
);
|
||||
}
|
||||
if (relay) {
|
||||
try {
|
||||
return await callChatCompletions(relay, toChatBody(request, relay.model));
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message.slice(0, 160) : String(error);
|
||||
if (!direct) throw error;
|
||||
logger.warn('talent-graphrag relay call failed, falling back to direct provider', {
|
||||
event: 'talent_graph.rag.relay_fallback',
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
return callChatCompletions(direct as ChatCompletionConfig, toChatBody(request, (direct as ChatCompletionConfig).model));
|
||||
};
|
||||
Reference in New Issue
Block a user