Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as opaque submodule gitlinks on GitHub) into fully-tracked first-party content. AITM now merges its 7 vendored repos as first-party engines: - memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts - git: vendored C git (bundled binary) + gitnexus (in-process code intelligence) - agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess) Includes the full session's product build: Gitgram web/mobile/backend features, AITM website/CLI/MCP, the unified memory layer, engine build tooling (scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md), and extensive test suites. Build artifacts, node_modules, external/, and gemini-cli are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
180 lines
5.0 KiB
TypeScript
180 lines
5.0 KiB
TypeScript
import { Response } from 'express';
|
|
import axios from 'axios';
|
|
import { AuthRequest } from '../middleware/auth';
|
|
import { asyncHandler, Errors } from '../middleware/errorHandler';
|
|
import logger from '../utils/logger';
|
|
|
|
const ALLOWED_EXTENSIONS = new Set([
|
|
'.ts', '.tsx', '.js', '.jsx', '.py', '.go', '.rs', '.md', '.json',
|
|
]);
|
|
|
|
const MAX_FILES = 100;
|
|
const MAX_FILE_CHARS = 50000;
|
|
|
|
interface GitHubTreeItem {
|
|
path: string;
|
|
type: string;
|
|
url: string;
|
|
size?: number;
|
|
}
|
|
|
|
interface GitHubTreeResponse {
|
|
tree: GitHubTreeItem[];
|
|
truncated: boolean;
|
|
}
|
|
|
|
interface GitHubContentResponse {
|
|
content?: string;
|
|
encoding?: string;
|
|
download_url?: string;
|
|
}
|
|
|
|
function getExtension(filePath: string): string {
|
|
const dotIndex = filePath.lastIndexOf('.');
|
|
if (dotIndex === -1) return '';
|
|
return filePath.slice(dotIndex).toLowerCase();
|
|
}
|
|
|
|
function parseGitHubUrl(url: string): { owner: string; repo: string } | null {
|
|
try {
|
|
const parsed = new URL(url);
|
|
if (parsed.hostname !== 'github.com') return null;
|
|
const parts = parsed.pathname.replace(/^\//, '').replace(/\/$/, '').split('/');
|
|
if (parts.length < 2 || !parts[0] || !parts[1]) return null;
|
|
return { owner: parts[0], repo: parts[1] };
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function buildAuthHeaders(): Record<string, string> {
|
|
const headers: Record<string, string> = {
|
|
Accept: 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version': '2022-11-28',
|
|
};
|
|
if (process.env.GITHUB_TOKEN) {
|
|
headers['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`;
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
/**
|
|
* Convert a repository to a single XML text blob
|
|
* POST /api/v1/tools/repo-to-text
|
|
*/
|
|
export const repoToTextHandler = asyncHandler(async (req: AuthRequest, res: Response) => {
|
|
const { repoUrl } = req.body;
|
|
|
|
if (!repoUrl || typeof repoUrl !== 'string') {
|
|
throw Errors.badRequest('repoUrl is required');
|
|
}
|
|
|
|
// Validate GitHub URL
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(repoUrl);
|
|
} catch {
|
|
throw Errors.badRequest('repoUrl must be a valid URL');
|
|
}
|
|
|
|
if (parsed.hostname !== 'github.com') {
|
|
throw Errors.badRequest('repoUrl must be a GitHub URL (github.com)');
|
|
}
|
|
|
|
const repoInfo = parseGitHubUrl(repoUrl);
|
|
if (!repoInfo) {
|
|
throw Errors.badRequest('Could not parse owner/repo from repoUrl');
|
|
}
|
|
|
|
const { owner, repo } = repoInfo;
|
|
const headers = buildAuthHeaders();
|
|
|
|
// Fetch the git tree
|
|
let treeData: GitHubTreeResponse;
|
|
try {
|
|
const treeResponse = await axios.get<GitHubTreeResponse>(
|
|
`https://api.github.com/repos/${owner}/${repo}/git/trees/HEAD?recursive=1`,
|
|
{ headers }
|
|
);
|
|
treeData = treeResponse.data;
|
|
} catch (err: unknown) {
|
|
if (axios.isAxiosError(err)) {
|
|
const status = err.response?.status;
|
|
if (status === 404) {
|
|
throw Errors.notFound(`Repository ${owner}/${repo}`);
|
|
}
|
|
if (status === 403 || status === 429) {
|
|
throw Errors.badRequest('GitHub API rate limit exceeded or access forbidden');
|
|
}
|
|
}
|
|
logger.error('Failed to fetch GitHub tree', err);
|
|
throw Errors.badRequest('Failed to fetch repository tree from GitHub');
|
|
}
|
|
|
|
// Filter to allowed file extensions, blob only
|
|
const filteredFiles = treeData.tree
|
|
.filter(
|
|
(item) =>
|
|
item.type === 'blob' &&
|
|
item.path &&
|
|
ALLOWED_EXTENSIONS.has(getExtension(item.path))
|
|
)
|
|
.slice(0, MAX_FILES);
|
|
|
|
// Fetch file contents in parallel
|
|
const fileResults = await Promise.all(
|
|
filteredFiles.map(async (item) => {
|
|
try {
|
|
const contentResponse = await axios.get<GitHubContentResponse>(
|
|
`https://api.github.com/repos/${owner}/${repo}/contents/${item.path}`,
|
|
{ headers }
|
|
);
|
|
|
|
const data = contentResponse.data;
|
|
let content = '';
|
|
|
|
if (data.encoding === 'base64' && data.content) {
|
|
const decoded = Buffer.from(data.content.replace(/\n/g, ''), 'base64').toString('utf-8');
|
|
content = decoded.slice(0, MAX_FILE_CHARS);
|
|
} else if (data.download_url) {
|
|
const rawResponse = await axios.get<string>(data.download_url, {
|
|
responseType: 'text',
|
|
});
|
|
content = (rawResponse.data as string).slice(0, MAX_FILE_CHARS);
|
|
}
|
|
|
|
return { path: item.path, content };
|
|
} catch (err: unknown) {
|
|
logger.error(`Failed to fetch file content for ${item.path}`, err);
|
|
return { path: item.path, content: '' };
|
|
}
|
|
})
|
|
);
|
|
|
|
// Build XML output
|
|
const xmlParts: string[] = [`<repository url="${repoUrl}">`];
|
|
for (const { path, content } of fileResults) {
|
|
// Escape XML special characters in content
|
|
const escaped = content
|
|
.replace(/&/g, '&')
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>');
|
|
xmlParts.push(` <file path="${path}">${escaped}</file>`);
|
|
}
|
|
xmlParts.push('</repository>');
|
|
|
|
const result = xmlParts.join('\n');
|
|
const estimatedTokens = Math.round(result.length / 4);
|
|
|
|
res.json({
|
|
success: true,
|
|
xml: result,
|
|
fileCount: fileResults.length,
|
|
estimatedTokens,
|
|
});
|
|
});
|
|
|
|
export default {
|
|
repoToTextHandler,
|
|
};
|