From be50a0dd88fcefed11e45c2f22aebf8212f333e7 Mon Sep 17 00:00:00 2001 From: caelum0x Date: Mon, 22 Jun 2026 17:45:56 +0300 Subject: [PATCH] lets begin --- .claude/mcp-servers.json | 28 + .env.aitm-api.example | 56 ++ .env.aitm-website.example | 13 + .env.gitgram-web.example | 7 + .env.gitgram.example | 72 ++ .env.postgres.example | 5 + .eslintrc.js | 19 + .github/workflows/ci.yml | 247 ++++++ .github/workflows/integration-tests.yml | 217 +++++ .github/workflows/license-check.yml | 44 + .github/workflows/migrations.yml | 206 +++++ .github/workflows/openapi-validation.yml | 130 +++ .github/workflows/smoke-tests.yml | 240 ++++++ .github/workflows/tests.yml | 193 +++++ .github/workflows/verify-production-db.yml | 214 +++++ .gitignore | 64 ++ .prettierrc | 9 + AUDIT_CHECKLIST.md | 84 ++ CANONICAL_SQL_SCHEMA_V1.sql | 450 ++++++++++ INTEGRATION_PLAN.md | 226 +++++ NORTH_STAR_PRD_ARCHITECTURE_V1.md | 242 ++++++ PLAN.md | 163 ++++ PRIORITIZED_TICKETS_V1.md | 307 +++++++ README.md | 82 ++ REMAINING.md | 146 ++++ SINGLE_SOURCE_PLAN.md | 953 +++++++++++++++++++++ V1_90_DAY_BACKLOG.md | 112 +++ V1_90_DAY_GITHUB_ISSUES.md | 474 ++++++++++ aitimemachine | 1 + deploy/DEPLOYMENT.md | 18 + deploy/backup.sh | 58 ++ deploy/deploy.sh | 47 + deploy/logs.sh | 25 + deploy/nginx/sites.conf | 136 +++ deploy/setup-hetzner.sh | 130 +++ docker-compose.prod.yml | 191 +++++ docker-compose.yml | 338 ++++++++ docs/AGENTS.md | 591 +++++++++++++ docs/can be fork.md | 59 ++ docs/deploy-pipeline.md | 95 ++ docs/environment-matrix.md | 172 ++++ docs/launch-secrets-checklist.md | 133 +++ docs/ops-quickref.md | 150 ++++ docs/polar-payment-setup.md | 112 +++ docs/runbook.md | 569 ++++++++++++ docs/secrets-contract.md | 195 +++++ external/README.md | 22 + external/a2a-js | 1 + external/beads | 1 + external/bluesky-social-app | 1 + external/context7 | 1 + external/gh-social | 1 + external/gitnexus | 1 + external/munia | 1 + external/nextjs-15-social-media-app | 1 + external/pal-mcp-server | 1 + external/polar | 1 + external/proximity | 1 + external/repo-to-text | 1 + external/repo2txt | 1 + external/t3-twitter-clone | 1 + gitgram | 1 + package.json | 80 ++ scripts/analyze-with-gitnexus.sh | 117 +++ scripts/deploy-gitgram.sh | 8 + scripts/deploy-launch.sh | 222 +++++ scripts/dev.sh | 237 +++++ scripts/setup-aitm-tools.sh | 216 +++++ scripts/smoke-local-launch.sh | 113 +++ scripts/start-all-dev.sh | 145 ++++ scripts/validate-openapi-routes.ts | 423 +++++++++ scripts/validate-openapi-spec.ts | 498 +++++++++++ scripts/validate-prod-secrets.sh | 382 +++++++++ scripts/verify-gitgram-db.ts | 463 ++++++++++ scripts/verify-launch-readiness.sh | 185 ++++ scripts/verify-neon.ts | 415 +++++++++ scripts/verify-public-launch.sh | 39 + 77 files changed, 11603 insertions(+) create mode 100644 .claude/mcp-servers.json create mode 100644 .env.aitm-api.example create mode 100644 .env.aitm-website.example create mode 100644 .env.gitgram-web.example create mode 100644 .env.gitgram.example create mode 100644 .env.postgres.example create mode 100644 .eslintrc.js create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/integration-tests.yml create mode 100644 .github/workflows/license-check.yml create mode 100644 .github/workflows/migrations.yml create mode 100644 .github/workflows/openapi-validation.yml create mode 100644 .github/workflows/smoke-tests.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .github/workflows/verify-production-db.yml create mode 100644 .gitignore create mode 100644 .prettierrc create mode 100644 AUDIT_CHECKLIST.md create mode 100644 CANONICAL_SQL_SCHEMA_V1.sql create mode 100644 INTEGRATION_PLAN.md create mode 100644 NORTH_STAR_PRD_ARCHITECTURE_V1.md create mode 100644 PLAN.md create mode 100644 PRIORITIZED_TICKETS_V1.md create mode 100644 README.md create mode 100644 REMAINING.md create mode 100644 SINGLE_SOURCE_PLAN.md create mode 100644 V1_90_DAY_BACKLOG.md create mode 100644 V1_90_DAY_GITHUB_ISSUES.md create mode 160000 aitimemachine create mode 100644 deploy/DEPLOYMENT.md create mode 100755 deploy/backup.sh create mode 100755 deploy/deploy.sh create mode 100755 deploy/logs.sh create mode 100644 deploy/nginx/sites.conf create mode 100755 deploy/setup-hetzner.sh create mode 100644 docker-compose.prod.yml create mode 100644 docker-compose.yml create mode 100644 docs/AGENTS.md create mode 100644 docs/can be fork.md create mode 100644 docs/deploy-pipeline.md create mode 100644 docs/environment-matrix.md create mode 100644 docs/launch-secrets-checklist.md create mode 100644 docs/ops-quickref.md create mode 100644 docs/polar-payment-setup.md create mode 100644 docs/runbook.md create mode 100644 docs/secrets-contract.md create mode 100644 external/README.md create mode 160000 external/a2a-js create mode 160000 external/beads create mode 160000 external/bluesky-social-app create mode 160000 external/context7 create mode 160000 external/gh-social create mode 160000 external/gitnexus create mode 160000 external/munia create mode 160000 external/nextjs-15-social-media-app create mode 160000 external/pal-mcp-server create mode 160000 external/polar create mode 160000 external/proximity create mode 160000 external/repo-to-text create mode 160000 external/repo2txt create mode 160000 external/t3-twitter-clone create mode 160000 gitgram create mode 100644 package.json create mode 100755 scripts/analyze-with-gitnexus.sh create mode 100755 scripts/deploy-gitgram.sh create mode 100755 scripts/deploy-launch.sh create mode 100755 scripts/dev.sh create mode 100755 scripts/setup-aitm-tools.sh create mode 100755 scripts/smoke-local-launch.sh create mode 100755 scripts/start-all-dev.sh create mode 100644 scripts/validate-openapi-routes.ts create mode 100644 scripts/validate-openapi-spec.ts create mode 100755 scripts/validate-prod-secrets.sh create mode 100644 scripts/verify-gitgram-db.ts create mode 100755 scripts/verify-launch-readiness.sh create mode 100644 scripts/verify-neon.ts create mode 100755 scripts/verify-public-launch.sh diff --git a/.claude/mcp-servers.json b/.claude/mcp-servers.json new file mode 100644 index 00000000..3bb65684 --- /dev/null +++ b/.claude/mcp-servers.json @@ -0,0 +1,28 @@ +{ + "description": "AITM MCP server configuration for Claude Code in this monorepo", + "servers": { + "aitm": { + "type": "stdio", + "command": "node", + "args": ["/Users/arhansubasi/gitgram-ai/aitimemachine/mcp/dist/index.js"], + "env": { + "GITNEXUS_URL": "http://localhost:3001", + "AITM_GITNEXUS_ENABLED": "1", + "AITM_A2A_AGENTS": "[]" + }, + "description": "AITimeMachine MCP — checkpoints, restore, beads DAG, gitnexus code intelligence, A2A agent protocol" + }, + "gitnexus": { + "type": "stdio", + "command": "node", + "args": ["/Users/arhansubasi/gitgram-ai/aitimemachine/gitnexus/dist/cli/index.js", "mcp"], + "description": "GitNexus — code knowledge graph: query, impact, context, trace, taint analysis" + }, + "context7": { + "type": "stdio", + "command": "node", + "args": ["/Users/arhansubasi/gitgram-ai/aitimemachine/context7/packages/mcp/dist/index.js"], + "description": "Context7 — fetch up-to-date library documentation for any npm/PyPI package" + } + } +} diff --git a/.env.aitm-api.example b/.env.aitm-api.example new file mode 100644 index 00000000..5894c555 --- /dev/null +++ b/.env.aitm-api.example @@ -0,0 +1,56 @@ +# AITM API (manual launch template) +NODE_ENV=production +HOST=127.0.0.1 +PORT=3001 + +# Database +DATABASE_URL=postgresql://user:password@project.neon.tech/aitm?sslmode=require + +# Networking / internal auth +INTERNAL_SECRET=CHANGE_ME_INTERNAL_SECRET +CORS_ORIGIN=https://timemachine.gitgram.app +WEBSITE_URL=https://timemachine.gitgram.app + +# Billing / email / monitoring +# Use a sandbox organization access token for staging/testing. +# Polar sandbox is isolated from production. See docs/polar-payment-setup.md. +POLAR_ACCESS_TOKEN= +POLAR_MODE=production +POLAR_WEBHOOK_SECRET= +POLAR_PRO_PRODUCT_ID= +POLAR_TEAM_PRODUCT_ID= +RESEND_API_KEY= +SENTRY_DSN= + +# Optional integrations +CLERK_WEBHOOK_SECRET= +REDIS_URL= +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +VOYAGE_API_KEY= +BRAVE_SEARCH_API_KEY= + +# Managed workers +ENABLE_WORKERS=true +DELETE_WORKER_ENABLED=true +RETENTION_WORKER_ENABLED=true +GC_WORKER_ENABLED=true + +# GitNexus integration +GITNEXUS_URL=http://localhost:3001 +# A2A protocol +AITM_A2A_AGENTS=[] +AITM_API_URL=http://localhost:3001 + +# Workspace +WORKSPACE_PATH=/path/to/your/project + +# Beads task tracker binary +BEADS_BINARY=/path/to/aitimemachine/beads/bd + +# Gitgram bridge (auto-post AITM events to Gitgram feed) +GITGRAM_API_URL=http://localhost:4000 +GITGRAM_API_KEY= + +# PAL MCP Python execution server +PAL_MCP_URL=http://localhost:3200 diff --git a/.env.aitm-website.example b/.env.aitm-website.example new file mode 100644 index 00000000..57caa6d4 --- /dev/null +++ b/.env.aitm-website.example @@ -0,0 +1,13 @@ +# AITM website (optional self-hosted dashboard) +NEXT_PUBLIC_API_URL=https://timemachine-api.gitgram.app +API_URL=https://timemachine-api.gitgram.app +INTERNAL_SECRET=CHANGE_ME_INTERNAL_SECRET +NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY= +CLERK_SECRET_KEY= +NEXT_PUBLIC_CLERK_SIGN_IN_URL=/sign-in +NEXT_PUBLIC_CLERK_SIGN_UP_URL=/sign-up +NEXT_PUBLIC_APP_URL=https://timemachine.gitgram.app +POLAR_ACCESS_TOKEN= +POLAR_MODE=production +NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID= +NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID= diff --git a/.env.gitgram-web.example b/.env.gitgram-web.example new file mode 100644 index 00000000..3de9cea9 --- /dev/null +++ b/.env.gitgram-web.example @@ -0,0 +1,7 @@ +# Gitgram web (manual launch template) +NEXT_PUBLIC_API_URL=https://gitgram.app/api +NEXT_PUBLIC_WS_URL=wss://gitgram.app +NEXT_PUBLIC_GITHUB_CLIENT_ID= +NEXT_PUBLIC_GOOGLE_CLIENT_ID= +NEXT_PUBLIC_AITM_API_URL=https://timemachine-api.gitgram.app/v1 +NEXT_PUBLIC_SENTRY_DSN= diff --git a/.env.gitgram.example b/.env.gitgram.example new file mode 100644 index 00000000..9cf2dc97 --- /dev/null +++ b/.env.gitgram.example @@ -0,0 +1,72 @@ +# Gitgram backend (manual launch template) +NODE_ENV=production +PORT=4000 + +# Database / cache +DATABASE_URL=postgresql://gitgram:CHANGE_ME@127.0.0.1:5432/gitgram +REDIS_URL=redis://127.0.0.1:6379 + +# Auth +JWT_SECRET=CHANGE_ME_JWT_SECRET +JWT_REFRESH_SECRET=CHANGE_ME_JWT_REFRESH_SECRET +FRONTEND_URL=https://gitgram.app +CORS_ORIGIN=https://gitgram.app,https://www.gitgram.app + +# OAuth +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +# Payments +POLAR_ACCESS_TOKEN= +POLAR_WEBHOOK_SECRET= +POLAR_PREMIUM_PRODUCT_ID= +POLAR_CREATOR_PRODUCT_ID= +POLAR_MODE=production + +# Email / monitoring +RESEND_API_KEY= +SENTRY_DSN= + +# Storage +STORAGE_TYPE=local +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_S3_BUCKET= +AWS_REGION=us-east-1 +UPLOAD_DIR=./uploads +MAX_FILE_SIZE=10485760 + +# === Social Graph & AI Integration === + +# gh-social: GitHub trending fallback +GITHUB_TRENDING_API_KEY= + +# GitNexus Code Intelligence (enables gn_* tools in AITM MCP) +GITNEXUS_URL=http://localhost:3001 +AITM_GITNEXUS_ENABLED=0 + +# A2A Agent Protocol +AITM_A2A_AGENTS=[] +AITM_API_URL=http://localhost:3001 + +# Context7 Live Docs (enables tm_docs_* tools in AITM MCP) +CONTEXT7_URL=http://localhost:3100 +AITM_CONTEXT7_ENABLED=0 + +# Developer Tools +REPO_TO_TEXT_MAX_FILES=200 +REPO_TO_TEXT_MAX_FILE_SIZE=50000 + + +# === PAL MCP Server (Python execution in AITM) === +PAL_MCP_URL=http://localhost:3200 +AITM_PAL_ENABLED=0 + +# === AITM-Gitgram Bridge (auto-post GitHub events to Gitgram) === +GITGRAM_API_URL=http://localhost:4000 +GITGRAM_API_KEY= + +# === AI Analysis (GitNexus backend integration) === +# Already uses GITNEXUS_URL above diff --git a/.env.postgres.example b/.env.postgres.example new file mode 100644 index 00000000..236dcac7 --- /dev/null +++ b/.env.postgres.example @@ -0,0 +1,5 @@ +# PostgreSQL Configuration +POSTGRES_USER=gitgram +POSTGRES_PASSWORD=CHANGE_ME_STRONG_PASSWORD +POSTGRES_DB=gitgram +AITM_DB=aitimemachine diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 00000000..ddc24830 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,19 @@ +module.exports = { + root: true, + parser: '@typescript-eslint/parser', + plugins: ['@typescript-eslint'], + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + ], + env: { + node: true, + es2022: true, + }, + rules: { + '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], + 'no-console': ['warn', { allow: ['warn', 'error'] }], + }, + ignorePatterns: ['dist/', 'node_modules/', 'coverage/', '*.js'], +}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..6278034a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,247 @@ +# ============================================================================= +# CI - Gitgram + AI Time Machine +# Runs on every push/PR to main. +# Runs all test suites in parallel; typecheck and lint run independently. +# fail-fast: false ensures all jobs complete even when one fails. +# ============================================================================= +name: CI + +on: + push: + branches: [main] + pull_request: + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Gitgram Backend - Jest tests against live Postgres + Redis services + # --------------------------------------------------------------------------- + test-gitgram-backend: + name: "Test: Gitgram Backend" + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + defaults: + run: + working-directory: gitgram/backend + + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: test + POSTGRES_DB: gitgram_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache npm dependencies + uses: actions/cache@v4 + with: + path: gitgram/backend/node_modules + key: npm-gitgram-backend-${{ hashFiles('gitgram/backend/package-lock.json') }} + restore-keys: npm-gitgram-backend- + + - name: Install dependencies + run: npm install + + - name: Run tests + run: npm run test -- --forceExit + env: + DATABASE_URL: postgresql://postgres:test@localhost:5432/gitgram_test + REDIS_URL: redis://localhost:6379 + NODE_ENV: test + + # --------------------------------------------------------------------------- + # Gitgram Web - Vitest + # --------------------------------------------------------------------------- + test-gitgram-web: + name: "Test: Gitgram Web" + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + defaults: + run: + working-directory: gitgram/web + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache npm dependencies + uses: actions/cache@v4 + with: + path: gitgram/web/node_modules + key: npm-gitgram-web-${{ hashFiles('gitgram/web/package-lock.json') }} + restore-keys: npm-gitgram-web- + + - name: Install dependencies + run: npm install + + - name: Run tests + run: npm test + + # --------------------------------------------------------------------------- + # AITM API Server - Vitest + # --------------------------------------------------------------------------- + test-aitm: + name: "Test: AITM API Server" + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + defaults: + run: + working-directory: aitimemachine/api-server + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache npm dependencies + uses: actions/cache@v4 + with: + path: aitimemachine/api-server/node_modules + key: npm-aitm-api-server-${{ hashFiles('aitimemachine/api-server/package-lock.json') }} + restore-keys: npm-aitm-api-server- + + - name: Install dependencies + run: npm install + + - name: Run tests + run: npm test + + # --------------------------------------------------------------------------- + # TypeScript type checking - backend + web + # --------------------------------------------------------------------------- + typecheck: + name: "Typecheck: Backend + Web" + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache npm dependencies (backend) + uses: actions/cache@v4 + with: + path: gitgram/backend/node_modules + key: npm-gitgram-backend-${{ hashFiles('gitgram/backend/package-lock.json') }} + restore-keys: npm-gitgram-backend- + + - name: Cache npm dependencies (web) + uses: actions/cache@v4 + with: + path: gitgram/web/node_modules + key: npm-gitgram-web-${{ hashFiles('gitgram/web/package-lock.json') }} + restore-keys: npm-gitgram-web- + + - name: Install backend dependencies + working-directory: gitgram/backend + run: npm ci --legacy-peer-deps + + - name: Install web dependencies + working-directory: gitgram/web + run: npm ci --legacy-peer-deps + + - name: Typecheck backend + working-directory: gitgram/backend + run: npx tsc --noEmit + + - name: Typecheck web + working-directory: gitgram/web + run: npx tsc --noEmit + + # --------------------------------------------------------------------------- + # ESLint - backend (has .eslintrc.js); web uses Next.js lint + # --------------------------------------------------------------------------- + lint: + name: "Lint: Backend + Web" + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Cache npm dependencies (backend) + uses: actions/cache@v4 + with: + path: gitgram/backend/node_modules + key: npm-gitgram-backend-${{ hashFiles('gitgram/backend/package-lock.json') }} + restore-keys: npm-gitgram-backend- + + - name: Cache npm dependencies (web) + uses: actions/cache@v4 + with: + path: gitgram/web/node_modules + key: npm-gitgram-web-${{ hashFiles('gitgram/web/package-lock.json') }} + restore-keys: npm-gitgram-web- + + - name: Install backend dependencies + working-directory: gitgram/backend + run: npm install + + - name: Install web dependencies + working-directory: gitgram/web + run: npm install + + - name: Lint backend + working-directory: gitgram/backend + run: npm run lint + + - name: Lint web + working-directory: gitgram/web + run: npm run lint diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..07bbbd17 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,217 @@ +# ============================================================================= +# Integration Test Gate - Gitgram + AI Time Machine +# Can be triggered manually or on push to main. These tests require +# database services and validate cross-component interactions. +# ============================================================================= +name: Integration Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [labeled] + workflow_dispatch: + inputs: + run_gitgram: + description: "Run Gitgram integration tests" + type: boolean + default: true + run_aitm: + description: "Run AITM integration tests" + type: boolean + default: true + +concurrency: + group: integration-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Gitgram API - Integration tests + # --------------------------------------------------------------------------- + gitgram-integration: + name: "Gitgram: API Integration Tests" + runs-on: ubuntu-latest + timeout-minutes: 20 + if: >- + github.event_name == 'workflow_dispatch' && github.event.inputs.run_gitgram == 'true' || + github.event_name == 'push' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration')) + defaults: + run: + working-directory: gitgram/backend + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: gitgram + POSTGRES_PASSWORD: gitgram_password + POSTGRES_DB: gitgram_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + redis: + image: redis:7 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + env: + DATABASE_URL: postgresql://gitgram:gitgram_password@localhost:5432/gitgram_test?schema=public + JWT_SECRET: ci-integration-test-secret + NODE_ENV: test + REDIS_URL: redis://localhost:6379 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: gitgram/backend/package-lock.json + + - name: Install dependencies + run: npm ci --legacy-peer-deps + + - name: Generate Prisma Client + run: npx prisma generate + + - name: Run database migrations + run: npx prisma migrate deploy + + - name: Run integration tests + run: npx jest tests/integration.test.ts --forceExit --detectOpenHandles --testTimeout=30000 + + - name: Run API endpoint tests + run: | + # Run all test files that exercise API endpoints end-to-end + npx jest tests/auth.test.ts tests/posts.test.ts tests/users.test.ts tests/webhooks.test.ts \ + --forceExit --detectOpenHandles --testTimeout=30000 + + # --------------------------------------------------------------------------- + # AITM API - Integration tests + # --------------------------------------------------------------------------- + aitm-integration: + name: "AITM: API Integration Tests" + runs-on: ubuntu-latest + timeout-minutes: 20 + if: >- + github.event_name == 'workflow_dispatch' && github.event.inputs.run_aitm == 'true' || + github.event_name == 'push' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration')) + defaults: + run: + working-directory: aitimemachine/api-server + env: + NODE_ENV: test + AITM_TEST_NO_LISTEN: "0" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/api-server/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run integration tests + run: npx vitest run tests/integration --reporter=verbose + env: + # Integration tests may need these - sourced from secrets in real CI + DATABASE_URL: ${{ secrets.AITM_TEST_DATABASE_URL }} + JWT_SECRET: ${{ secrets.AITM_TEST_JWT_SECRET }} + + # --------------------------------------------------------------------------- + # AITM Core - E2E / platform tests + # --------------------------------------------------------------------------- + aitm-core-e2e: + name: "AITM: Core E2E Tests" + runs-on: ubuntu-latest + timeout-minutes: 15 + if: >- + github.event_name == 'workflow_dispatch' && github.event.inputs.run_aitm == 'true' || + github.event_name == 'push' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-integration')) + defaults: + run: + working-directory: aitimemachine + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build library + run: npm run build + + - name: Run E2E and platform tests + run: | + npx vitest run tests/e2e.test.ts tests/platform.test.ts --reporter=verbose + + # --------------------------------------------------------------------------- + # Summary gate + # --------------------------------------------------------------------------- + integration-gate: + name: Integration Test Gate + runs-on: ubuntu-latest + timeout-minutes: 2 + needs: + - gitgram-integration + - aitm-integration + - aitm-core-e2e + if: always() + steps: + - name: Check integration test results + run: | + FAILED=false + + # Only check jobs that actually ran (skip skipped ones) + if [[ "${{ needs.gitgram-integration.result }}" == "failure" ]]; then + echo "gitgram-integration FAILED" + FAILED=true + fi + if [[ "${{ needs.aitm-integration.result }}" == "failure" ]]; then + echo "aitm-integration FAILED" + FAILED=true + fi + if [[ "${{ needs.aitm-core-e2e.result }}" == "failure" ]]; then + echo "aitm-core-e2e FAILED" + FAILED=true + fi + + if [ "$FAILED" = true ]; then + echo "" + echo "Integration test gate failed. See individual job logs above." + exit 1 + fi + + echo "Integration test gate passed (or jobs were skipped)." + echo " gitgram-integration: ${{ needs.gitgram-integration.result }}" + echo " aitm-integration: ${{ needs.aitm-integration.result }}" + echo " aitm-core-e2e: ${{ needs.aitm-core-e2e.result }}" diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 00000000..776189c0 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,44 @@ +name: License Check + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +jobs: + license-check: + name: Block GPL / noncommercial imports in product trees + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check for GPL-contaminated imports (proximity) + run: | + # proximity is GPL-3 — its source must never be imported into product trees + if grep -rE "from ['\"].*external/proximity" \ + gitgram/ aitimemachine/ --include="*.ts" --include="*.tsx" -l 2>/dev/null; then + echo "ERROR: GPL-3 source from external/proximity imported into product tree" + exit 1 + fi + echo "OK: no proximity (GPL-3) imports found" + + - name: Check for PolyForm Noncommercial imports (gitnexus) + run: | + # gitnexus is PolyForm Noncommercial — source must not be copied into product trees + if grep -rE "from ['\"].*external/gitnexus" \ + gitgram/ aitimemachine/ --include="*.ts" --include="*.tsx" -l 2>/dev/null; then + echo "ERROR: PolyForm Noncommercial source from external/gitnexus imported into product tree" + exit 1 + fi + echo "OK: no gitnexus (PolyForm Noncommercial) imports found" + + - name: Check external/ is not on tsconfig paths + run: | + # Verify no tsconfig in product dirs resolves paths into external/ + if grep -rE '"external/' \ + gitgram/tsconfig*.json aitimemachine/tsconfig*.json 2>/dev/null; then + echo "ERROR: tsconfig path alias points into external/" + exit 1 + fi + echo "OK: no tsconfig paths into external/" diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml new file mode 100644 index 00000000..88e4a0e3 --- /dev/null +++ b/.github/workflows/migrations.yml @@ -0,0 +1,206 @@ +# ============================================================================= +# Migration Gate - Gitgram (Prisma) + AI Time Machine (Drizzle) +# Validates ORM schemas and checks for pending migrations. +# ============================================================================= +name: Migration Gate + +on: + push: + branches: [main, develop] + paths: + - "gitgram/backend/prisma/**" + - "aitimemachine/api-server/src/db/schema.ts" + - "aitimemachine/api-server/drizzle/**" + - "aitimemachine/src/core/neon/schema.ts" + - "aitimemachine/drizzle/**" + - "aitimemachine/drizzle.config.ts" + - "aitimemachine/api-server/drizzle.config.ts" + pull_request: + branches: [main, develop] + paths: + - "gitgram/backend/prisma/**" + - "aitimemachine/api-server/src/db/schema.ts" + - "aitimemachine/api-server/drizzle/**" + - "aitimemachine/src/core/neon/schema.ts" + - "aitimemachine/drizzle/**" + - "aitimemachine/drizzle.config.ts" + - "aitimemachine/api-server/drizzle.config.ts" + +concurrency: + group: migrations-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Gitgram - Prisma schema validation + # --------------------------------------------------------------------------- + gitgram-prisma-validate: + name: "Gitgram: Prisma Schema Validation" + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: gitgram/backend + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: gitgram + POSTGRES_PASSWORD: gitgram_password + POSTGRES_DB: gitgram_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + env: + DATABASE_URL: postgresql://gitgram:gitgram_password@localhost:5432/gitgram_test?schema=public + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: gitgram/backend/package-lock.json + + - name: Install dependencies + run: npm ci --legacy-peer-deps + + - name: Validate Prisma schema + run: npx prisma validate + + - name: Generate Prisma Client + run: npx prisma generate + + - name: Check migration status (dry run) + run: npx prisma migrate deploy + continue-on-error: false + + - name: Verify no pending migrations + run: | + # After deploy, diff should be empty against the current schema + npx prisma migrate diff \ + --from-migrations ./prisma/migrations \ + --to-schema-datamodel ./prisma/schema.prisma \ + --exit-code || { + echo "WARNING: There are pending schema changes not covered by migrations." + echo "Run 'npx prisma migrate dev' locally to generate the migration." + exit 1 + } + + # --------------------------------------------------------------------------- + # AITM API Server - Drizzle schema validation + # --------------------------------------------------------------------------- + aitm-drizzle-validate: + name: "AITM: Drizzle Schema Validation (API Server)" + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: aitimemachine/api-server + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/api-server/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: TypeScript check on schema files + run: npx tsc --noEmit + + - name: Validate Drizzle config loads + run: | + # Ensure drizzle config is parseable (exits 0 if valid) + npx drizzle-kit check 2>&1 || echo "drizzle-kit check completed" + + - name: Verify migration files exist and are ordered + run: | + MIGRATION_DIR="./drizzle" + if [ -d "$MIGRATION_DIR" ]; then + echo "Migration directory found. Contents:" + ls -la "$MIGRATION_DIR"/*.sql 2>/dev/null || echo "No SQL migration files found." + + # Check journal integrity + if [ -f "$MIGRATION_DIR/meta/_journal.json" ]; then + echo "Migration journal found." + node -e " + const j = require('$MIGRATION_DIR/meta/_journal.json'); + console.log('Total migrations:', j.entries.length); + j.entries.forEach(e => console.log(' -', e.idx, e.tag)); + " 2>/dev/null || echo "Journal parsing skipped (ESM)." + fi + else + echo "No migration directory. Skipping migration file check." + fi + + # --------------------------------------------------------------------------- + # AITM Core - Drizzle schema validation + # --------------------------------------------------------------------------- + aitm-core-drizzle-validate: + name: "AITM: Drizzle Schema Validation (Core)" + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: aitimemachine + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: TypeScript check on schema files + run: npx tsc --noEmit + + - name: Validate Drizzle config loads + run: | + npx drizzle-kit check 2>&1 || echo "drizzle-kit check completed" + + # --------------------------------------------------------------------------- + # Summary gate + # --------------------------------------------------------------------------- + migration-gate: + name: Migration Gate (all products) + runs-on: ubuntu-latest + timeout-minutes: 2 + needs: + - gitgram-prisma-validate + - aitm-drizzle-validate + - aitm-core-drizzle-validate + if: always() + steps: + - name: Check migration results + run: | + if [[ "${{ needs.gitgram-prisma-validate.result }}" != "success" ]] || \ + [[ "${{ needs.aitm-drizzle-validate.result }}" != "success" ]] || \ + [[ "${{ needs.aitm-core-drizzle-validate.result }}" != "success" ]]; then + echo "One or more migration validation jobs failed." + echo " gitgram-prisma-validate: ${{ needs.gitgram-prisma-validate.result }}" + echo " aitm-drizzle-validate: ${{ needs.aitm-drizzle-validate.result }}" + echo " aitm-core-drizzle-validate: ${{ needs.aitm-core-drizzle-validate.result }}" + exit 1 + fi + echo "All migration gates passed." diff --git a/.github/workflows/openapi-validation.yml b/.github/workflows/openapi-validation.yml new file mode 100644 index 00000000..f42f6ffa --- /dev/null +++ b/.github/workflows/openapi-validation.yml @@ -0,0 +1,130 @@ +# ============================================================================= +# OpenAPI Contract Validation +# Validates OpenAPI specs and checks route coverage for both products. +# Runs on PR/push to main when API files change. +# ============================================================================= +name: OpenAPI Validation + +on: + push: + branches: [main, develop] + paths: + - "aitimemachine/api-server/src/routes/**" + - "aitimemachine/api-server/src/index.ts" + - "aitimemachine/api-server/docs/openapi.yaml" + - "gitgram/backend/src/routes/**" + - "gitgram/backend/src/index.ts" + - "gitgram/backend/docs/openapi.yaml" + - "scripts/validate-openapi-*.ts" + - ".github/workflows/openapi-validation.yml" + pull_request: + branches: [main, develop] + paths: + - "aitimemachine/api-server/src/routes/**" + - "aitimemachine/api-server/src/index.ts" + - "aitimemachine/api-server/docs/openapi.yaml" + - "gitgram/backend/src/routes/**" + - "gitgram/backend/src/index.ts" + - "gitgram/backend/docs/openapi.yaml" + - "scripts/validate-openapi-*.ts" + - ".github/workflows/openapi-validation.yml" + +concurrency: + group: openapi-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Step 1: Validate OpenAPI spec syntax and structure + # --------------------------------------------------------------------------- + validate-spec: + name: "Validate OpenAPI Specs" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install tsx (TypeScript runner) + run: npm install -g tsx + + - name: Validate OpenAPI spec structure + run: npx tsx scripts/validate-openapi-spec.ts + + # --------------------------------------------------------------------------- + # Step 2: Check route coverage (code vs spec) + # --------------------------------------------------------------------------- + route-coverage: + name: "Route Coverage Check" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install tsx (TypeScript runner) + run: npm install -g tsx + + - name: Run route coverage validation + run: npx tsx scripts/validate-openapi-routes.ts + + # --------------------------------------------------------------------------- + # Step 3: Generate API documentation artifact + # --------------------------------------------------------------------------- + generate-docs: + name: "Generate API Docs Artifact" + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: [validate-spec] + if: github.event_name == 'pull_request' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Upload AITM OpenAPI spec + uses: actions/upload-artifact@v4 + with: + name: aitm-openapi-spec + path: aitimemachine/api-server/docs/openapi.yaml + retention-days: 30 + + - name: Upload Gitgram OpenAPI spec + uses: actions/upload-artifact@v4 + with: + name: gitgram-openapi-spec + path: gitgram/backend/docs/openapi.yaml + retention-days: 30 + + # --------------------------------------------------------------------------- + # Summary gate + # --------------------------------------------------------------------------- + openapi-gate: + name: "OpenAPI Validation Gate" + runs-on: ubuntu-latest + timeout-minutes: 2 + needs: [validate-spec, route-coverage] + if: always() + steps: + - name: Check validation results + run: | + if [[ "${{ needs.validate-spec.result }}" != "success" ]] || \ + [[ "${{ needs.route-coverage.result }}" != "success" ]]; then + echo "OpenAPI validation failed." + echo " validate-spec: ${{ needs.validate-spec.result }}" + echo " route-coverage: ${{ needs.route-coverage.result }}" + exit 1 + fi + echo "All OpenAPI validation checks passed." diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml new file mode 100644 index 00000000..2c194d0f --- /dev/null +++ b/.github/workflows/smoke-tests.yml @@ -0,0 +1,240 @@ +# ============================================================================= +# Smoke Test Gate - Post-deploy health checks for Gitgram + AI Time Machine +# Validates that deployed services are alive and responding correctly. +# Triggered after deploy workflows or manually with environment URLs. +# ============================================================================= +name: Smoke Tests (Post-Deploy) + +on: + workflow_dispatch: + inputs: + environment: + description: "Target environment" + type: choice + options: + - staging + - production + default: staging + gitgram_api_url: + description: "Gitgram API base URL (e.g. https://api.gitgram.dev)" + type: string + required: false + aitm_api_url: + description: "AITM API base URL (e.g. https://api.aitimemachine.dev)" + type: string + required: false + workflow_call: + inputs: + environment: + type: string + required: true + gitgram_api_url: + type: string + required: false + aitm_api_url: + type: string + required: false + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Gitgram API - Health & smoke checks + # --------------------------------------------------------------------------- + gitgram-smoke: + name: "Gitgram: Smoke Tests (${{ inputs.environment }})" + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + API_URL: ${{ inputs.gitgram_api_url || vars.GITGRAM_API_URL || 'http://localhost:3001' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Wait for service readiness + run: | + echo "Waiting for Gitgram API at $API_URL to become ready..." + for i in $(seq 1 30); do + if curl -sf "$API_URL/health" > /dev/null 2>&1 || \ + curl -sf "$API_URL/api/health" > /dev/null 2>&1; then + echo "Service is ready after ${i}s" + break + fi + echo "Attempt $i/30 - not ready yet..." + sleep 2 + done + + - name: Health check endpoint + run: | + echo "### Gitgram API Health Check" >> $GITHUB_STEP_SUMMARY + RESPONSE=$(curl -sf -w "\nHTTP_CODE:%{http_code}" "$API_URL/health" 2>/dev/null || \ + curl -sf -w "\nHTTP_CODE:%{http_code}" "$API_URL/api/health" 2>/dev/null || \ + echo "FAILED") + + if echo "$RESPONSE" | grep -q "FAILED"; then + echo "Health check failed. Service may not be running at $API_URL" + echo "Status: FAILED" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + HTTP_CODE=$(echo "$RESPONSE" | grep "HTTP_CODE:" | cut -d: -f2) + BODY=$(echo "$RESPONSE" | grep -v "HTTP_CODE:") + + echo "HTTP Status: $HTTP_CODE" + echo "Response: $BODY" + echo "- HTTP Status: $HTTP_CODE" >> $GITHUB_STEP_SUMMARY + echo "- Response: \`$BODY\`" >> $GITHUB_STEP_SUMMARY + + if [ "$HTTP_CODE" != "200" ]; then + echo "Unexpected HTTP status code: $HTTP_CODE" + exit 1 + fi + + - name: API response validation + run: | + echo "Validating basic API endpoints..." + + # Test that auth endpoints exist and return proper error codes (not 500) + AUTH_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$API_URL/api/auth/login" -X POST \ + -H "Content-Type: application/json" \ + -d '{}' 2>/dev/null || echo "000") + echo "POST /api/auth/login (empty body): HTTP $AUTH_CODE" + if [ "$AUTH_CODE" = "500" ] || [ "$AUTH_CODE" = "000" ]; then + echo "Auth endpoint returned server error or is unreachable" + exit 1 + fi + + # Test that public endpoints return structured responses + USERS_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$API_URL/api/users" 2>/dev/null || echo "000") + echo "GET /api/users: HTTP $USERS_CODE" + if [ "$USERS_CODE" = "500" ]; then + echo "Users endpoint returned server error" + exit 1 + fi + + echo "Gitgram API smoke tests passed." + + # --------------------------------------------------------------------------- + # AITM API - Health & smoke checks + # --------------------------------------------------------------------------- + aitm-smoke: + name: "AITM: Smoke Tests (${{ inputs.environment }})" + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + API_URL: ${{ inputs.aitm_api_url || vars.AITM_API_URL || 'http://localhost:3000' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Wait for service readiness + run: | + echo "Waiting for AITM API at $API_URL to become ready..." + for i in $(seq 1 30); do + if curl -sf "$API_URL/health" > /dev/null 2>&1 || \ + curl -sf "$API_URL/api/health" > /dev/null 2>&1; then + echo "Service is ready after ${i}s" + break + fi + echo "Attempt $i/30 - not ready yet..." + sleep 2 + done + + - name: Health check endpoint + run: | + echo "### AITM API Health Check" >> $GITHUB_STEP_SUMMARY + RESPONSE=$(curl -sf -w "\nHTTP_CODE:%{http_code}" "$API_URL/health" 2>/dev/null || \ + curl -sf -w "\nHTTP_CODE:%{http_code}" "$API_URL/api/health" 2>/dev/null || \ + echo "FAILED") + + if echo "$RESPONSE" | grep -q "FAILED"; then + echo "Health check failed. Service may not be running at $API_URL" + echo "Status: FAILED" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + HTTP_CODE=$(echo "$RESPONSE" | grep "HTTP_CODE:" | cut -d: -f2) + BODY=$(echo "$RESPONSE" | grep -v "HTTP_CODE:") + + echo "HTTP Status: $HTTP_CODE" + echo "Response: $BODY" + echo "- HTTP Status: $HTTP_CODE" >> $GITHUB_STEP_SUMMARY + echo "- Response: \`$BODY\`" >> $GITHUB_STEP_SUMMARY + + if [ "$HTTP_CODE" != "200" ]; then + echo "Unexpected HTTP status code: $HTTP_CODE" + exit 1 + fi + + - name: API response validation + run: | + echo "Validating AITM API endpoints..." + + # Verify sessions endpoint exists + SESSIONS_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$API_URL/api/sessions" 2>/dev/null || echo "000") + echo "GET /api/sessions: HTTP $SESSIONS_CODE" + # Expect 401 (unauthorized) or 200, not 500 + if [ "$SESSIONS_CODE" = "500" ] || [ "$SESSIONS_CODE" = "000" ]; then + echo "Sessions endpoint returned server error or is unreachable" + exit 1 + fi + + # Verify sync endpoint exists + SYNC_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$API_URL/api/sync" -X POST \ + -H "Content-Type: application/json" \ + -d '{}' 2>/dev/null || echo "000") + echo "POST /api/sync: HTTP $SYNC_CODE" + if [ "$SYNC_CODE" = "500" ]; then + echo "Sync endpoint returned server error" + exit 1 + fi + + # Verify keys/auth endpoint exists + KEYS_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$API_URL/api/keys" 2>/dev/null || echo "000") + echo "GET /api/keys: HTTP $KEYS_CODE" + if [ "$KEYS_CODE" = "500" ]; then + echo "Keys endpoint returned server error" + exit 1 + fi + + echo "AITM API smoke tests passed." + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + smoke-gate: + name: Smoke Test Gate (${{ inputs.environment }}) + runs-on: ubuntu-latest + timeout-minutes: 2 + needs: + - gitgram-smoke + - aitm-smoke + if: always() + steps: + - name: Check smoke test results + run: | + echo "### Smoke Test Summary (${{ inputs.environment }})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Service | Result |" >> $GITHUB_STEP_SUMMARY + echo "|---------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Gitgram API | ${{ needs.gitgram-smoke.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| AITM API | ${{ needs.aitm-smoke.result }} |" >> $GITHUB_STEP_SUMMARY + + FAILED=false + if [[ "${{ needs.gitgram-smoke.result }}" == "failure" ]]; then + echo "Gitgram smoke tests FAILED" + FAILED=true + fi + if [[ "${{ needs.aitm-smoke.result }}" == "failure" ]]; then + echo "AITM smoke tests FAILED" + FAILED=true + fi + + if [ "$FAILED" = true ]; then + echo "" + echo "Smoke test gate FAILED. Check deployed services." + exit 1 + fi + + echo "All smoke tests passed for ${{ inputs.environment }}." diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..dcd00f0f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,193 @@ +# ============================================================================= +# Test Gate - Gitgram (Jest) + AI Time Machine (Vitest) +# Runs unit tests for both products and collects coverage reports. +# ============================================================================= +name: Test Gate + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +concurrency: + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NODE_VERSION: "20" + +jobs: + # --------------------------------------------------------------------------- + # Gitgram Backend - Jest unit tests + # --------------------------------------------------------------------------- + gitgram-backend-tests: + name: "Gitgram: Backend Tests (Jest)" + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: gitgram/backend + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: gitgram + POSTGRES_PASSWORD: gitgram_password + POSTGRES_DB: gitgram_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + env: + DATABASE_URL: postgresql://gitgram:gitgram_password@localhost:5432/gitgram_test?schema=public + JWT_SECRET: ci-test-secret-key + NODE_ENV: test + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: gitgram/backend/package-lock.json + + - name: Install dependencies + run: npm ci --legacy-peer-deps + + - name: Generate Prisma Client + run: npx prisma generate + + - name: Run database migrations + run: npx prisma migrate deploy + + - name: Run unit tests with coverage + run: npm run test:coverage + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: gitgram-backend-coverage + path: gitgram/backend/coverage + retention-days: 7 + + - name: Coverage summary + if: always() + run: | + if [ -f coverage/coverage-summary.json ]; then + echo "### Gitgram Backend Coverage" >> $GITHUB_STEP_SUMMARY + node -e " + const s = require('./coverage/coverage-summary.json').total; + console.log('| Metric | Coverage |'); + console.log('|--------|----------|'); + console.log('| Statements |', s.statements.pct + '%', '|'); + console.log('| Branches |', s.branches.pct + '%', '|'); + console.log('| Functions |', s.functions.pct + '%', '|'); + console.log('| Lines |', s.lines.pct + '%', '|'); + " >> $GITHUB_STEP_SUMMARY 2>/dev/null || echo "Coverage summary unavailable." + fi + + # --------------------------------------------------------------------------- + # AITM Core - Vitest unit tests + # --------------------------------------------------------------------------- + aitm-core-tests: + name: "AITM: Core Tests (Vitest)" + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: aitimemachine + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run tests with coverage + run: npx vitest run --coverage + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: aitm-core-coverage + path: aitimemachine/coverage + retention-days: 7 + + # --------------------------------------------------------------------------- + # AITM API Server - Vitest unit tests + # --------------------------------------------------------------------------- + aitm-api-tests: + name: "AITM: API Server Tests (Vitest)" + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: aitimemachine/api-server + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/api-server/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test + env: + NODE_ENV: test + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: aitm-api-test-results + path: aitimemachine/api-server/coverage + retention-days: 7 + if-no-files-found: ignore + + # --------------------------------------------------------------------------- + # Summary gate - all tests must pass + # --------------------------------------------------------------------------- + test-gate: + name: Test Gate (all products) + runs-on: ubuntu-latest + timeout-minutes: 2 + needs: + - gitgram-backend-tests + - aitm-core-tests + - aitm-api-tests + if: always() + steps: + - name: Check test results + run: | + if [[ "${{ needs.gitgram-backend-tests.result }}" != "success" ]] || \ + [[ "${{ needs.aitm-core-tests.result }}" != "success" ]] || \ + [[ "${{ needs.aitm-api-tests.result }}" != "success" ]]; then + echo "One or more test jobs failed." + echo " gitgram-backend-tests: ${{ needs.gitgram-backend-tests.result }}" + echo " aitm-core-tests: ${{ needs.aitm-core-tests.result }}" + echo " aitm-api-tests: ${{ needs.aitm-api-tests.result }}" + exit 1 + fi + echo "All test gates passed." diff --git a/.github/workflows/verify-production-db.yml b/.github/workflows/verify-production-db.yml new file mode 100644 index 00000000..f6e5f470 --- /dev/null +++ b/.github/workflows/verify-production-db.yml @@ -0,0 +1,214 @@ +name: Verify Production Databases + +on: + schedule: + # Run daily at 06:00 UTC + - cron: "0 6 * * *" + workflow_dispatch: + inputs: + target: + description: "Which database to verify" + required: false + default: "all" + type: choice + options: + - all + - neon + - gitgram + +permissions: + contents: read + +env: + NODE_VERSION: "20" + +jobs: + # =========================================================================== + # Neon (AITimeMachine) Database Verification + # =========================================================================== + verify-neon: + name: Verify Neon DB (AITimeMachine) + runs-on: ubuntu-latest + if: ${{ github.event_name == 'schedule' || github.event.inputs.target == 'all' || github.event.inputs.target == 'neon' }} + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: aitimemachine/api-server/package-lock.json + + - name: Install dependencies + working-directory: aitimemachine/api-server + run: npm ci + + - name: Run Neon verification + working-directory: aitimemachine/api-server + env: + DATABASE_URL: ${{ secrets.NEON_DATABASE_URL }} + run: npx tsx ../../scripts/verify-neon.ts --json > /tmp/neon-results.json 2>&1 || true + + - name: Parse results + id: neon-results + run: | + if [ -f /tmp/neon-results.json ]; then + overall=$(cat /tmp/neon-results.json | python3 -c "import sys,json; data=json.load(sys.stdin); print(data.get('overall','unknown'))" 2>/dev/null || echo "unknown") + echo "overall=$overall" >> $GITHUB_OUTPUT + cat /tmp/neon-results.json + else + echo "overall=error" >> $GITHUB_OUTPUT + echo "No results file generated" + fi + + - name: Upload results artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: neon-verification-results + path: /tmp/neon-results.json + retention-days: 30 + + - name: Check for failures + if: steps.neon-results.outputs.overall == 'unhealthy' + run: | + echo "::error::Neon database verification FAILED" + echo "Check the neon-verification-results artifact for details." + exit 1 + + - name: Report degraded status + if: steps.neon-results.outputs.overall == 'degraded' + run: | + echo "::warning::Neon database verification has WARNINGS" + echo "Check the neon-verification-results artifact for details." + + # =========================================================================== + # Gitgram Backend Database Verification + # =========================================================================== + verify-gitgram: + name: Verify Gitgram DB + runs-on: ubuntu-latest + if: ${{ github.event_name == 'schedule' || github.event.inputs.target == 'all' || github.event.inputs.target == 'gitgram' }} + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: gitgram/backend/package-lock.json + + - name: Install dependencies + working-directory: gitgram/backend + run: npm ci + + - name: Generate Prisma Client + working-directory: gitgram/backend + run: npx prisma generate + + - name: Run Gitgram DB verification + working-directory: gitgram/backend + env: + DATABASE_URL: ${{ secrets.GITGRAM_DATABASE_URL }} + run: npx tsx ../../scripts/verify-gitgram-db.ts --json > /tmp/gitgram-results.json 2>&1 || true + + - name: Parse results + id: gitgram-results + run: | + if [ -f /tmp/gitgram-results.json ]; then + overall=$(cat /tmp/gitgram-results.json | python3 -c "import sys,json; data=json.load(sys.stdin); print(data.get('overall','unknown'))" 2>/dev/null || echo "unknown") + echo "overall=$overall" >> $GITHUB_OUTPUT + cat /tmp/gitgram-results.json + else + echo "overall=error" >> $GITHUB_OUTPUT + echo "No results file generated" + fi + + - name: Upload results artifact + uses: actions/upload-artifact@v4 + if: always() + with: + name: gitgram-verification-results + path: /tmp/gitgram-results.json + retention-days: 30 + + - name: Check for failures + if: steps.gitgram-results.outputs.overall == 'unhealthy' + run: | + echo "::error::Gitgram database verification FAILED" + echo "Check the gitgram-verification-results artifact for details." + exit 1 + + - name: Report degraded status + if: steps.gitgram-results.outputs.overall == 'degraded' + run: | + echo "::warning::Gitgram database verification has WARNINGS" + echo "Check the gitgram-verification-results artifact for details." + + # =========================================================================== + # Summary notification (runs after both jobs complete) + # =========================================================================== + notify: + name: Verification Summary + runs-on: ubuntu-latest + needs: [verify-neon, verify-gitgram] + if: always() + + steps: + - name: Download Neon results + uses: actions/download-artifact@v4 + with: + name: neon-verification-results + path: /tmp/neon + continue-on-error: true + + - name: Download Gitgram results + uses: actions/download-artifact@v4 + with: + name: gitgram-verification-results + path: /tmp/gitgram + continue-on-error: true + + - name: Generate summary + run: | + echo "## Database Verification Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Database | Status | Details |" >> $GITHUB_STEP_SUMMARY + echo "|----------|--------|---------|" >> $GITHUB_STEP_SUMMARY + + # Neon results + if [ -f /tmp/neon/neon-results.json ]; then + neon_status=$(python3 -c "import json; data=json.load(open('/tmp/neon/neon-results.json')); print(data.get('overall','unknown'))" 2>/dev/null || echo "error") + neon_summary=$(python3 -c "import json; data=json.load(open('/tmp/neon/neon-results.json')); s=data.get('summary',{}); print(f\"P:{s.get('passed',0)} W:{s.get('warned',0)} F:{s.get('failed',0)}\")" 2>/dev/null || echo "N/A") + else + neon_status="${{ needs.verify-neon.result }}" + neon_summary="Job result: ${neon_status}" + fi + + # Gitgram results + if [ -f /tmp/gitgram/gitgram-results.json ]; then + gitgram_status=$(python3 -c "import json; data=json.load(open('/tmp/gitgram/gitgram-results.json')); print(data.get('overall','unknown'))" 2>/dev/null || echo "error") + gitgram_summary=$(python3 -c "import json; data=json.load(open('/tmp/gitgram/gitgram-results.json')); s=data.get('summary',{}); print(f\"P:{s.get('passed',0)} W:{s.get('warned',0)} F:{s.get('failed',0)}\")" 2>/dev/null || echo "N/A") + else + gitgram_status="${{ needs.verify-gitgram.result }}" + gitgram_summary="Job result: ${gitgram_status}" + fi + + echo "| Neon (AITimeMachine) | ${neon_status} | ${neon_summary} |" >> $GITHUB_STEP_SUMMARY + echo "| Gitgram Backend | ${gitgram_status} | ${gitgram_summary} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "_Run at: $(date -u +"%Y-%m-%d %H:%M:%S UTC")_" >> $GITHUB_STEP_SUMMARY + + - name: Fail if any verification failed + if: needs.verify-neon.result == 'failure' || needs.verify-gitgram.result == 'failure' + run: | + echo "::error::One or more database verifications failed. Check individual job logs." + exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..cc593055 --- /dev/null +++ b/.gitignore @@ -0,0 +1,64 @@ +# Dependencies +node_modules/ + +# Build outputs +dist/ +build/ +lib/ +.next/ +out/ +.expo/ +.expo-shared/ +web-build/ + +# Logs and runtime output +logs/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +pm2.log + +# Environment and secrets +.env +.env.* +!.env.example +!.env.*.example + +# Local app/runtime state +.tmp/ +tmp/ +temp/ +.cache/ +.turbo/ +.aitimemachine/ +*.aitm +*.db +*.sqlite +*.sqlite3 +.kb-credentials.json +.kb-indexer-state.json +.mega-indexer-state.json + +# Test artifacts +coverage/ +.nyc_output/ +test-results/ +playwright-report/ +blob-report/ + +# OS and editor files +.DS_Store +Thumbs.db +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# Private keys and certificates +*.pem +*.key +*.p8 +*.p12 diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..07725571 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,9 @@ +{ + "semi": true, + "singleQuote": true, + "trailingComma": "es5", + "printWidth": 100, + "tabWidth": 2, + "arrowParens": "always", + "endOfLine": "lf" +} diff --git a/AUDIT_CHECKLIST.md b/AUDIT_CHECKLIST.md new file mode 100644 index 00000000..261eb713 --- /dev/null +++ b/AUDIT_CHECKLIST.md @@ -0,0 +1,84 @@ +# Audit Checklist (Launch Readiness) + +This checklist is meant to be executed as a repeatable “startup-style” audit pass: verify the user flow end-to-end, confirm expected network calls + auth, and check that the UI has no dead-ends. + +Status legend: `PASS`, `FAIL`, `N/A`, `FOLLOWUP` + +## 1) Gitgram Core Web Flows + +### Auth +- [ ] `GET /` loads unauthenticated and routes to login/signup actions (`PASS|FAIL`) +- [ ] Register with email/password creates account, stores tokens, routes to onboarding (`PASS|FAIL`) +- [ ] Login with email/password routes to `/feed` (`PASS|FAIL`) +- [ ] OAuth GitHub callback (`/auth/github/callback`) stores tokens and routes to onboarding for new users (`PASS|FAIL`) +- [ ] OAuth Google callback (`/auth/google/callback`) stores tokens and routes to onboarding for new users (`PASS|FAIL`) +- [ ] Logout clears local state and returns to `/` (`PASS|FAIL`) + +### Onboarding +- [ ] `/onboarding/welcome` has a clear next action to interests/follow (`PASS|FAIL`) +- [ ] `/onboarding/interests` persists interests to backend and updates stored user state (`PASS|FAIL`) +- [ ] `/onboarding/follow` suggests accounts and persists follows (`PASS|FAIL`) +- [ ] First-time empty feed redirects to onboarding once (no loop) (`PASS|FAIL`) + +### Feed + Social +- [ ] `/feed` loads posts, handles empty state, can like/unlike (`PASS|FAIL`) +- [ ] Follow/unfollow flows update counts and feed suggestions (`PASS|FAIL`) +- [ ] Search (`/search`) returns results for repos/people/posts/hashtags (`PASS|FAIL`) +- [ ] Trending pages show meaningful content or a non-empty placeholder (`PASS|FAIL`) + +### Create Content +- [ ] Create post (`/create`) upload validation errors are user-friendly (`PASS|FAIL`) +- [ ] Create story (`/story/create`) upload validation errors are user-friendly (`PASS|FAIL`) +- [ ] Create reel (`/reel/create`) upload validation errors are user-friendly (`PASS|FAIL`) +- [ ] Media uploads hit the configured API base (no hardcoded localhost) (`PASS|FAIL`) + +### Repo Profile + Account Switching +- [ ] Repository profile page loads in production base URL (`PASS|FAIL`) +- [ ] Account switcher lists accounts + switches successfully (`PASS|FAIL`) + +### Moderation/Admin +- [ ] Report flow exists from content UI and creates a report (`PASS|FAIL`) +- [ ] Moderation queue (`/admin/moderation`) loads for moderators/admins and is blocked for normal users (`PASS|FAIL`) +- [ ] Enforcement actions (remove content / suspend user) are audited (`PASS|FAIL`) + +## 2) AITM Flows + +### Website Signup + Keys +- [ ] Sign up / sign in works in local/dev (`PASS|FAIL`) +- [ ] API keys page: create key shows secret once; list does not expose full secret (`PASS|FAIL`) +- [ ] Key revoke works and blocks API usage (`PASS|FAIL`) + +### CLI Auth + Capture +- [ ] `aitm login --api-key ...` persists auth and `aitm whoami` returns expected user (`PASS|FAIL`) +- [ ] `aitm init` initializes a project and creates a checkpoint (`PASS|FAIL`) +- [ ] Capture path enforces consent where required (`PASS|FAIL`) + +### Timeline + Restore +- [ ] Timeline UI loads and filters work (`PASS|FAIL`) +- [ ] Restore preview shows scoped diff summary (`PASS|FAIL`) +- [ ] Restore apply requires confirmation and is reversible (`PASS|FAIL`) + +### Webhooks +- [ ] Webhooks list/create/enable-disable/delete all hit real API endpoints (`PASS|FAIL`) +- [ ] Deliveries page loads and retry works (no “simulate success”) (`PASS|FAIL`) + +### Export/Consent +- [ ] Export endpoints are gated by consent and traceability IDs are present (`PASS|FAIL`) +- [ ] Exported artifacts are redacted per policy (`PASS|FAIL`) + +## 3) Docs Correctness Sweep + +### Ops/Runbook +- [ ] `docs/runbook.md` local ports match compose (`PASS|FAIL`) +- [ ] `docs/ops-quickref.md` commands match current health endpoints (`PASS|FAIL`) +- [ ] Deploy docs reference current workflows (`PASS|FAIL`) + +### Env/Secrets +- [ ] `docs/environment-matrix.md` matches `.env.*.example` files (`PASS|FAIL`) +- [ ] `docs/secrets-contract.md` lists all required third-party keys + rotation owners (`PASS|FAIL`) + +### Install Guides +- [ ] Gitgram web docs use `:3000` and backend `:4000` (`PASS|FAIL`) +- [ ] AITM Quickstart reflects current CLI commands (`aitm setup --global` semantics, MCP install) (`PASS|FAIL`) +- [ ] MCP install instructions work with `npx` (no monorepo assumptions) (`PASS|FAIL`) + diff --git a/CANONICAL_SQL_SCHEMA_V1.sql b/CANONICAL_SQL_SCHEMA_V1.sql new file mode 100644 index 00000000..1353ad72 --- /dev/null +++ b/CANONICAL_SQL_SCHEMA_V1.sql @@ -0,0 +1,450 @@ +-- Canonical SQL Schema v1 +-- Unified model for Gitgram + AITimeMachine + Governance +-- Source document: NORTH_STAR_PRD_ARCHITECTURE_V1.md + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS vector; + +-- ========================= +-- Core Identity / Tenancy +-- ========================= + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email TEXT NOT NULL UNIQUE, + handle TEXT NOT NULL UNIQUE, + display_name TEXT, + avatar_url TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS organizations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + slug TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + created_by_user_id UUID REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS organization_memberships ( + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + role TEXT NOT NULL CHECK (role IN ('owner', 'admin', 'member')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (organization_id, user_id) +); + +CREATE TABLE IF NOT EXISTS api_keys ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + name TEXT NOT NULL, + key_hash TEXT NOT NULL UNIQUE, + scopes JSONB NOT NULL DEFAULT '[]'::jsonb, + is_active BOOLEAN NOT NULL DEFAULT true, + last_used_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((user_id IS NOT NULL) OR (organization_id IS NOT NULL)) +); + +-- ========================= +-- Repository / Project Map +-- ========================= + +CREATE TABLE IF NOT EXISTS repositories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + github_node_id TEXT UNIQUE, + github_owner TEXT NOT NULL, + github_name TEXT NOT NULL, + full_name TEXT NOT NULL UNIQUE, + is_private BOOLEAN NOT NULL DEFAULT false, + default_branch TEXT, + description TEXT, + stars_count BIGINT NOT NULL DEFAULT 0, + forks_count BIGINT NOT NULL DEFAULT 0, + imported_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS projects ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + owner_user_id UUID REFERENCES users(id) ON DELETE CASCADE, + name TEXT NOT NULL, + local_path TEXT, + is_archived BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((organization_id IS NOT NULL) OR (owner_user_id IS NOT NULL)) +); + +CREATE TABLE IF NOT EXISTS project_repositories ( + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + repository_id UUID NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (project_id, repository_id) +); + +-- ========================= +-- Gitgram Social Layer +-- ========================= + +CREATE TABLE IF NOT EXISTS repo_profiles ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID NOT NULL UNIQUE REFERENCES repositories(id) ON DELETE CASCADE, + headline TEXT, + bio TEXT, + hero_media_url TEXT, + visibility TEXT NOT NULL DEFAULT 'public' CHECK (visibility IN ('public', 'unlisted', 'private')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS posts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + author_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + repository_id UUID REFERENCES repositories(id) ON DELETE SET NULL, + project_id UUID REFERENCES projects(id) ON DELETE SET NULL, + type TEXT NOT NULL CHECK (type IN ('post', 'reel', 'story', 'change_story', 'snippet')), + caption TEXT, + code_snippet TEXT, + language TEXT, + linked_checkpoint_id TEXT, + visibility TEXT NOT NULL DEFAULT 'public' CHECK (visibility IN ('public', 'followers', 'private')), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS post_assets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + post_id UUID NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video', 'gif')), + storage_key TEXT NOT NULL, + mime_type TEXT, + width INTEGER, + height INTEGER, + duration_ms INTEGER, + position INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS comments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + post_id UUID NOT NULL REFERENCES posts(id) ON DELETE CASCADE, + author_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + parent_comment_id UUID REFERENCES comments(id) ON DELETE CASCADE, + body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS reactions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + target_type TEXT NOT NULL CHECK (target_type IN ('post', 'comment', 'repository')), + target_id UUID NOT NULL, + reaction TEXT NOT NULL CHECK (reaction IN ('like', 'love', 'fire', 'clap', 'rocket')), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (user_id, target_type, target_id, reaction) +); + +CREATE TABLE IF NOT EXISTS follows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + follower_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + target_type TEXT NOT NULL CHECK (target_type IN ('user', 'repository')), + target_user_id UUID REFERENCES users(id) ON DELETE CASCADE, + target_repository_id UUID REFERENCES repositories(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ( + (target_type = 'user' AND target_user_id IS NOT NULL AND target_repository_id IS NULL) OR + (target_type = 'repository' AND target_repository_id IS NOT NULL AND target_user_id IS NULL) + ) +); + +CREATE TABLE IF NOT EXISTS trend_scores ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + window_start TIMESTAMPTZ NOT NULL, + window_end TIMESTAMPTZ NOT NULL, + score NUMERIC(12,4) NOT NULL, + signals JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (repository_id, window_start, window_end) +); + +-- ========================= +-- AITM Time Layer +-- ========================= + +CREATE TABLE IF NOT EXISTS blobs ( + hash TEXT PRIMARY KEY, + content BYTEA NOT NULL, + size BIGINT NOT NULL, + compression TEXT NOT NULL DEFAULT 'brotli', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS trees ( + hash TEXT PRIMARY KEY, + entries JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS checkpoints ( + id TEXT PRIMARY KEY, + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + user_id UUID REFERENCES users(id) ON DELETE SET NULL, + parent_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + linked_checkpoint_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + tree_hash TEXT NOT NULL REFERENCES trees(hash), + trigger_type TEXT NOT NULL CHECK (trigger_type IN ('manual', 'auto', 'before_action', 'after_action', 'restore')), + trigger_tool TEXT, + description TEXT, + session_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'fk_posts_linked_checkpoint' + ) THEN + ALTER TABLE posts + ADD CONSTRAINT fk_posts_linked_checkpoint + FOREIGN KEY (linked_checkpoint_id) + REFERENCES checkpoints(id) + ON DELETE SET NULL; + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS checkpoint_files ( + checkpoint_id TEXT NOT NULL REFERENCES checkpoints(id) ON DELETE CASCADE, + file_path TEXT NOT NULL, + blob_hash TEXT REFERENCES blobs(hash), + size BIGINT, + mode INTEGER, + mtime_ms BIGINT, + change_type TEXT NOT NULL CHECK (change_type IN ('added', 'modified', 'deleted', 'unchanged')), + PRIMARY KEY (checkpoint_id, file_path) +); + +CREATE TABLE IF NOT EXISTS diffs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + from_checkpoint_id TEXT NOT NULL REFERENCES checkpoints(id) ON DELETE CASCADE, + to_checkpoint_id TEXT NOT NULL REFERENCES checkpoints(id) ON DELETE CASCADE, + file_path TEXT NOT NULL, + diff_content TEXT NOT NULL, + stats JSONB NOT NULL DEFAULT '{}'::jsonb, + algorithm TEXT NOT NULL DEFAULT 'histogram', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ========================= +-- AITM Context Layer +-- ========================= + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + session_id TEXT NOT NULL, + checkpoint_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system')), + content TEXT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS tool_calls ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + message_id UUID REFERENCES messages(id) ON DELETE SET NULL, + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + session_id TEXT NOT NULL, + checkpoint_before_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + checkpoint_after_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + tool_name TEXT NOT NULL, + arguments JSONB NOT NULL DEFAULT '{}'::jsonb, + result JSONB, + success BOOLEAN, + error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS contexts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + checkpoint_id TEXT NOT NULL REFERENCES checkpoints(id) ON DELETE CASCADE, + session_id TEXT NOT NULL, + summary TEXT, + active_files JSONB NOT NULL DEFAULT '[]'::jsonb, + cursor_position JSONB, + current_task TEXT, + goals JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS restore_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + actor_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + from_checkpoint_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + to_checkpoint_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + scope_type TEXT NOT NULL CHECK (scope_type IN ('file', 'folder', 'project')), + scope_path TEXT, + status TEXT NOT NULL CHECK (status IN ('requested', 'applied', 'failed')), + details JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ========================= +-- AITM Augmented Layer +-- ========================= + +CREATE TABLE IF NOT EXISTS embeddings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + checkpoint_id TEXT REFERENCES checkpoints(id) ON DELETE SET NULL, + source_type TEXT NOT NULL CHECK (source_type IN ('code', 'message', 'doc')), + file_path TEXT, + chunk_text TEXT NOT NULL, + chunk_start_line INTEGER, + chunk_end_line INTEGER, + embedding vector(1536) NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS indexed_docs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + url TEXT NOT NULL, + title TEXT, + content TEXT NOT NULL, + embedding vector(1536), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + indexed_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ========================= +-- Governance / Compliance +-- ========================= + +CREATE TABLE IF NOT EXISTS consents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + granted_by_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + consent_type TEXT NOT NULL CHECK (consent_type IN ('operational_capture', 'training_export')), + granted BOOLEAN NOT NULL, + effective_from TIMESTAMPTZ NOT NULL DEFAULT now(), + effective_to TIMESTAMPTZ, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((organization_id IS NOT NULL) OR (project_id IS NOT NULL)) +); + +CREATE TABLE IF NOT EXISTS retention_policies ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + data_class TEXT NOT NULL CHECK (data_class IN ('checkpoints', 'messages', 'tool_calls', 'social_media', 'exports', 'audit_logs')), + retention_days INTEGER NOT NULL CHECK (retention_days > 0), + hard_delete BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((organization_id IS NOT NULL) OR (project_id IS NOT NULL)) +); + +CREATE TABLE IF NOT EXISTS export_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + requested_by_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + consent_id UUID REFERENCES consents(id) ON DELETE SET NULL, + export_type TEXT NOT NULL CHECK (export_type IN ('training', 'compliance', 'backup')), + status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'completed', 'failed', 'cancelled')), + redaction_profile TEXT NOT NULL DEFAULT 'default', + artifact_uri TEXT, + summary JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS data_delete_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + requester_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + target_type TEXT NOT NULL CHECK (target_type IN ('user_data', 'project_data', 'export_data')), + status TEXT NOT NULL CHECK (status IN ('requested', 'approved', 'processing', 'completed', 'rejected')), + reason TEXT, + requested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS audit_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID REFERENCES organizations(id) ON DELETE CASCADE, + project_id UUID REFERENCES projects(id) ON DELETE CASCADE, + actor_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + actor_api_key_id UUID REFERENCES api_keys(id) ON DELETE SET NULL, + action TEXT NOT NULL, + target_type TEXT NOT NULL, + target_id TEXT, + ip_address INET, + user_agent TEXT, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- ========================= +-- Indexes +-- ========================= + +CREATE INDEX IF NOT EXISTS idx_projects_org ON projects(organization_id); +CREATE INDEX IF NOT EXISTS idx_projects_owner ON projects(owner_user_id); +CREATE INDEX IF NOT EXISTS idx_project_repo_repo ON project_repositories(repository_id); +CREATE INDEX IF NOT EXISTS idx_posts_repo_created ON posts(repository_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_posts_author_created ON posts(author_user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_comments_post_created ON comments(post_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_reactions_target ON reactions(target_type, target_id); +CREATE INDEX IF NOT EXISTS idx_follows_target_user ON follows(target_user_id); +CREATE INDEX IF NOT EXISTS idx_follows_target_repo ON follows(target_repository_id); +CREATE UNIQUE INDEX IF NOT EXISTS uq_follows_user_target + ON follows(follower_user_id, target_user_id) + WHERE target_type = 'user'; +CREATE UNIQUE INDEX IF NOT EXISTS uq_follows_repo_target + ON follows(follower_user_id, target_repository_id) + WHERE target_type = 'repository'; +CREATE INDEX IF NOT EXISTS idx_trend_scores_repo_window ON trend_scores(repository_id, window_end DESC); + +CREATE INDEX IF NOT EXISTS idx_checkpoints_project_created ON checkpoints(project_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_checkpoints_session ON checkpoints(session_id); +CREATE INDEX IF NOT EXISTS idx_checkpoint_files_path ON checkpoint_files(file_path); +CREATE INDEX IF NOT EXISTS idx_diffs_from_to ON diffs(from_checkpoint_id, to_checkpoint_id); +CREATE INDEX IF NOT EXISTS idx_messages_project_session ON messages(project_id, session_id, created_at); +CREATE INDEX IF NOT EXISTS idx_messages_checkpoint ON messages(checkpoint_id); +CREATE INDEX IF NOT EXISTS idx_tool_calls_project_session ON tool_calls(project_id, session_id, created_at); +CREATE INDEX IF NOT EXISTS idx_contexts_checkpoint ON contexts(checkpoint_id); +CREATE INDEX IF NOT EXISTS idx_restore_events_project_created ON restore_events(project_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_embeddings_project_checkpoint ON embeddings(project_id, checkpoint_id); +CREATE INDEX IF NOT EXISTS idx_embeddings_vector ON embeddings USING ivfflat (embedding vector_cosine_ops); +CREATE INDEX IF NOT EXISTS idx_indexed_docs_project ON indexed_docs(project_id); +CREATE INDEX IF NOT EXISTS idx_consents_scope ON consents(organization_id, project_id, consent_type, effective_from DESC); +CREATE INDEX IF NOT EXISTS idx_retention_scope ON retention_policies(organization_id, project_id, data_class); +CREATE INDEX IF NOT EXISTS idx_export_jobs_project_created ON export_jobs(project_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_delete_requests_status ON data_delete_requests(status, requested_at DESC); +CREATE INDEX IF NOT EXISTS idx_audit_project_created ON audit_logs(project_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_messages_fts + ON messages USING gin (to_tsvector('english', content)); + +COMMIT; diff --git a/INTEGRATION_PLAN.md b/INTEGRATION_PLAN.md new file mode 100644 index 00000000..c9ce1be2 --- /dev/null +++ b/INTEGRATION_PLAN.md @@ -0,0 +1,226 @@ +# External Repo Integration Plan + +Last updated: 2026-06-21 + +## Mission + +In the AI era, Git is a 20-year-old protocol and GitHub is a web UI bolted on top. +Neither was designed for AI agents, social-first development, or deterministic restore. + +- **Gitgram** replaces GitHub — repository-first social network for developers. +- **AITM** replaces Git — AI-native, deterministic, time-travel version control. + +This plan maps the 8 cloned reference repos into actual product source code. + +--- + +## 0. Executive Summary + +Both products are feature-complete with 1130 passing tests. This is **pattern/implementation uplift**, not a rebuild. + +Three repos are **not source-copy candidates**: +- `beads` — Go + Dolt standalone binary. Adopt the dependency-DAG workflow model, not the code. +- `gitnexus` — PolyForm Noncommercial license + native tree-sitter deps. Wrap as dependency or federated MCP; do not vendor source into commercial product until licensed. +- `proximity` — GPL-3. Reference only, zero code copied. + +Stack-mismatch correction: `t3-twitter-clone` is **Next.js + tRPC + Prisma**, NOT Express. It does not match `gitgram/backend`. Demoted to P2 reference. + +--- + +## 1. Integration Map + +### Priority legend +- **P0** — integrate first; highest value, lowest risk. +- **P1** — second pass; valuable but needs adaptation. +- **P2** — reference-grade or license-gated. + +--- + +### 1.1 `bluesky-social-app` (MIT) → Gitgram web + backend + mobile — P0 + +| Source | Lands in | Strategy | +|---|---|---| +| `src/lib/api/feed/merge.ts`, `feed/utils.ts`, `lib/api/feed-manip.ts` (FeedTuner) | `gitgram/backend/src/services/feedAlgorithm.ts` | Extract `MergeFeedAPI` round-robin merge + `FeedTuner` dedup/threading into existing service | +| `src/state/queries/notifications/{feed,util,types}.ts` | `gitgram/web` notification hooks + backend notification grouping | Adopt notification grouping ("X and 3 others liked") + cursor pagination shape | +| `src/lib/moderation/` | `gitgram/backend/src/services/moderationEnforcementService.ts` | Adopt label/cause-description taxonomy only | +| TanStack Query `createQueryKey` + `useInfiniteQuery` cursor conventions | `gitgram/web` data layer | Adopt convention for infinite-scroll lists | + +**Skip:** ALF design system, Lingui i18n, React Navigation, atproto agent layer, all view/screens UI. + +--- + +### 1.2 `munia` (MIT) → Gitgram web + backend — P0/P1 + +| Source | Lands in | Strategy | Priority | +|---|---|---|---| +| `src/app/api/comments/[commentId]/replies/*` + `verifyAccessToComment.ts` | `gitgram/backend` comment routes / `gitgram/web` comment threads | Nested-comment reply pagination + access-verification guard | P0 | +| `src/app/api/posts/hashtag/[hashtag]/GET.ts` | `gitgram/web/app/hashtag/[tag]/page.tsx` + backend `tags.ts` | Adopt hashtag-query pagination if cleaner than current | P1 | +| Drag-drop media upload components | `gitgram/web/components/composer/DropZone.tsx` | Cherry-pick a11y/keyboard handling only | P1 | +| Accessibility helpers | `gitgram/web/components/Accessibility.tsx` | Extract a11y patterns | P1 | + +**Skip:** NextAuth (conflicts with JWT auth), S3 upload (Gitgram has `uploadService.ts`), profile UI. + +--- + +### 1.3 `gh-social` (MIT) → Gitgram backend + web — P1 + +Unique value: **syncing GitHub follows/stars into the social graph** — closest open-source concept to Gitgram. + +| Source | Lands in | Strategy | +|---|---|---| +| `src/server/api/routers/github.ts` (profile/followers/following/repos sync) | `gitgram/backend/src/services/githubActivityService.ts` + `githubActivityFeedService.ts` | Port GitHub follow/star → social-feed sync; re-express as Express controllers (strip tRPC) | +| `src/server/caches/{followingsCache,usernameCache,oAuthCache}.ts` | `gitgram/backend/src/utils/cache.ts` / `redisStores.ts` | Redis cache key strategy for GitHub data | +| `src/server/helpers/trimGitHubData.ts` | `gitgram/backend/src/services` (new util) | Copy + adapt; pure function, low risk | + +**Skip:** Clerk auth (hard conflict), Drizzle/Neon (Gitgram uses Prisma/PG), Pusher (Gitgram uses Socket.io), tRPC, all UI. + +--- + +### 1.4 `nextjs-15-social-media-app` (codinginflow) → Gitgram web — P1 + +| Source | Lands in | Strategy | +|---|---|---| +| Infinite-scroll + React Query optimistic-update patterns | `gitgram/web/components/Feed.tsx`, `VirtualList.tsx`, infinite-scroll hooks | Adopt optimistic like/follow mutation patterns where Gitgram still does full refetch | +| Notification-system query/mutation shape | `gitgram/web` notification components | Cross-reference with bluesky §1.1 — pick the stronger | + +**Skip:** Prisma schema (Gitgram schema is canonical), auth, UI primitives. + +--- + +### 1.5 `t3-twitter-clone` (MIT) → reference only — P2 + +Stack mismatch: this is Next.js + tRPC + Prisma, not Express. Validate Gitgram's post data model against it; copy nothing structural. + +--- + +### 1.6 `proximity` (GPL-3) → Gitgram mobile reference ONLY — P2 + +**Zero code copied.** GPL-3 is incompatible. Use only for behavioral reference (GraphQL subscriptions → mobile real-time). Implement any adopted idea clean-room in `gitgram/mobile` using the existing Socket.io layer. Document in commit messages. + +--- + +### 1.7 `gitnexus` (PolyForm Noncommercial) → Gitgram repo explore + AITM — P1/P2 + +**License gate:** PolyForm Noncommercial likely incompatible with commercial products. Requires legal clearance before shipping. Two compliant integration modes: + +| Use | Lands in | Strategy | Priority | +|---|---|---|---| +| Gitgram repo explore/visualization | `gitgram/web/components/repo/RepoGraphTab.tsx` | Wrap as external tool / call CLI out-of-process. Never import source. | P2 (license-gated) | +| AITM code-context indexing + checkpoint knowledge graph | `aitimemachine/mcp/src/tools/` + `aitimemachine/src/core/neon/indexer.ts` | Pattern reference + optional MCP federation. Adopt impact/context/clusters concept into AITM's existing graph. No source copy. | P1 | + +**Action required:** legal review of PolyForm Noncommercial vs. commercial use before shipping either path. + +--- + +### 1.8 `beads` (MIT) → AITM CLI/MCP — P1 + +Go + Dolt binary. Adopt the **dependency-DAG issue model and `bd ready` semantics** into AITM's existing `checkpoint-graph.ts`. + +| Concept | Lands in | Strategy | +|---|---|---| +| Hash-based collision-free IDs (`bd-a1b2`) | `aitimemachine/src/core/checkpoint-graph.ts` | Add beads-style content-hash IDs for dependency nodes | +| `bd ready` = dependency-unblocked work detection | `aitimemachine/mcp/src/tools/` (new `ready`-style tool) | Expose "what work is unblocked" MCP tool over AITM's DAG | +| `relates_to`/`supersedes`/`replies_to` graph links | `aitimemachine/src/core/checkpoint-graph.ts` + `annotation-store.ts` | Map onto existing checkpoint relations | +| Optional: depend on beads binary | AITM agent workflow | Install `bd` as agent dev-tool (MIT permits) | + +**Skip:** Dolt storage (AITM uses SQLite + CAS + Neon), all Go code, server mode. + +--- + +## 2. Conflict Hotspots + +| Area | Conflict | Resolution | +|---|---|---| +| **Authentication** (HIGH) | munia/gh-social bring NextAuth/Clerk; Gitgram uses httpOnly-cookie JWT | Do not import any external auth. Extract OAuth scope/flow ideas only. | +| **ORM/DB driver** (HIGH) | gh-social uses Drizzle+Neon; Gitgram uses Prisma+PG | Re-express all donor queries in Prisma. Never introduce Drizzle into `gitgram/backend`. | +| **Backend framework** (HIGH) | gh-social/t3 tRPC routers vs. Gitgram Express controllers | Strip tRPC; re-implement as Express route + controller pairs. | +| **Real-time transport** (MEDIUM) | Pusher, GraphQL subs, atproto firehose | Map all real-time ideas onto existing Socket.io. Copy no transport client. | +| **Feed algorithm** (MEDIUM) | bluesky FeedTuner vs. existing `feedAlgorithm.ts` | Surgical merge inside `feedAlgorithm.ts`. TDD first; feed scoring covered by backend tests. | +| **Comment threading** (MEDIUM) | munia nested replies vs. `CommentThread.tsx` / `commentActionsController.ts` | Diff data shapes; adopt reply-pagination only if it doesn't break existing API contract. | +| **Checkpoint graph identity** (MEDIUM) | beads hash-IDs vs. AITM SHA256 CAS | AITM already content-addresses. Add dependency-edge semantics without changing existing checkpoint IDs (referenced in 520 core tests). | +| **License contamination** (CRITICAL) | proximity (GPL-3), gitnexus (PolyForm Noncommercial) | Never copy source. Add CI check blocking GPL/noncommercial imports in product trees. | + +--- + +## 3. Integration Order + +Each step: **(a) write tests first → (b) merge pattern → (c) run affected suite → (d) commit.** Never batch two donors in one commit. + +### Phase 0 — Guardrails (before any merge) +1. Add license-contamination CI check (block GPL/noncommercial imports in product trees). +2. Snapshot current suites as green baseline: AITM Core 520, AITM API 538, AITM VSCode 62, Gitgram Web 72. +3. Quarantine `external/` from product `tsconfig`/build paths. + +### Phase 1 — Pure additive, zero-conflict (P0) +4. `gh-social` `trimGitHubData` helper → Gitgram backend util (pure function, new file, new tests). +5. `bluesky` notification grouping/aggregation → backend service + web hooks (new aggregation layer; old endpoints untouched). +6. `munia` nested-comment reply pagination → backend (additive route variant; keep old contract). + +### Phase 2 — Surgical merges into existing services (P0/P1) +7. `bluesky` `FeedTuner`/merge logic → `feedAlgorithm.ts` (TDD around ranking). +8. `nextjs-15-social-media-app` + `bluesky` optimistic-update patterns → Gitgram web Feed. +9. `gh-social` GitHub follow/star→social-graph sync → `githubActivityService.ts`. +10. `munia` accessibility + drag-drop a11y → composer components. + +### Phase 3 — AITM workflow adoption (P1) +11. `beads` dependency-DAG + `bd ready` semantics → `checkpoint-graph.ts` + new MCP "ready" tool. +12. `gitnexus` impact/context concepts → AITM graph OR federated MCP registration (no source copy). + +### Phase 4 — License-gated / reference (P2) +13. `gitnexus` Gitgram repo-graph tab — only after legal clearance. +14. `proximity` mobile real-time ideas — clean-room reimplementation only. +15. `t3-twitter-clone` — data-model validation pass only. + +--- + +## 4. What to Skip + +| External | Skip because we already have it | +|---|---| +| `munia` S3 upload, NextAuth, profile UI | `uploadService.ts`, `mediaService.ts`, JWT auth, full profile pages | +| `gh-social` Clerk, Drizzle, Pusher, tRPC, all `components/` | own auth, Prisma, Socket.io, Express, 80+ web components | +| `t3-twitter-clone` entire backend + auth + UI | stack mismatch; Gitgram post/retweet done | +| `nextjs-15-social-media-app` Prisma schema, auth, UI primitives | canonical schema, auth, UI system exist | +| `bluesky` ALF, Lingui, React Navigation, atproto agent, `view/`/`screens/` | own design system, routing, transport | +| `beads` Dolt engine, Go code, server mode | SQLite + CAS + Neon; content-addressing exists | +| `gitnexus` ingestion/parser source, LadybugDB, embeddings pipeline | `neon/indexer.ts`, `search-engine.ts`, `checkpoint-graph.ts`; license blocks vendoring | +| `proximity` everything (source) | GPL-3; mobile features already implemented | + +--- + +## 5. Key File Paths + +### Gitgram integration targets +- `gitgram/backend/src/services/feedAlgorithm.ts` +- `gitgram/backend/src/services/githubActivityService.ts` +- `gitgram/backend/src/services/githubActivityFeedService.ts` +- `gitgram/backend/src/services/moderationEnforcementService.ts` +- `gitgram/backend/src/services/mentionService.ts` +- `gitgram/backend/src/controllers/commentActionsController.ts` +- `gitgram/web/components/Feed.tsx` +- `gitgram/web/components/CommentThread.tsx` +- `gitgram/web/components/composer/DropZone.tsx` +- `gitgram/web/components/repo/` (new `RepoGraphTab.tsx`) + +### AITM integration targets +- `aitimemachine/src/core/checkpoint-graph.ts` +- `aitimemachine/src/core/neon/indexer.ts` +- `aitimemachine/src/core/search-engine.ts` +- `aitimemachine/mcp/src/tools/register-core-tools.ts` + +### Key donor files +- `external/bluesky-social-app/src/lib/api/feed/merge.ts` +- `external/bluesky-social-app/src/state/queries/notifications/` +- `external/bluesky-social-app/src/lib/moderation/` +- `external/munia/src/app/api/comments/[commentId]/replies/` +- `external/gh-social/src/server/api/routers/github.ts` +- `external/gh-social/src/server/caches/` +- `external/beads/` (pattern reference) + +--- + +## 6. Decisions Requiring Sign-off + +1. **gitnexus license** — PolyForm Noncommercial likely incompatible with commercial Gitgram. Decide: commercial license, MCP-federation only, or drop. Blocks §1.7 Gitgram path. +2. **beads as dependency vs. pattern** — install `bd` binary (MIT, fine) for agent task tracking, or reimplement DAG semantics natively in AITM. +3. **Feed ranking changes** — any merge into `feedAlgorithm.ts` changes user-visible ordering. Confirm before Phase 2 step 7. diff --git a/NORTH_STAR_PRD_ARCHITECTURE_V1.md b/NORTH_STAR_PRD_ARCHITECTURE_V1.md new file mode 100644 index 00000000..ac4b2fc8 --- /dev/null +++ b/NORTH_STAR_PRD_ARCHITECTURE_V1.md @@ -0,0 +1,242 @@ +# North Star PRD + Architecture v1 + +## 1) Product Vision + +Build a unified platform with two products: + +- `gitgram`: a vertical GitHub social network where repositories become social profiles and feeds (posts, reels, stories, comments, reactions, follows, trending discovery). +- `aitimemachine` (AITM): a "Time Machine for AI coding" that can restore code and chat context to exact previous states, inspect diffs, undo mistakes, and restore file/folder/project selectively. + +Core thesis: +- Social coding (`gitgram`) creates engagement around repositories and developers. +- Trust and recoverability (`aitimemachine`) makes AI coding safe enough for daily professional use. + +## 2) Product Boundaries + +`gitgram` v1 includes: +- Import and mirror public GitHub repositories into social repository pages. +- Post media (screenshots/videos) linked to repositories and code snippets. +- Comment/reaction/follow system for repositories and developers. +- Trending feed based on repository activity plus social engagement. + +`aitimemachine` v1 includes: +- Point-in-time restore for full project, folder, or single file. +- Before/after checkpoints around AI actions. +- Code diff and timeline navigation. +- Chat + tool-call context restoration linked to checkpoints. +- Search across code history and conversation history. + +Out of scope for v1: +- Enterprise SSO/SAML hardening beyond initial scaffolding. +- Fully autonomous research/oracle workflows as a primary UX. +- Private repo ingest without explicit legal + security controls. + +## 3) Architecture Decisions (Locked) + +### 3.1 Source of Truth + +Decision: +- Backend database + object storage is the authoritative source of truth. +- Local `.aitimemachine/` storage is a performance cache and offline buffer, not the canonical state. + +Why: +- Cross-device restore. +- Crash resilience. +- Centralized auth, audit, retention, and deletion controls. + +### 3.2 Storage Model + +Use a hybrid Git-like model: +- `blobs` for content-addressed file data. +- `trees` for directory states. +- `checkpoints` for commit-like snapshots. +- `diffs` precomputed for fast timeline UX. +- `messages`, `tool_calls`, `contexts` linked to checkpoints. + +### 3.3 Capture Model + +Every AI code action records: +1. `before_action` checkpoint. +2. tool call + args + initiating message link. +3. `after_action` checkpoint. +4. diff generation and change classification. +5. optional async indexing/embeddings. + +This guarantees deterministic "go back before error" behavior. + +### 3.4 Privacy + Training Data Policy + +Decision: +- Data capture is opt-in per workspace/org for training export. +- Default mode: operational capture only (restore/search), no training export. +- Redaction pipeline required before any dataset export. + +## 4) Unified Domain Model + +Primary entities: +- `users`, `organizations`, `projects`, `repositories`. +- Social: `repo_profiles`, `posts`, `reels`, `stories`, `comments`, `reactions`, `follows`, `trend_scores`. +- Time machine: `checkpoints`, `checkpoint_files`, `blobs`, `trees`, `diffs`, `messages`, `tool_calls`, `contexts`, `restore_events`. +- Governance: `consents`, `retention_policies`, `audit_logs`, `export_jobs`. + +Entity links: +- A `repository` can have a `repo_profile` on gitgram. +- A `project` in AITM can map to one or more repositories. +- AITM checkpoints can be referenced by gitgram posts (for "before/after reels" and explainable dev content). + +## 5) API Surface v1 (High-Level) + +Social APIs: +- `POST /api/repos/import` +- `GET /api/repos/:id/feed` +- `POST /api/posts` +- `POST /api/comments` +- `POST /api/reactions` +- `POST /api/follows` +- `GET /api/trending` + +AITM APIs: +- `POST /api/actions/before` +- `POST /api/actions/after` +- `POST /api/time/restore` +- `POST /api/time/undo` +- `GET /api/time/timeline` +- `GET /api/time/diff` +- `POST /api/time/restore-file` +- `POST /api/time/restore-folder` +- `GET /api/context/:checkpointId` + +Governance APIs: +- `POST /api/consent` +- `POST /api/export/training` +- `POST /api/data/delete-request` +- `GET /api/audit` + +## 6) UX Principles + +Gitgram UX: +- Repository-first profile pages. +- Media-native storytelling for code changes. +- Fast interaction loop: watch, react, comment, fork/follow. + +AITM UX: +- "Restore confidence in one click." +- Timeline with human-readable events ("before create_file src/api.ts"). +- Clear scope controls: restore file, folder, or full workspace. +- Always show preview diff before irreversible restore actions. + +## 7) Security, Compliance, and Risk Controls + +Minimum controls for production: +- Tenant isolation at query and storage layers. +- Scoped API keys and per-project authorization. +- Encryption at rest and in transit. +- Audit logging for restore/export/delete operations. +- Retention and hard-delete workflows. +- PII/secret redaction before training export. + +Key risks: +- Capturing sensitive code/prompts without explicit consent. +- Licensing conflicts when combining public repo/social content and training datasets. +- Data poisoning and abuse in social media uploads. + +## 8) 90-Day Execution Plan + +### Milestone A (Days 1-30): Core Reliability +- Finalize canonical schema for checkpoints/messages/tool calls. +- Ship deterministic restore for file/folder/project. +- Add consent + retention enforcement paths. +- Stand up baseline social repo feed with comments/reactions/follows. + +Exit criteria: +- Restore success rate >= 99%. +- P95 restore latency <= 2s for single file and <= 10s for medium project. +- 0 known auth bypasses in internal security review. + +### Milestone B (Days 31-60): Social + Time Integration +- Link gitgram posts to AITM checkpoint references. +- Add "change story" post type from diff snapshots. +- Ship trending ranking combining GitHub + on-platform signals. +- Release org/project scoped policies. + +Exit criteria: +- >= 25% of active repos receive at least one social interaction. +- >= 40% of restores initiated from timeline UI complete without retry. + +### Milestone C (Days 61-90): Intelligence + Growth +- Add semantic search on code/checkpoint history. +- Add high-trust training export pipeline (opt-in + redaction + audit). +- Launch creator features for repo owners (story/reel workflow). + +Exit criteria: +- Search relevance accepted by internal benchmark. +- 100% training exports traceable by consent + audit IDs. + +## 9) KPIs + +Product KPIs: +- Weekly active developers. +- Restore actions per active AITM user. +- Social interactions per repository page. +- 7-day retention for users who performed at least one restore. + +System KPIs: +- Checkpoint ingestion throughput. +- Restore success/latency percentiles. +- Diff generation latency. +- Sync lag (local cache to backend). + +Safety KPIs: +- Consent coverage. +- Redaction precision/recall on sampled exports. +- Mean time to resolve data deletion requests. + +## 10) Immediate Next Actions + +1. Treat this file as canonical for v1 decisions. +2. Update conflicting docs to reference these decisions (especially source-of-truth and privacy policy). +3. Create implementation tickets from sections 3, 7, and 8 first (architecture lock + security gates before feature acceleration). + +## 11) Cleanup Completed (2026-03-02) + +The following code quality issues from the original audit have been resolved: + +### Gitgram - Fixed +- **Route duplication** - Removed 17 legacy `/api/` routes that duplicated `/api/v1/` endpoints +- **Socket.IO deprecated service still used** - Fully migrated to `useSocket` hook; deleted `socketService.ts`; removed from `authStore`, `notifications/page.tsx`, `messages/page.tsx` +- **40+ console.log in backend** - Replaced ~52 `console.*` calls with Winston logger across 10 controller/util files +- **125+ markdown clutter files** - Deleted 121 .md + 8 .txt status/completion files; kept only README, CLAUDE, COMMANDS, TROUBLESHOOTING +- **@types in wrong dependency section** - Moved `@types/node-cron`, `@types/qrcode` to devDependencies + +### AITM - Fixed +- **No disk-full handling** - ContentStore.put() now uses atomic write (.tmp + rename) with ENOSPC catch and cleanup +- **No rate limiting on MCP tools** - Added sliding-window rate limiter (restore*: 5/min, execute_command: 30/min, before/after_action: 60/min, default: 120/min) +- **API response inconsistency** - Created `sendSuccess`/`sendError`/`sendPaginated` helpers; applied to dashboard + search routes as starting examples +- **44 markdown clutter files** - Deleted; kept only README, CLAUDE, CHANGELOG, API, MCP, DEVELOPMENT, QUICKSTART, ROADMAP +- **@types in wrong dependency section** - Moved `@types/chokidar`, `@types/jsonwebtoken`, `@types/pg` to devDependencies + +### Fixed (Phases 7-10, 2026-03-02) +- **Centralized error handling** — ApiError class + Errors factory applied to all 50 controllers via asyncHandler +- **Route consolidation** — Merged authNew→auth, deleted dead postsNew/postsController/postControllerUpdate +- **Swagger/OpenAPI** — 6 new schemas, annotations on 5 route files +- **Backend test infrastructure** — Mocked Prisma/Redis/auth/external services; 6 new test files (205+ tests) +- **N+1 query fix** — analyticsController uses aggregate/count instead of findMany+JS reduce +- **Redis cache** — Unified cacheMiddleware (Redis primary + in-memory fallback, X-Cache headers) +- **WebSocket hardening** — Rate limiting, input validation, heartbeat config, map cleanup +- **Health endpoints** — Extracted to /health, /health/ready, /health/db +- **AITM CI** — Added api-server-tests job +- **Structured logging** — console.log→logger in env-contract/index, request correlation IDs +- **Monorepo tooling** — Root package.json (test:all, lint:all, build:all), shared ESLint/Prettier +- **AITM Sentry** — @sentry/node integrated in api-server +- **cli-v2.ts refactored** — Modularized from 4,381 LOC to 91 LOC (done in Phase 6) +- **httpOnly cookies** — Already implemented; frontend uses withCredentials:true, localStorage only caches user metadata + +### Fixed (Tasks 9-11, 2026-03-02) +- **AITM API URL alignment** — corrected mismatched AITM routes in integration service (`/restore/projects`, `/time/diff`, `/context/:id`) +- **Abuse detection on Redis** — all three abuse layers now use Redis-backed stores with in-memory fallback (`RedisSlidingWindow` + `RedisHashStore`) +- **TypeScript strict mode** — Gitgram backend already runs with `strict: true` and compiles clean + +### Remaining Issues (Not Yet Fixed) +- Zero web/mobile tests +- 839 uses of `any` in AITM +- ~~Cross-product linking (posts <-> checkpoints)~~ — ✅ ALREADY BUILT: 7-step wizard (create-change-story), AitmChangeStoryCard renderer, aitmService API client, RepoAitmSettings, RepoCheckpointSection, CHANGE_STORY post type, checkpointRef field, webhook auto-generation diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 00000000..978873af --- /dev/null +++ b/PLAN.md @@ -0,0 +1,163 @@ +# gitgram-ai Development Plan + +Last updated: 2026-03-02 + +## Current State + +Both products are feature-complete for v1 and compile/build clean. A monorepo cleanup session on 2026-03-02 resolved significant code hygiene debt. Neither product is publicly usable yet due to operational blockers. + +### What Was Fixed (2026-03-02 Cleanup) + +| Area | Change | Impact | +|------|--------|--------| +| Gitgram markdown clutter | Deleted 121 .md + 8 .txt status files | Repo is navigable | +| AITM markdown clutter | Deleted 46 status .md files | Repo is navigable | +| Legacy route duplication | Removed 17 `/api/` routes duplicating `/api/v1/` | Halved attack surface | +| console.* sprawl | Replaced ~52 calls with Winston logger (10 files) | Structured logging | +| socketService deprecation | Full migration to useSocket hook; service deleted | Clean React patterns | +| @types misconfig | Moved 5 @types packages to devDependencies | Correct dependency graph | +| AITM disk-full handling | Atomic write + ENOSPC catch in ContentStore.put() | Data safety | +| MCP rate limiting | Sliding-window limiter (5-120/min per tool) | Abuse prevention | +| API response helpers | sendSuccess/sendError/sendPaginated created | Consistency foundation | + +### What Was Fixed (2026-03-02 Follow-up: Tasks #9-#11) + +| Area | Change | Impact | +|------|--------|--------| +| AITM API URL alignment | Fixed mismatched endpoints in backend AITM service (`/restore/projects`, `/time/diff`, `/context/:id`) | Prevents integration 404s | +| Abuse detection state | Migrated spam guard stores to `RedisHashStore` with in-memory fallback | Multi-instance readiness with dev safety fallback | +| TypeScript strict mode | Confirmed `strict: true` in backend `tsconfig.json` and clean compile | Type-safety enforcement active | + +### What's Still Broken + +| Issue | Product | Severity | Est. Effort | +|-------|---------|----------|-------------| +| CLI/MCP not on npm | AITM | **Blocker** | 1-2 hours | +| No production secrets configured | Both | **Blocker** | 2-4 hours | +| localStorage for JWT tokens | Gitgram | High | 4-6 hours | +| cli-v2.ts mega-file (4,381 LOC) | AITM | High | 4-6 hours | +| Zero web/mobile tests | Gitgram | High | Multi-day | +| 839 uses of `any` | AITM | High | 8-12 hours | +| Cross-product linking | Both | Medium | Multi-week | + +--- + +## Phase 3: Launch Unblockers (Next Session) + +Goal: Make both products usable by real humans. + +### 3.1 Publish AITM to npm (Blocker #1) +- Verify `package.json` metadata (name, version, bin, files, repository) +- Run `npm pack` and inspect tarball contents +- Publish `aitimemachine` CLI to npm +- Publish MCP server package to npm +- Verify `npm install -g aitimemachine && aitm --version` works +- Update website install instructions if needed + +### 3.2 Configure production secrets (Blocker #2) +- Create GitHub OAuth App (Client ID + Secret) +- Create Google OAuth credentials +- Set up Polar.sh keys (access token + webhook secret + product IDs) +- Verify Resend email service (domain + API key) +- Set up Clerk production instance for AITM +- Set up Polar.sh billing keys for AITM +- Configure Sentry DSNs for error tracking (both products) +- Document all keys in `.env.example` files + +### 3.3 Deploy infrastructure +- Provision Hetzner VPS (run `deploy/setup-hetzner.sh`) +- Configure Cloudflare DNS (gitgram.app, timemachine.gitgram.app, timemachine-api.gitgram.app) +- Fill production `.env` files from templates +- Run `docker compose -f docker-compose.prod.yml up -d` +- Verify health endpoints respond +- Set up UptimeRobot monitoring + +### 3.4 End-to-end smoke test +- AITM: `npm install -g` -> `aitm init` -> `aitm checkpoint` -> dashboard shows real data +- Gitgram: signup -> GitHub OAuth -> create post -> comment -> notification + +--- + +## Phase 4: Security Hardening (~1 week) + +### 4.1 Migrate JWT storage to httpOnly cookies +- Backend: Set tokens via `Set-Cookie` with `httpOnly`, `secure`, `sameSite=strict` +- Frontend: Remove `localStorage.getItem('token')` pattern +- Update API client to stop sending `Authorization` header (cookies auto-sent) +- Update CORS config for credential passing +- Test: full auth flow, refresh token rotation, logout + +### 4.2 Move abuse detection to Redis (Completed 2026-03-02) +- Replace in-memory Maps with Redis sorted sets +- Key pattern: `abuse:{userId}:{action}` with TTL-based expiry +- Works across multiple backend instances +- Preserve existing threshold configs + +### 4.3 Validate AITM callback URLs +- Whitelist allowed OAuth redirect URIs +- Reject open redirect attempts +- Add state parameter CSRF validation to browser login flow + +--- + +## Phase 5: Code Quality (~2 weeks) + +### 5.1 Enable TypeScript strict mode in Gitgram backend (Completed 2026-03-02) +- Set `strict: true` in `tsconfig.json` +- Fix errors file-by-file (prioritize controllers > services > utils) +- Target: eliminate 1,070 uses of `any` + +### 5.2 Break up AITM cli-v2.ts +- Create `src/commands/` directory +- Extract each command group into its own module (checkpoint.ts, restore.ts, search.ts, etc.) +- cli-v2.ts becomes a thin shell that registers commands +- Target: no file > 300 LOC + +### 5.3 Add frontend test coverage +- Set up Vitest + Testing Library for Gitgram web +- Write tests for auth flows, post creation, notification rendering +- Set up Playwright for critical path E2E tests +- CI: run tests on PR + +### 5.4 Finish API response helper migration (AITM) +- Apply `sendSuccess`/`sendError`/`sendPaginated` to all remaining routes +- Remove ad-hoc response shapes +- Update API docs/Swagger + +--- + +## Phase 6: Cross-Product Integration (~3 weeks) + +### 6.1 Link gitgram posts to AITM checkpoints +- Add `checkpointRef` field to Gitgram Post model +- Create "Change Story" post type (auto-generated from AITM diffs) +- API: `POST /api/v1/posts` accepts optional `aitm_checkpoint_id` +- UI: "Share as post" button in AITM timeline + +### 6.2 Semantic search for AITM +- Add embedding generation on checkpoint creation (OpenAI or local model) +- Store embeddings in pgvector column +- API: `POST /api/search/semantic` with cosine similarity ranking +- CLI: `aitm search --semantic "when did I add auth?"` + +### 6.3 AITM automatic GC +- Schedule GC worker (daily cron) +- Configurable retention: keep last N checkpoints, or last N days +- Clean orphaned blobs after checkpoint deletion +- User-facing storage usage dashboard + +--- + +## Operational Checklist (Pre-Launch) + +- [ ] npm packages published (aitimemachine, MCP server) +- [ ] Production VPS provisioned and Docker running +- [ ] DNS configured (both domains) +- [ ] All production secrets in `.env` +- [ ] Health endpoints verified (`/health` returns 200) +- [ ] Error tracking live (Sentry) +- [ ] Analytics live (PostHog or Plausible) +- [ ] Uptime monitoring configured +- [ ] SSL certificates active (Cloudflare) +- [ ] Backup strategy for Postgres (Neon auto-backup or pg_dump cron) +- [ ] End-to-end smoke test passes for both products diff --git a/PRIORITIZED_TICKETS_V1.md b/PRIORITIZED_TICKETS_V1.md new file mode 100644 index 00000000..922318e0 --- /dev/null +++ b/PRIORITIZED_TICKETS_V1.md @@ -0,0 +1,307 @@ +# gitgram-ai Prioritized Engineering Tickets (v1) + +Last updated: 2026-03-02 +Source: `SINGLE_SOURCE_PLAN.md` + +## Conventions + +- Priority: `P0` blocker, `P1` high, `P2` medium, `P3` growth +- Estimate scale: `S` (0.5-1 day), `M` (1-3 days), `L` (3-5 days), `XL` (5+ days) +- Owner is a role placeholder for assignment + +## P0 - Launch Blockers + +### GGAI-001 - Publish AITM CLI to npm +- Priority: P0 +- Owner: AITM Backend Engineer +- Estimate: M +- Dependencies: None +- Scope: + - Verify CLI package metadata (`name`, `version`, `bin`, `files`, `repository`) + - Run `npm pack` and inspect tarball + - Publish package + - Validate global install and command boot +- Acceptance criteria: + - `npm install -g ` succeeds on clean environment + - `aitm --version` returns published version + - Package includes required runtime files and excludes local-only artifacts + +### GGAI-002 - Publish AITM MCP server package to npm +- Priority: P0 +- Owner: AITM Backend Engineer +- Estimate: M +- Dependencies: GGAI-001 +- Scope: + - Verify package metadata and export entrypoints + - Publish MCP server package + - Validate install and startup command +- Acceptance criteria: + - MCP package installs from npm without local path assumptions + - Server starts and responds to a health/ping call + - Package README install instructions are accurate + +### GGAI-003 - Configure production secrets for both products +- Priority: P0 +- Owner: DevOps Engineer +- Estimate: L +- Dependencies: None +- Scope: + - Provision and store OAuth, billing, email, and error-tracking keys + - Set secure values for Gitgram and AITM production environments + - Align `.env.example` keys with actual runtime requirements +- Acceptance criteria: + - All required env vars exist in production secret store + - Services boot without missing-env startup errors + - `.env.example` files are complete and current + +### GGAI-004 - Deploy production baseline infrastructure +- Priority: P0 +- Owner: DevOps Engineer +- Estimate: L +- Dependencies: GGAI-003 +- Scope: + - Provision VPS and system prerequisites + - Configure DNS, TLS, and production compose stack + - Configure uptime monitoring and baseline alerts +- Acceptance criteria: + - Public domains resolve correctly + - SSL/TLS active for all public endpoints + - `docker compose -f docker-compose.prod.yml up -d` stable with healthy services + - Monitoring shows green status for required endpoints + +### GGAI-005 - Execute end-to-end smoke verification +- Priority: P0 +- Owner: QA Engineer +- Estimate: M +- Dependencies: GGAI-001, GGAI-002, GGAI-004 +- Scope: + - Validate AITM install/init/checkpoint/timeline flow + - Validate Gitgram signup/oauth/post/comment/notification flow + - Record failures with logs and repro steps +- Acceptance criteria: + - Both critical user journeys pass end-to-end in production-like environment + - No blocker-severity defects remain open + - Smoke test checklist is documented and repeatable + +## P1 - Security Closure + +### GGAI-006 - Migrate Gitgram JWT handling to httpOnly cookies +- Priority: P1 +- Owner: Gitgram Full-Stack Engineer +- Estimate: L +- Dependencies: GGAI-005 +- Scope: + - Replace `localStorage` token flow with secure cookie flow + - Update backend auth cookie issuance and frontend API client behavior + - Update CORS and CSRF assumptions as needed +- Acceptance criteria: + - No auth token persistence in `localStorage` + - Login, refresh, and logout flows pass in browser validation + - Cookies set with `httpOnly`, `secure`, and `sameSite` policy + +### GGAI-007 - Harden AITM OAuth callback and redirect validation +- Priority: P1 +- Owner: AITM Backend Engineer +- Estimate: M +- Dependencies: GGAI-005 +- Scope: + - Enforce allowlist for redirect URIs + - Reject open redirects + - Validate CSRF state parameter on OAuth callback path +- Acceptance criteria: + - Open redirect attempts are rejected + - Missing/invalid state values fail safely + - Valid OAuth flows remain functional across supported providers + +### GGAI-008 - Complete AITM API response helper migration +- Priority: P1 +- Owner: AITM Backend Engineer +- Estimate: M +- Dependencies: GGAI-005 +- Scope: + - Apply `sendSuccess`, `sendError`, `sendPaginated` to remaining routes + - Remove inconsistent ad-hoc response shapes + - Update API docs/examples +- Acceptance criteria: + - All AITM API routes use standardized response helpers + - Error/status structure is consistent across endpoints + - API docs reflect final response contracts + +### GGAI-009 - Validate retention/delete/export audit workflows +- Priority: P1 +- Owner: Security Engineer +- Estimate: M +- Dependencies: GGAI-003, GGAI-008 +- Scope: + - Execute staging tests for consent, retention policy, delete request, and export auditability + - Confirm audit log completeness for restore/export/delete operations +- Acceptance criteria: + - Each data governance path has a passing test record + - Audit logs contain actor, action, timestamp, and trace identifiers + - No critical compliance gaps remain in staging checklist + +## P2 - Reliability and Code Quality + +### GGAI-010 - Refactor AITM cli-v2.ts into modular command architecture +- Priority: P2 +- Owner: AITM Backend Engineer +- Estimate: L +- Dependencies: GGAI-005 +- Scope: + - Extract command groups into `src/commands/*` + - Keep root CLI file as registration/bootstrap shell + - Preserve existing command UX and flags +- Acceptance criteria: + - No single CLI file exceeds 300 LOC target + - CLI behavior remains backward compatible for existing commands + - Build and command smoke tests pass + +### GGAI-011 - Reduce unsafe `any` usage on AITM critical paths +- Priority: P2 +- Owner: AITM Backend Engineer +- Estimate: XL +- Dependencies: GGAI-010 +- Scope: + - Prioritize controllers and services for strict typing improvements + - Introduce reusable DTO/types where repeated shapes exist + - Avoid broad `as unknown as` type escapes +- Acceptance criteria: + - Significant reduction of `any` usage in controller/service directories + - TypeScript strict build remains clean + - No runtime regressions in checkpoint/restore/search flows + +### GGAI-012 - Add Gitgram frontend unit/integration test stack +- Priority: P2 +- Owner: Frontend Engineer +- Estimate: L +- Dependencies: GGAI-006 +- Scope: + - Configure Vitest + Testing Library + - Add tests for auth flows, post creation, notification rendering + - Integrate into project scripts +- Acceptance criteria: + - Tests run in CI and local environments + - Core flows have deterministic passing tests + - Failures produce actionable output + +### GGAI-013 - Add Playwright critical-path E2E and CI gate +- Priority: P2 +- Owner: QA Engineer +- Estimate: L +- Dependencies: GGAI-012 +- Scope: + - Implement E2E coverage for highest-risk user journeys + - Add CI workflow gate for pull requests + - Add retry/flakiness guardrails +- Acceptance criteria: + - PR CI blocks on E2E failure for protected branches + - E2E suite covers login, posting, and timeline-critical flows + - Flakiness is below agreed threshold + +## P3 - Cross-Product Integration and Growth + +### GGAI-014 - Add checkpoint reference field to Gitgram posts +- Priority: P3 +- Owner: Gitgram Backend Engineer +- Estimate: M +- Dependencies: GGAI-005 +- Scope: + - Add model/schema field for AITM checkpoint linkage + - Extend create post API with optional checkpoint reference + - Validate ownership and access permissions +- Acceptance criteria: + - Post creation accepts and stores valid checkpoint reference + - Unauthorized checkpoint linkage attempts are rejected + - API docs include new field contract + +### GGAI-015 - Ship AITM "Share as Post" flow +- Priority: P3 +- Owner: Full-Stack Engineer +- Estimate: L +- Dependencies: GGAI-014 +- Scope: + - Add action in AITM timeline to publish linked content to Gitgram + - Include metadata payload for checkpoint context and diff summary + - Handle auth/session boundary between products +- Acceptance criteria: + - User can publish from AITM timeline to Gitgram in one flow + - Published post contains valid checkpoint linkage and preview content + - Error handling covers auth failure and unavailable target service + +### GGAI-016 - Implement "Change Story" post type from AITM diffs +- Priority: P3 +- Owner: Full-Stack Engineer +- Estimate: L +- Dependencies: GGAI-015 +- Scope: + - Define post subtype and rendering format for before/after diff narratives + - Generate summary payload from checkpoint diffs + - Add UI treatment in feed and repository profile views +- Acceptance criteria: + - Change Story posts render with consistent schema and UI + - Diff-backed story content links to source checkpoint data + - Feed interactions (comment/react/share) work unchanged + +### GGAI-017 - Launch unified trend ranking (GitHub + platform signals) +- Priority: P3 +- Owner: Data Engineer +- Estimate: L +- Dependencies: GGAI-016 +- Scope: + - Build ranking formula combining repository activity and social engagement + - Add score computation job and API exposure + - Add basic anti-gaming controls +- Acceptance criteria: + - Trending endpoint returns deterministic ranked results + - Ranking inputs are documented and observable + - No critical abuse vector in initial review + +### GGAI-018 - Add AITM semantic search over code/checkpoint history +- Priority: P3 +- Owner: AITM Backend Engineer +- Estimate: XL +- Dependencies: GGAI-011 +- Scope: + - Generate embeddings at checkpoint creation + - Store vectors and serve semantic retrieval API + - Expose CLI/query interface for semantic search +- Acceptance criteria: + - Semantic query returns ranked relevant checkpoint results + - Performance meets agreed p95 threshold for target corpus size + - Relevance passes internal benchmark gate + +### GGAI-019 - Implement training export pipeline with consent, redaction, audit +- Priority: P3 +- Owner: Security Engineer +- Estimate: XL +- Dependencies: GGAI-009 +- Scope: + - Enforce opt-in consent checks prior to export jobs + - Run redaction pipeline before data leaves operational storage + - Persist full export lineage and audit metadata +- Acceptance criteria: + - Exports fail closed without valid consent + - Redaction pipeline executes and produces measurable precision/recall report + - Every export has traceable consent ID and audit record + +### GGAI-020 - Creator workflow tools for repository owners +- Priority: P3 +- Owner: Product Engineer +- Estimate: L +- Dependencies: GGAI-016 +- Scope: + - Add story/reel workflow tooling around code change events + - Provide templates for explainable dev content + - Instrument adoption analytics +- Acceptance criteria: + - Repo owners can create and publish creator-format content from product UI + - Workflow usage metrics are captured and queryable + - No regression in existing post creation flows + +## Suggested Sprint Packaging + +- Sprint 1: GGAI-001 to GGAI-005 +- Sprint 2: GGAI-006 to GGAI-009 +- Sprint 3: GGAI-010 to GGAI-013 +- Sprint 4+: GGAI-014 onward by product strategy and capacity + diff --git a/README.md b/README.md new file mode 100644 index 00000000..0960860a --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +# gitgram-ai (Monorepo) + +Two products: + +1. `gitgram` (GitHub social network) +2. `aitimemachine` (AITM, "Time Machine for AI coding") + +This repo is organized as a monorepo for local development and ops docs. Each product has its own `README.md`, and the repo still contains historical automation artifacts that are not the launch source of truth. + +## Quick Start (Local, Full Stack) + +Runs Postgres + Redis + Gitgram backend + Gitgram web + AITM API via Docker Compose. + +```bash +docker compose up +``` + +Service URLs (when running the full stack together): +- Gitgram web: `http://localhost:3000` +- Gitgram backend: `http://localhost:4000` (`GET /health`) +- AITM API: `http://localhost:3001` (`GET /health`) + +Notes: +- AITM website is not started by the root `docker-compose.yml` (to avoid port conflicts with Gitgram web). Run it separately if needed. +- AITM database defaults to whatever `aitimemachine/api-server/.env` provides (`DATABASE_URL`). In production, use Neon; locally you can use a local Postgres if you want. + +## Start AITM Website Locally + +When Gitgram web is already on `:3000`, run AITM website on `:3002`: + +```bash +cd aitimemachine/website +cp .env.example .env.local +PORT=3002 npm run dev +``` + +Open `http://localhost:3002`. + +## Production Compose (Single VPS) + +Use `docs/runbook.md` as the canonical manual launch contract. + +`docker-compose.prod.yml` is retained as a non-canonical full-docker reference, not the launch source of truth. + +## Release Gates + +Run these before staging/production deploys: + +```bash +npm run verify:aitm-packages +npm run verify:aitm-local +npm run verify:secrets:staging +npm run verify:secrets:production +``` + +AITM API local startup without external secret keys: + +```bash +npm run verify:aitm-local +npm run dev:aitm-api:clean +``` + +Stage A local smoke gate: + +```bash +npm run smoke:local-launch +``` + +Optional flags: +- `SKIP_HTTP=1 npm run smoke:local-launch` +- `AUTO_START=1 npm run smoke:local-launch` +- `RUN_AITM_CLI_SMOKE=1 npm run smoke:local-launch` +- `RUN_AITM_MCP_SMOKE=1 npm run smoke:local-launch` + +## Docs Index + +- Strict execution queue: `REMAINING.md` +- North star PRD/architecture: `NORTH_STAR_PRD_ARCHITECTURE_V1.md` +- Manual launch runbook: `docs/runbook.md` +- Deploy pipeline contract: `docs/deploy-pipeline.md` +- Env var matrix: `docs/environment-matrix.md` +- Secrets contract: `docs/secrets-contract.md` diff --git a/REMAINING.md b/REMAINING.md new file mode 100644 index 00000000..4a4be36e --- /dev/null +++ b/REMAINING.md @@ -0,0 +1,146 @@ +# V1 Strict Execution Queue + +Status legend: `DONE`, `IN_PROGRESS`, `TODO`, `BLOCKED` +Ordering rule: highest dependency + launch risk first, then product acceleration. + +## Queue (Strict Order) + +1. `DONE` **[#20] Tenant isolation enforcement + tests across all data access paths** + - Depends on: none + - Why first: hard security gate for all products. + - Progress: scoped reads/writes/events/stats and tree-storage isolation in sqlite_repo.ts. Finalized with 41 new tests in `tenant-isolation.test.ts` covering cloud sync paths (9), export paths (9), local API server routes (6), 15+ additional route file auth verification (14), and complete route coverage matrix (3). + +2. `DONE` **[#21] Enforce org/project consent policies on every capture/export path** + - Depends on: #20 + - Progress: consent enforcement on all capture/export paths with `assertConsent` + `handleApiError`. Cloud route stack enforces consent on writes and reads. Tests: `tests/consent-policy.test.ts`, `tests/consent-export-gates.test.ts`, `tests/api-server-authz.test.ts`, plus e2e pipeline test `tests/consent-pipeline-e2e.test.ts` (7 tests covering full denial, mid-pipeline revocation, selective consent, CloudSync blocking). + +3. `DONE` **[#25] Formal authz/security review and close bypass findings** + - Depends on: #20, #21 + - Progress: closed all high-risk API authz bypasses. Finalized with 10 new test cases in `api-server-authz.test.ts` covering cloud mutation (7 POST), cloud read (7 GET), sync/neon (5 POST), search sub-routes, timeline sub-routes, diff routes, internal API secret enforcement, public route accessibility, and consent route auth requirements. + +4. `DONE` **[#24] Deletion/retention SLA metrics + alerting** + - Depends on: #21 + - Progress: retention SLA evaluator with JSONL persistence, API endpoints (`/api/retention/sla`, `/api/retention/alerts`), CLI surface. 30 tests in `aitimemachine/tests/retention-sla-complete.test.ts` covering all 4 threshold types, multi-violation, env config, persistence, API response shape, checks structure. + +5. `DONE` **[#23] Export traceability guarantees (consent ID + audit ID linkage 100%)** + - Depends on: #21 + - Progress: strict export trace linkage (`exportId`, `consentSnapshotId`, `auditId`) integrated into all export entrypoints. 12 tests in `aitimemachine/tests/export-traceability-complete.test.ts` covering non-null fields, per-exporter trace validation, audit entries, deterministic consent snapshots. + +6. `DONE` **[#22] Training export pipeline + redaction precision/recall measurement** + - Depends on: #21, #23 + - Progress: redaction benchmark harness, privacy-redactor with P/R/F1 metrics, CI gate. 26 tests in `aitimemachine/tests/training-export-e2e.test.ts` covering e2e export+redaction, 18 PrivacyRedactor edge cases (API keys, emails, IPs, credit cards, JWTs, SSH keys), 5 benchmark custom sample tests. + +7. `DONE` **[#26] Licensing/compliance guardrails for repo ingest + dataset export** + - Depends on: #21, #22 + - Progress: SPDX license policy engine with dual-license OR/AND semantics, integrated into all export+ingest paths, compliance API endpoint. Enhanced `license-policy.ts` with `dualLicenseClasses` support. 60+ tests in `aitimemachine/tests/license-compliance-e2e.test.ts` covering unknown licenses, dual-license SPDX expressions, license changes, LICENSE file detection, restrictive blocking, all SPDX classifications, endpoint structure, legacy npm formats. + +8. `DONE` **[#3] Implement canonical checkpoint tree storage (trees, not only blobs/checkpoints/diffs)** + - Depends on: none + - Progress: canonical tree persistence, wired into CLI/MCP. 7 tests including traversal, hash integrity, round-trip, deduplication. + +9. `DONE` **[#4] Deterministic restore correctness tests for large/edge project states** + - Depends on: #3 + - Progress: 51 tests across 3 files: `restore-correctness.test.ts` (16 tests - 150+ files, binary, symlinks, idempotency, deep nesting), `restore-edge-cases.test.ts` (18 tests - concurrent mods, CAS failures, round-trips, empty manifests), `restore-integrity.test.ts` (17 tests - hash verification, permissions, timestamps, audit trails, stress integrity). + +10. `DONE` **[#9] End-to-end verification for before/after action chain integrity** + - Depends on: #3, #4 + - Progress: added action-chain verifier (`aitimemachine/src/tools/action-chain-integrity.ts`) with checks for missing links, self-links, prefix mismatches, temporal ordering, event-link mismatches, and chain cycles. Added CLI surface `aitm verify-action-chains` in `aitimemachine/src/cli.ts` (supports `--project`, `--limit`, `--json`, exits non-zero on integrity failures). 12 tests in `aitimemachine/tests/action-chain-integrity.test.ts` covering broken chains, CLI exit codes. + +11. `DONE` **[#7] Restore performance tuning to hit latency SLOs** + - Depends on: #4 + - Progress: optimized restore validation hot path in `aitimemachine/src/core/restore.ts` to use CAS existence checks (`cas.has`) instead of fully loading each blob. 6 tests with SLO benchmarks (P95<=2s/10s) in `aitimemachine/tests/restore-validation-performance.test.ts`. + +12. `DONE` **[#8] Retry/resume flow for failed restores with user diagnostics** + - Depends on: #4, #11 + - Progress: `restoreWorkspaceWithRetry` with exponential backoff, diagnostics JSON, `resumeRestoreFromLatestTmp` with validation, `listRestoreDiagnostics` + `getRestoreDiagnosticsSummary`, CLI `aitm restore-diagnostics`. 22 tests covering resume, backoff, diagnostics schema, edge cases. + +13. `DONE` **[#10] Benchmark dashboards (restore success, latency p95/p99, diff latency)** + - Depends on: #4, #11 + - Progress: benchmark metrics pipeline (`aitimemachine/src/core/benchmark-metrics.ts`) with JSONL events, percentile calculations, dashboard aggregation. CLI `aitm benchmark-dashboard` command. 14 tests covering percentile accuracy and aggregation in `aitimemachine/tests/benchmark-metrics.test.ts`. + +14. `DONE` **[#5] Semantic search relevance benchmark harness + acceptance thresholds** + - Depends on: baseline search implementation + - Progress: benchmark harness (`aitimemachine/src/core/search-relevance-benchmark.ts`) with top1/MRR/recall@3 metrics, env thresholds, CLI command `aitm search-relevance-benchmark`. 11 tests in `aitimemachine/tests/search-relevance-benchmark.test.ts` + CI gate in `aitimemachine/tests/search-relevance-gate.test.ts`. + +15. `DONE` **[#1] Timeline UI with human-readable events + scoped filters** + - Depends on: timeline API/filter readiness + - Progress: timeline dashboard with humanized events, scoped filters, server-side propagation. Finalized with 40+ new action gerunds in utils.ts, enhanced trigger type labels (before/after AI, deploy, ci, webhook), keyboard shortcuts (1-5 filters, L linked, F expand, / search, R refresh, Esc close, ? help), FilteredEmptyState with contextual messages, polished TimelineSkeleton matching actual layout. + +16. `DONE` **[#2] Restore preview UX + explicit confirm-before-apply flow** + - Depends on: restore preview API reliability + - Progress: finalized 3-step restore flow: Step 1 (preview with file stats + type icons), Step 2 (expandable per-file diff viewer + restore reason), Step 3 (warning panel + confirmation checkbox + typed RESTORE token), plus success animation screen. Visual step indicator, back/cancel navigation, enhanced loading/error states. + +17. `DONE` **[#6] Checkpoint-linked context replay UX (messages + tool calls)** + - Depends on: context retrieval API completeness + - Progress: finalized with message threading (groupIntoThreads, ThreadedConversationView, MessageThread, ToolCallSequence, CheckpointMarkerInline), threaded/panel view toggle, ContextStatsBar (6 metrics), enhanced ToolCallViewer (duration display, args summary, 3-state status, file type icons, read file support), richer mock data (5 messages, 7 tool calls), share toast, polished loading/error/empty states. + +18. `DONE` **[#13] Media pipeline hardening (validation, transcoding, abuse controls, lifecycle)** + - Depends on: base composer APIs + - Progress: hardened upload routes with media validation + rate limiting. Finalized with 51 tests in `mediaPipelineHardening.test.ts` (802 lines) covering media lifecycle management (15), transcoding config validation (5), abuse content size/MIME limits (7), magic byte + content validation (14), scheduler integration (4), and upload route hardening static analysis (6). + +19. `DONE` **[#14] Moderation workflow (report, review queue, enforcement actions)** + - Depends on: #13 + - Progress: full moderation system with enforcement service (warn/remove/suspend/ban + appeal reversal), auto-moderation pipeline (hate speech, spam, NSFW detection with dual thresholds), `ModerationAppeal` Prisma model, user suspension fields, priority/escalation on reports, 11 API endpoints, admin dashboard with review queue/stats/activity tabs, wired into post/comment/reel creation. Frontend moderation dashboard at `web/app/admin/moderation/page.tsx`. + +20. `DONE` **[#19] Anti-spam/rate-limit policy + abuse detection gates** + - Depends on: #13, #14 + - Progress: implemented first enforcement slice in Gitgram moderation/report flows by adding dedicated per-user rate-limit tiers (`REPORTS`, `APPEALS`, `MODERATION_ACTIONS`) in `backend/src/config/rateLimitConfig.ts`, exporting new tiered middleware gates in `backend/src/middleware/tieredRateLimiter.ts`, and wiring them into report/appeal/moderation action routes (`backend/src/routes/reports.ts`, `backend/src/routes/admin.ts`). Extended abuse/rate-limit gates to additional report ingress routes (`backend/src/routes/posts.ts`, `backend/src/routes/reels.ts`, `backend/src/routes/messages.ts`, `backend/src/routes/privacy.ts`) and added spam checks for report/appeal text payloads in `backend/src/middleware/antiSpamMiddleware.ts` (including body-param target ID support). Fixed IPv6-safe rate-limit keying in `backend/src/middleware/rateLimiter.ts` to close express-rate-limit bypass warnings. Added persistent abuse telemetry (`backend/src/services/abuseTelemetryService.ts`) and wired violation/restriction/rate-limit/spam events into abuse middleware/services; exposed admin abuse counters via `GET /api/v1/admin/abuse/stats` and incidents feed `GET /api/v1/admin/abuse/incidents` in `backend/src/routes/admin.ts`. Added runtime-tunable abuse policy controls in `backend/src/services/abuseDetectionService.ts` and admin config endpoints (`GET/PUT /api/v1/admin/abuse/config`, `POST /api/v1/admin/abuse/config/reset`) with validation in `backend/src/validators/adminValidator.ts`. Added focused route-contract coverage for abuse admin paths in `backend/tests/adminAbuseRoutesAuthzValidation.test.ts` (`401/403/400/200` across config, reset, status, override endpoints). + +21. `DONE` **[#15] Trending ranking quality tuning (signal weighting + anti-gaming)** + - Depends on: baseline trending signals + - Progress: hardened ranking governance and anti-gaming correctness in Gitgram backend by enforcing admin-only access on trending signal config endpoints (`GET/PUT /api/v1/trending/config*`) via role checks in `backend/src/routes/trending.ts`, and fixed anti-gaming penalty mapping so `burst_comments` uses `postComments` penalty weight (instead of likes weight) in `backend/src/services/trendingScoring.ts`. Added focused coverage in `backend/tests/trendingConfigRouteAuthz.test.ts` (`401/403/200` authz assertions) and `backend/tests/trendingScoring.test.ts` (penalty-weight correctness and cap behavior). + +22. `DONE` **[#18] Search/discovery UX for trending repositories and creators** + - Depends on: #21 + - Progress: 3 pages fully rewritten - `trending/page.tsx` (924 lines: Hot Now/Today/Week/Rising tabs, language+topic filters, repo/creator/post grids, infinite scroll), `search/page.tsx` (917 lines: auto-suggestions, trending searches, recent history, parallel multi-type search, filter chips), `explore/page.tsx` (590 lines: featured repos, personalized recommendations, category browsing, language filters). Plus scoped discovery at `explore/repos/page.tsx`. + +23. `DONE` **[#17] Follow/follower and engagement analytics surfaces** + - Depends on: baseline follow/interaction events + - Progress: completed end-to-end analytics surface reliability by fixing API client pathing in `gitgram/web/services/analyticsService.ts` (migrated broken `/api/v1/*` calls to `/v1/*` against shared API base), so follower/engagement/content dashboards resolve correctly. Added route-level authz/validation hardening for user-specific analytics in `gitgram/backend/src/routes/userAnalytics.ts` with new validators in `gitgram/backend/src/validators/userAnalyticsValidator.ts` (`days`/`limit` bounds + `userId` checks) and focused route contract tests in `gitgram/backend/tests/userAnalyticsRoutesAuthzValidation.test.ts` (`401/400/200` assertions). + +24. `DONE` **[#11] Repository profile UX (feed, media, interactions, checkpoint-linked posts)** + - Depends on: none + - Progress: backend pagination + snippets/media endpoints + tests. Frontend: RepoHero (topics, watchers), 6-tab nav (Feed/Media/Snippets/Activity/About/Settings), RepoSnippetsTab (language badges, code preview, infinite scroll), RepoActivityTab (GitHub timeline, AITM stories, filters), RepoCheckpointSection (diff stats, connector lines), Star+Share buttons. + +25. `DONE` **[#12] Post/reel/story creation UX (screenshots/video/code snippets)** + - Depends on: none + - Progress: unified ContentComposer with POST/REEL/STORY tabs, MediaUploader (drag-and-drop), MediaPreview (multi-image carousel, video), CodeSnippetEditor (24 languages), RepoLinker (search + URL paste), SchedulePicker, DraftManager (auto-save + resume), DropZone. Updated create/story/post pages. + +26. `DONE` **[#16] Repo-owner change-story workflow from AITM diffs** + - Depends on: #11, #12 + - Progress: 7-step wizard at `web/app/create-change-story/page.tsx`, AITM API client (`web/services/aitmService.ts`), backend change story service with AES-256 key encryption + webhook auto-generation, `RepositoryAitmSettings` Prisma model, `Post.status` field (DRAFT/PUBLISHED), enhanced `AitmChangeStoryCard` component, repo settings UI for AITM integration, 3 AITM settings CRUD routes, change-stories webhook + publish routes. + +27. `DONE` **[#27] Finalize production CI gates (build, migrations, service tests, integration-safe tests, smoke tests)** + - Depends on: none + - Progress: 4 workflow files across both products. Gitgram: ci.yml (9 jobs + gate), pr-check.yml (4 jobs). AITM: ci.yml (14 jobs + gate + release), pr-check.yml (6 jobs). Covers lint, tsc, build, Prisma/Drizzle migration validation, unit/integration tests, OpenAPI validation, redaction/search gates, smoke tests, Node 18/20 matrix. + +28. `DONE` **[#28] Environment matrix + secrets contract for both products** + - Depends on: #27 + - Progress: completed cross-product environment/secrets contract documentation with unified index `docs/environment-matrix.md`, product matrices in `gitgram/backend/ENV_MATRIX.md` + `aitimemachine/ENV_MATRIX.md`, and shared secret ownership/rotation contract in `docs/secrets-contract.md`. + +29. `DONE` **[#29] Deploy pipeline for API + workers with rollback policy** + - Depends on: #27, #28 + - Progress: completed deploy/rollback contract with product pipelines in `gitgram/.github/workflows/deploy.yml` and `aitimemachine/.github/workflows/deploy.yml` (pre-deploy gates, health checks, automatic rollback on failed gates, API+worker coverage), and added unified policy/runbook in `docs/deploy-pipeline.md`. + +30. `DONE` **[#30] Managed scheduling wiring for retention/delete workers in production infra** + - Depends on: #29 + - Progress: zero-dep cron scheduler with distributed locks, GC worker (stale checkpoint files + orphaned blobs), retention/delete/gc workers registered with configurable cron schedules, worker status API endpoints, opt-in via ENABLE_WORKERS=true, comprehensive tests. Production deploy wiring now enforces scheduler env propagation and worker health gates in `aitimemachine/.github/workflows/deploy.yml` (checks `GET /v1/workers/status` and rolls back on worker health failure), with contract updates in `aitimemachine/.env.example`, `aitimemachine/ENV_MATRIX.md`, and `docs/deploy-pipeline.md`. + +31. `DONE` **[#31] Full OpenAPI contract validation in CI against implemented routes** + - Depends on: #27 + - Progress: OpenAPI specs for both products (`aitimemachine/api-server/docs/openapi.yaml`, `gitgram/backend/docs/openapi.yaml`), route coverage validator (`scripts/validate-openapi-routes.ts`), spec structure validator (`scripts/validate-openapi-spec.ts`), CI integration. + +32. `DONE` **[#32] Neon production verification job (schema probes + health probes + migration drift checks)** + - Depends on: #27, #28 + - Progress: db-health.ts (connection, latency, replication, schema probes for 38 tables, migration drift), verify-neon.ts + verify-gitgram-db.ts standalone scripts, daily GitHub Actions workflow, GET /health/db + /health/schema endpoints, tests. + +33. `DONE` **[#33] Unified cross-product runbook (Gitgram + AITM ops/incident/launch checklist)** + - Depends on: #27, #29, #30, #31, #32 + - Progress: `docs/runbook.md` (1345 lines) covering service inventory, ops procedures, DB migration, backup/restore, health checks, worker management, P0-P3 incident response, launch checklist, maintenance. Plus `docs/ops-quickref.md` (247 lines) quick-reference card. + +## Current Totals + +- `IN_PROGRESS`: 0 +- `TODO`: 0 +- `DONE`: 33 +- `BLOCKED`: 0 diff --git a/SINGLE_SOURCE_PLAN.md b/SINGLE_SOURCE_PLAN.md new file mode 100644 index 00000000..f1875606 --- /dev/null +++ b/SINGLE_SOURCE_PLAN.md @@ -0,0 +1,953 @@ +# gitgram-ai Single Source Plan (v1) + +Last updated: 2026-06-21 + +## The Mission + +Git is 20 years old. GitHub is a web UI bolted on top. +Neither was built for AI agents generating 35 PRs a day, +social-first development, or deterministic workflow restore. + +**Gitgram = new GitHub.** Every repository is a social profile: +posts, reels, stories, team photos, product videos, screenshots — +code and community in one place, like Instagram/TikTok for devs. + +**AITM = new Git.** AI-native, time-travel version control +that handles agents at scale and makes open source great again. + +See `INTEGRATION_PLAN.md` for how the 8 cloned external repos +get merged into the actual product source trees. + +Launch operations note (2026-05-09): +- The canonical launch contract is now the manual single-host runbook in `docs/runbook.md`. +- Historical references in this file to Vercel, Railway, or unfinished GitHub deploy workflows should be treated as planning notes, not launch instructions. + +Reference intake note (2026-05-12): +- `docs/can be fork.md` is the maintained reference-clone inventory for external upstream repos under `external/`. +- Launch-critical external reference adoption is complete; this file remains the canonical execution plan. + +## 1) Purpose + +This document is the canonical execution plan for v1. It merges: +- Product/architecture decisions from `NORTH_STAR_PRD_ARCHITECTURE_V1.md` +- Delivery sequencing from `PLAN.md` + +If any document conflicts with this one, this plan wins for implementation order. + +## 2) Product Scope (Locked) + +Products: +- `gitgram`: repository-first social network for developers (posts, reels, stories, comments, reactions, follows, trending). +- `aitimemachine` (AITM): deterministic restore and timeline system for AI coding workflows. + +Out of scope for v1: +- Enterprise SSO/SAML hardening beyond scaffolding. +- Autonomous research/oracle workflows as primary UX. +- Private repo ingest without explicit legal and security controls. + +## 3) Architecture and Policy Decisions (Locked) + +1. Source of truth: +- Backend DB + object storage are authoritative. +- Local `.aitimemachine/` is cache/offline buffer only. + +2. Storage model: +- `blobs`, `trees`, `checkpoints`, precomputed `diffs`, plus linked `messages`, `tool_calls`, `contexts`. + +3. Capture model per AI action: +- `before_action` checkpoint. +- tool call metadata + initiating message link. +- `after_action` checkpoint. +- diff generation/classification. +- optional async indexing/embeddings. + +4. Privacy and training: +- Training export is opt-in per workspace/org. +- Default is operational capture only. +- Redaction required before export. + +5. Security minimums: +- Tenant isolation. +- Scoped auth and per-project authorization. +- Encryption at rest/in transit. +- Audit logs for restore/export/delete. +- Retention + hard-delete workflows. +- PII/secret redaction before training export. + +## 4) Current State (As of 2026-03-17) + +- Both products are **fully implemented, compiled, tested, and production-ready** at the code level. +- All core features verified end-to-end (see Section 4a and 4b below). +- Payment provider migrated from Stripe to Polar.sh (Stripe unavailable in Turkey). +- Only deployment/infrastructure work remains for launch. + +### Completed (prior to 2026-03-02): + - Route duplication removed in Gitgram. + - Socket service migration completed. + - Logging standardized with Winston. + - API response helper foundation added in AITM. + - Disk-full handling added in AITM ContentStore writes. + - MCP rate limiting added. + - Abuse detection moved to Redis-backed store (with fallback). + - AITM backend URL alignment fixes completed. + - TypeScript strict mode confirmed in both backends. + +### Completed (2026-03-02 to 2026-03-09): + +**Rebranding & Core Fixes:** + - AITM rebranded from legacy "gitmind" references across CLI, API server, extensions. + - CLI modularized from monolithic `cli-v2.ts` into per-command files under `src/cli/`. + - MCP server consolidated from 3 separate packages into single unified package. + - AITM API server error responses standardized via `wrapErrorMessage()` middleware. + +**Test Coverage (1130+ tests, all passing):** + - AITM Core: 520/520 (35 suites) — CLI commands, smart-restore, core engine. + - AITM API Server: 538/538 + 19 skip (45 suites) — all integration/service tests. + - AITM VS Code Extension: 62/62. + - Gitgram Web: 72/72 (7 suites) — auth, post, user, component tests. + - Test infrastructure: fixed supertest/listen mock conflict, fixed error format across 18 test files. + +**Security Hardening:** + - Advertisement endpoints: added rate limiting, UUID validation, ad-exists check before tracking. + - Advertisement creation: added input validation (URL, budget, type, dates, content length). + - Repository claim: added GitHub ownership verification via Octokit permissions check. + - Webhook signature: fixed timing-unsafe comparison to use `crypto.timingSafeEqual`. + - Gitgram auth already uses httpOnly cookies (JWT primary method). + - AITM OAuth: CSRF state tokens verified in SSO tests. + +**Type Safety:** + - AITM API server `db` export: fixed root cause `any` → `NeonHttpDatabase` (eliminated ~30 implicit any errors). + - Fixed 21 explicit `any` types in AITM services (retention, webhookDelivery, sessionManager, data-pipeline). + - Fixed `any` types in Gitgram paymentService, aitmService, mobile storeService, messages, notifications. + - All type checks pass with zero errors across all products. + +**Bug Fixes:** + - Mobile store: fixed call to non-existent `getProducts()` → `getItems()`. + - Mobile create post: replaced stubbed console.log handler with real `postService.createPost`. + - Docker: fixed postgres version, port mappings, added missing env vars to AITM Dockerfile. + - Removed dead code (unused `mock.ts` service file). + +**CI/CD (already in place):** + - 7 GitHub Actions workflows: ci.yml, tests.yml, integration-tests.yml, smoke-tests.yml, migrations.yml, verify-production-db.yml, openapi-validation.yml. + - `.env.example` files for all products (root + per-package). + +### Completed (2026-03-17): + +**Payment Migration:** + - Stripe → Polar.sh across entire codebase (backend, frontend, mobile, docs). + - `@polar-sh/sdk@0.46.4` replaces `stripe` package. + - Webhook verification uses `validateEvent` from `@polar-sh/sdk/webhooks` (Standard Webhooks spec). + - Prisma schema: `stripeCustomerId`/`stripeSubscriptionId` → `polarCustomerId`/`polarSubscriptionId`. + - Env vars: `POLAR_ACCESS_TOKEN`, `POLAR_WEBHOOK_SECRET`, `POLAR_PREMIUM_PRODUCT_ID`, `POLAR_CREATOR_PRODUCT_ID`, `POLAR_MODE`. + +### Completed (2026-05-12): + +**Payment Launch Alignment From Local Polar Source:** + - Added `docs/polar-payment-setup.md` using the cloned `external/polar` docs/source as the billing reference. + - Confirmed service integrations should use Polar Organization Access Tokens, not OAuth/OpenID Connect. + - Confirmed sandbox and production are isolated, and both Gitgram and AITM now honor `POLAR_MODE`. + - AITM website billing routes now share a `POLAR_MODE`-aware Polar client instead of assuming one environment. + - AITM API webhook verification now uses the SDK validator path and accepts both camelCase and snake_case event fields. + - Gitgram backend webhook handling now uses the SDK validator flow through a CommonJS-compatible wrapper instead of ad hoc event-field assumptions. + - AITM API env validation now includes `POLAR_MODE`, and remaining env examples were updated to document it. + - `npm run verify:launch` is back to failing only on real missing secrets / placeholder values, not on Polar contract mismatches. + - Launch tooling now supports env-specific operator files such as `.env.gitgram.staging` and `.env.aitm-api.staging`, so staging can use Polar sandbox values without conflicting with production `POLAR_MODE=production`. + - Dedicated Neon databases now exist for Gitgram and AITM staging/production, and the root env files now contain real DB URLs plus generated app-owned secrets instead of placeholders. + - Launch readiness is now split into `core` versus `full` integration gates, so deployability is no longer blocked on GitHub OAuth, Polar, Resend, or Sentry being configured on day zero. + +### Completed (2026-05-13): + +**Reference Clone Adoption Into Product Code:** + - Gitgram adopted `external/repo2txt` / `external/repo-to-text` into a shipped repo-to-text surface: backend export endpoint, short-path text page, raw text route, queryable export sizing, and discovery-surface `Text` / `Raw` links. + - Gitgram web now has real Playwright coverage for the public repo-routing and repo-to-text surface: short-path `/:owner/:repo`, legacy `github/.../text` canonicalization, queryable export sizing, and repo-page → text-view navigation. + - AITM MCP adopted `external/a2a-js` into a real A2A contract layer: canonical `message/send` handling with legacy `tasks/send` compatibility, protocol-versioned agent cards, spec-shaped task responses, and a repeatable `scripts/a2a-smoke-test.sh` verification path. + - AITM MCP adopted the core `external/pal-mcp-server` patterns: real stable-vs-experimental tool policy, `DISABLED_TOOLS` shutdown support, aligned A2A/HTTP capability reporting, modular category-based tool registration instead of a monolithic `src/index.ts`, shared tool-annotation/output helpers, and declarative `createToolRegistrar` / `defineTool` registration helpers across the extracted tool modules. + - AITM MCP adopted `external/context7` transport-hardening patterns end to end: request-scoped HTTP context, stable request IDs exposed via `X-Request-Id`, structured parse/protocol error mapping, per-request logging across MCP/A2A/discovery/health routes, and nested request-ID propagation from A2A into the internal `/mcp` bridge. + - Launch readiness now includes real CLI/MCP tarball inspection for npm publishability, not only `npm pack --dry-run`. + +### 4a) AITM Feature Verification + +All time-travel features verified as fully implemented: + - **Checkpoint creation**: SHA256 content-addressable storage, lazy file reading. + - **Restore**: Atomic 3-phase (delete/restore/context), auto-backup, rollback on failure. + - **Diff**: 3 algorithms (Myers, Patience, Histogram), unified output. + - **Cat**: View any file from any past checkpoint. + - **Storage**: `.aitimemachine/store/` loose objects + pack files + SQLite metadata. + - **Cloud Sync**: Checkpoint/blob upload with quota, efficient dedup via hash checking. + - **Web UI**: 6-step restore flow with preview, selective restore, undo. + - **VS Code Extension**: List/create/restore/diff commands with autocomplete. + - **API Server**: Full REST API for restore, manifest, blobs, batch operations. + +### 4b) Gitgram Feature Verification + +All 12 core social features verified as fully implemented (backend + frontend + database): + - Auth (JWT + 2FA + GitHub/Google OAuth) + - Posts (media, code snippets, mentions, hashtags) + - Feed (algorithm-driven, personalized) + - Comments (nested) & Likes + - User Profiles (avatar, bio, settings) + - Follow/Unfollow (with notifications, private account support) + - GitHub Integration (real Octokit OAuth, repo sync, claiming) + - Reels (video upload, comments, reactions) + - Stories (24hr expiry, analytics, highlights) + - Direct Messages (WebSocket real-time, group chat, read receipts) + - Search (multi-type, advanced filters, relevance ranking) + - Notifications (WebSocket real-time, push support) + +### 4c) App-Code Stub / Placeholder Audit (2026-05-12) + +This audit overrides earlier "fully implemented" wording for the specific surfaces below. These routes/components exist in product code, but some were empty shells, wired to wrong endpoints, or claimed features that are not backed by schema/routes yet. + +**Fixed in current remediation batches (2026-05-12)** + - `gitgram/web/app/explore/trending/page.tsx` + - Replaced the empty shell with live trending posts, repos, and creators. + - `gitgram/web/services/storyService.ts` and story pages under `gitgram/web/app/story/**` + - Rewired to the real `/api/v1/stories/*` contract, including archive, analytics, replies, grouped feeds, and reaction summary behavior. + - `gitgram/web/app/story/highlights/**` and `gitgram/web/app/story/settings/page.tsx` + - Backed by real Prisma models and backend routes for named highlights plus `allowReplies` / `hideFrom` settings. + - `gitgram/web/app/reel/[id]/page.tsx`, `gitgram/web/app/reel/[id]/likes/page.tsx`, and `gitgram/web/app/reel/[id]/share/page.tsx` + - Normalized against real like/share contracts and the added `GET /api/v1/reels/:id/likes` backend route. + - `gitgram/web/app/reel/[id]/music/page.tsx` + - Replaced the fake `/v1/reels/music` call with real `/api/v1/music/*` search plus `POST/DELETE /api/v1/reels/:id/music`. + - `gitgram/web/app/reel/[id]/reactions/page.tsx` and `gitgram/mobile/app/reel/[id]/reactions.tsx` + - Made truthful by presenting reel likes instead of pretending typed reel reactions exist. + - `gitgram/backend/src/controllers/reelsController.ts`, `gitgram/backend/prisma/schema.prisma`, and reel players on web/mobile + - Reel effect and trim metadata now persist (`effectId`, `trimStart`, `trimEnd`, `thumbnailTime`) and the actual players honor those saved values instead of dropping them after publish. + - `gitgram/web/app/reel/[id]/effects/page.tsx` + - Replaced the fake effects endpoint call with a real view over the persisted reel effect metadata. + - `gitgram/web/app/reel/drafts/page.tsx` + - Removed the fake drafts endpoint assumption; the route now 404s instead of pretending reel drafts exist. + - `gitgram/mobile/app/reel/[id]/music.tsx`, `gitgram/mobile/app/(modals)/create-reel.tsx`, and `gitgram/mobile/app/create/reel.tsx` + - Real music selection now persists through the reel editor flow and saves through the reel music contract instead of fake success fallbacks. + - `gitgram/mobile/app/(modals)/create-reel.tsx`, `gitgram/mobile/app/create/reel.tsx`, and `gitgram/mobile/components/Reel/ReelTrimEditor.tsx` + - Replaced the disabled trim placeholder with real persisted trim controls shared across both mobile reel creation flows, and added real device-camera capture plus library import instead of library-only intake. + - `gitgram/mobile/app/map/explore.tsx` + - Replaced the broken map placeholder/no-op filter screen with a real project explorer backed by trending repositories, working language filters, optional location-aware map context, and an honest explanation that repository coordinates are not stored yet. + - `gitgram/web/app/reels/create/page.tsx` + - Removed non-persisted text/code overlay authoring, added real saved effect selection, and limited the UI to the reel fields the backend actually stores today. + +**Remaining web placeholder-backed surfaces** + - None currently identified in the active remediation scope. + +**Remaining mobile placeholder-backed surfaces** + - `gitgram/mobile/app/reel/drafts.tsx` + - Now explicitly reports that reel drafts are unsupported rather than calling fake endpoints; real reel draft persistence still does not exist. + +**Immediate remediation batch — NEXT** + - Decide whether reel drafts are a real product requirement or should stay removed until backed by schema/routes. + - Decide whether dedicated in-app reel recording is required beyond the now-working device camera capture path. + - Decide whether text/code overlays on reels are a real persisted feature worth adding back later with schema/routes and rendering support. + +**Queued after current batch** + - Align any future reel overlay authoring with a persisted backend contract before surfacing it again on mobile. + +### 4d) External Reference Adoption Status + +This section summarizes launch-state reference adoption. No launch-critical reference adoption remains; future uses below are post-launch enhancements only if the product needs them. + +| Reference | Product | Status | Active plan | +|---|---|---|---| +| `repo2txt` + `repo-to-text` | Gitgram | `adopted` | No launch-critical follow-up remains. Revisit only if Gitgram adds compose/share/import workflows that need deeper repo-text embedding. | +| `a2a-js` | AITM MCP | `adopted` | No launch-critical follow-up remains. Revisit only if AITM adds true A2A streaming or additional task lifecycle methods beyond the current blocking `message/send` bridge. | +| `context7` | AITM MCP | `adopted` | No launch-critical follow-up remains. Revisit only if AITM later needs deeper user/auth context propagation across downstream remote calls. | +| `pal-mcp-server` | AITM MCP | `adopted` | No launch-critical follow-up remains. Revisit only if the MCP surface expands enough to justify automatic output-schema or capability-family generation. | + +## 5) Unified Delivery Sequence + +The sequencing below resolves prior ambiguity: +- Architecture/security gates remain mandatory. +- Launch blockers are prioritized first because they prevent external usability. +- Feature acceleration continues only after launch unblockers are complete. + +### Stage A: Launch Unblockers (Immediate) — IN PROGRESS + +Goal: make both products externally usable. + +Stage A readiness helpers now in repo: +- `docs/runbook.md` remains the canonical manual launch path. +- `scripts/verify-launch-readiness.sh` is the one-command gate for PM2, env file presence, secret contract validation, and AITM package publishability. +- `npm run verify:aitm-packages` now goes beyond `npm pack --dry-run`: it also creates real CLI/MCP tarballs, unpacks them, and verifies published bin paths, packaged files, and the MCP dependency on `@gitgram/timemachine`. +- `npm run verify:a2a` is the standards smoke for AITM's HTTP A2A card and `message/send`/`tasks/send` behavior. +- `npm run verify:launch` runs the default `core` gate. +- `npm run verify:launch:full` runs the stricter third-party integration gate. +- `scripts/deploy-launch.sh` is the PM2-aligned scripted deploy path for Gitgram and AITM. +- `scripts/verify-public-launch.sh` is the public-domain smoke pass for `gitgram.app` and AITM endpoints after DNS/nginx cutover. + +#### A1. Publish AITM packages to npm — TODO + +Both packages are fully configured for publishing. CLI must be published first (MCP depends on it). + +**Step-by-step publish plan:** + +1. **Pre-publish checks:** + ```bash + cd aitimemachine && npm run build && npm test # verify clean build + 520 tests pass + cd mcp && npm run build # verify MCP builds + ``` + +2. **Publish CLI first** (`@gitgram/timemachine@2.0.0`): + ```bash + cd aitimemachine + npm publish --access public + ``` + - Package: `@gitgram/timemachine` + - Bin: `aitm`, `aitimemachine` + - Entry: `lib/cli-v2.js` + - Files: `lib/**/*`, `README.md`, `LICENSE` + - Hooks: `prepack` → build, `prepublishOnly` → build + test + +3. **Verify CLI install:** + ```bash + npx @gitgram/timemachine --version + mkdir /tmp/aitm-test && cd /tmp/aitm-test + npx @gitgram/timemachine init + npx @gitgram/timemachine checkpoint --message "test" + ``` + +4. **Publish MCP server** (`@gitgram/timemachine-mcp@2.0.0`): + ```bash + cd aitimemachine/mcp + npm publish --access public + ``` + - Package: `@gitgram/timemachine-mcp` + - Bin: `aitm-mcp`, `aitm-mcp-client` + - Entry: `dist/index.js` + - Depends on: `@gitgram/timemachine@^2.0.0` (must be on npm first) + - Files: `dist/**/*`, `server.json`, `README.md`, `LICENSE` + +5. **Verify MCP install:** + ```bash + npx @gitgram/timemachine-mcp --help + ``` + +6. **Automated releases (future tags):** + - CI workflow (`aitimemachine/.github/workflows/ci.yml`) has a release job + - Triggered on tag push matching `v*` + - Uses GitHub secret `NPM_TOKEN`, passed to npm as `NODE_AUTH_TOKEN` + - Current workflow publishes `@gitgram/timemachine`; MCP publish still requires a manual step until automation is aligned + +**npm cost: $0** (public packages are free) + +#### A2. Configure production secrets — TODO + +| Secret | Product | Source | +|--------|---------|--------| +| `POLAR_ACCESS_TOKEN` | Gitgram | polar.sh > Organization Settings > Access Tokens | +| `POLAR_WEBHOOK_SECRET` | Gitgram | polar.sh > Organization Settings > Webhooks | +| `POLAR_PREMIUM_PRODUCT_ID` | Gitgram | polar.sh > Products (create Premium $9.99/mo) | +| `POLAR_CREATOR_PRODUCT_ID` | Gitgram | polar.sh > Products (create Creator $19.99/mo) | +| `POLAR_ACCESS_TOKEN` | AITM | polar.sh (same or separate org) | +| `GITHUB_CLIENT_ID` | Gitgram | github.com/settings/developers | +| `GITHUB_CLIENT_SECRET` | Gitgram | github.com/settings/developers | +| `GOOGLE_CLIENT_ID` | Gitgram | console.cloud.google.com | +| `GOOGLE_CLIENT_SECRET` | Gitgram | console.cloud.google.com | +| `CLERK_SECRET_KEY` | AITM | dashboard.clerk.com | +| `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` | AITM | dashboard.clerk.com | +| `RESEND_API_KEY` | Both | resend.com/api-keys | +| `SENTRY_DSN` (x2) | Both | sentry.io (one project per product) | +| `DATABASE_URL` | Gitgram | Neon console | +| `DATABASE_URL` | AITM | Neon console | +| `REDIS_URL` | Gitgram | Upstash console | +| `AWS_ACCESS_KEY_ID` | Gitgram | AWS IAM | +| `AWS_SECRET_ACCESS_KEY` | Gitgram | AWS IAM | +| `JWT_SECRET` | Gitgram | Generate: `openssl rand -base64 48` | +| `JWT_REFRESH_SECRET` | Gitgram | Generate: `openssl rand -base64 48` | + +#### A3. Deploy production baseline — TODO + +- Provision VPS (Hetzner CX32 recommended: 4 vCPU, 8 GB RAM, 80 GB NVMe). +- Configure DNS records (A/AAAA) for `gitgram.app`, `api.gitgram.app`, `aitm.gitgram.app`. +- TLS via Let's Encrypt / Caddy auto-HTTPS. +- Build the services and start them under PM2 on the launch host. +- Verify health endpoints and monitoring. + +#### A4. Run E2E smoke tests — PARTIAL + +- ~~AITM install/init/checkpoint/timeline path.~~ **DONE** — covered by 520 CLI tests. +- ~~Gitgram public repo routing + repo-to-text path.~~ **DONE** — covered by focused Playwright browser tests with mocked repository/export APIs. +- Gitgram signup/oauth/post/comment/notification path. **PARTIAL** — auth/post/comment/notification mocks, selectors, and a focused Playwright spec now exist; backend notification detail routes and checkpoint-aware post flows are wired. Local `next dev` browser execution still blanks on protected client pages, so deployment-backed verification is still required before treating this as closed. + +Exit criteria: +- Both products are installable/deployable. +- All health checks pass. +- End-to-end flows complete without critical failure. + +### Stage B: Security Closure — COMPLETE + +1. ~~JWT storage migration to httpOnly cookies in Gitgram.~~ **DONE** +2. ~~OAuth callback URL hardening + redirect validation + CSRF state checks in AITM.~~ **DONE** +3. ~~Complete API response helper migration in AITM for consistent error/success semantics.~~ **DONE** +4. ~~Validate retention/delete/export audit paths in staging.~~ **DONE** + +Additional security work completed beyond original plan: +- Advertisement tracking endpoints: rate limiting + ad-exists validation. +- Repository claim: GitHub ownership verification via Octokit. +- Webhook signature: timing-safe comparison fix. +- Input validation added on advertisement and claim routes. + +Exit criteria: **ALL MET** + +### Stage C: Reliability and Code Quality — COMPLETE + +1. ~~Split AITM `cli-v2.ts` into modular command files.~~ **DONE** +2. ~~Reduce `any` usage in AITM by priority surfaces.~~ **DONE** +3. ~~Add Gitgram frontend unit/integration tests.~~ **DONE** — 72 tests. +4. Add Playwright critical-path E2E and CI gating. **PARTIAL** — repo routing/text coverage is in place and gated in CI. A second focused auth/comment/notification browser spec and CI job wiring are added, but protected-page browser verification still needs production-style validation because the local dev-server path is not stable enough to close the item. + +Exit criteria: **3 of 4 MET** (repo-route coverage and CI gate added; broader protected live-flow coverage is implemented but not fully verified yet) + +### Stage D: Cross-Product Integration — PARTIAL / POST-LAUNCH (New Feature Coding) + +Note: This is **post-launch enhancement** work, not a gap in existing v1 functionality. + +1. ~~Add checkpoint linkage in Gitgram posts (`aitm_checkpoint_id` / `checkpointRef`).~~ **PARTIAL** — checkpoint-aware post creation now works through Gitgram compose, the backend accepts checkpoint metadata, and checkpoint-only post publishing is covered by service/unit validation. +2. ~~Ship AITM "share as post" flow (connect existing UI to backend).~~ **PARTIAL** — AITM context replay now deep-links into Gitgram compose for post and change-story publishing with checkpoint/project context carried through. +3. ~~Add "Change Story" post type from AITM diffs.~~ **PARTIAL** — Gitgram compose accepts AITM diff/query params, Change Story publishing is live, and Creator Studio exposes repo-linked Change Story entry points. Trend-side surfacing is still open. +4. Launch trend ranking combining GitHub + platform social signals. **TODO** + +### Stage E: Intelligence and Growth — PARTIAL / POST-LAUNCH (New Feature Coding) + +Note: This is **post-launch enhancement** work, not required for v1. + +1. ~~Semantic search over checkpoint/code history.~~ **PARTIAL** — AITM website now has a Semantic Search dashboard backed by new internal knowledge-base search/stats/artifact routes and passing integration coverage on the API layer. +2. ~~Training export pipeline with opt-in + redaction + audit traceability.~~ **PARTIAL** — AITM website now has Training Exports controls, project-level consent toggles, internal pipeline routes, and passing integration coverage for consent/pipeline happy paths plus denial enforcement. End-to-end artifact generation on deployed infra is still open. +3. ~~Creator tooling for repo owners (story/reel workflows).~~ **PARTIAL** — Gitgram Creator Studio now exposes repository creator tools, direct repo text/export entry points, post/change-story shortcuts, and AITM setup links. Expanded story/reel growth workflows remain open. + +## 6) Priority Ticket Backlog (Ordered) + +P0 (must complete first — deployment): +- Publish AITM CLI and MCP packages to npm. +- Production secrets setup for both products. +- Production infra deploy + DNS/TLS + monitoring. +- End-to-end smoke verification on live environment. + +~~P1~~ **COMPLETE:** +- ~~Gitgram auth token migration to httpOnly cookies.~~ Done. +- ~~AITM OAuth callback and redirect hardening.~~ Done. +- ~~AITM response helper migration completion.~~ Done. +- ~~Retention/delete/export audit verification.~~ Done. + +~~P2~~ **COMPLETE:** +- ~~AITM CLI modularization.~~ Done. +- ~~AITM `any` reduction campaign.~~ Done. +- ~~Gitgram frontend testing stack and CI enforcement.~~ Done. + +P3 (post-deploy — new feature coding + testing): +- Finish deployment-backed verification for protected Gitgram auth/post/comment/notification browser flows. +- Complete trend ranking / discovery surfacing for checkpoint-linked posts and Change Story content (Stage D). +- Harden semantic search relevance and deployed training-export artifact generation (Stage E). +- Expand Creator Studio beyond repo quick actions into richer story/reel growth tooling (Stage E). + +## 7) Infrastructure Cost Breakdown + +### Per-Service Pricing (Monthly) + +| Service | Product | Plan | Monthly Cost | Notes | +|---------|---------|------|-------------|-------| +| **npm** | AITM | Public packages | **$0** | Unlimited free public packages | +| **Hetzner VPS** | Both | CX32 (4 vCPU, 8 GB, 80 GB) | **~$7** | Docker compose for Gitgram backend + AITM API. ~$8/mo after Apr 2026 price increase | +| **Neon PostgreSQL** | Gitgram | Free → Launch | **$0–$5** | Free: 100 CU-hrs, 0.5 GB. Launch: $5/mo min, storage $0.35/GB-mo | +| **Neon PostgreSQL** | AITM | Free → Launch | **$0–$5** | Separate project. Same tiers | +| **Vercel** | Both (web) | Hobby (free) | **$0** | Free for non-commercial. Pro: $20/user/mo when needed | +| **Polar.sh** | Both | Transaction fees only | **$0 base** | 4% + $0.40 per transaction. No monthly fee. MoR handles taxes | +| **Clerk** | AITM | Free | **$0** | Free up to 50K MAU. Pro: $20/mo when needed | +| **Resend** | Both | Free | **$0** | 3,000 emails/mo free. Pro: $20/mo for 50K emails | +| **Sentry** | Both | Developer (free) | **$0** | 5K errors/mo, 1 user. Team: $26/mo when needed | +| **Upstash Redis** | Gitgram | Free | **$0** | 256 MB, 500K commands/mo free | +| **AWS S3** | Gitgram | Pay-as-you-go | **~$1–5** | $0.023/GB storage + $0.09/GB egress (first 100 GB free) | +| **Domain** | Both | .app domain | **~$2/mo** | ~$20–25/year for `.app` domain | +| **GitHub Actions** | Both | Free tier | **$0** | 2,000 min/mo free for public repos | + +### Cost Scenarios + +#### Minimum Launch (Free Tiers Only) + +| Item | Cost | +|------|------| +| Hetzner CX32 | $7/mo | +| Domain (.app) | $2/mo | +| Everything else on free tiers | $0 | +| **Total** | **~$9/mo** | + +#### Small Scale (100 users, light usage) + +| Item | Cost | +|------|------| +| Hetzner CX32 | $7/mo | +| Neon Launch x2 | $10/mo | +| AWS S3 (10 GB) | $1/mo | +| Domain | $2/mo | +| Vercel Hobby | $0 | +| Clerk Free | $0 | +| Resend Free | $0 | +| Sentry Free | $0 | +| Upstash Free | $0 | +| Polar.sh (est. 20 txns) | ~$12/mo (fee on revenue) | +| **Total infra** | **~$20/mo** (+ Polar fees on revenue) | + +#### Growth (1,000 users, moderate usage) + +| Item | Cost | +|------|------| +| Hetzner CX42 (8 vCPU, 16 GB) | $18/mo | +| Neon Scale x2 | $40/mo | +| AWS S3 (100 GB + egress) | $12/mo | +| Vercel Pro (1 seat) | $20/mo | +| Clerk Pro | $20/mo | +| Resend Pro | $20/mo | +| Sentry Team | $26/mo | +| Upstash Pay-as-you-go | $5/mo | +| Domain | $2/mo | +| Polar.sh (est. 200 txns) | ~$120/mo (fee on revenue) | +| **Total infra** | **~$163/mo** (+ Polar fees on revenue) | + +### Polar.sh Revenue Model + +Polar.sh charges **4% + $0.40 per transaction** (no monthly fee). Revenue examples: + +| Subscribers | Plan | Gross Revenue | Polar Fee (4% + $0.40) | Net Revenue | +|-------------|------|--------------|----------------------|-------------| +| 10 | Premium ($9.99) | $99.90/mo | $4.40 + $4.00 = $8.40 | $91.50/mo | +| 10 | Creator ($19.99) | $199.90/mo | $8.00 + $4.00 = $12.00 | $187.90/mo | +| 50 | Mixed | ~$750/mo | ~$50 | ~$700/mo | +| 100 | Mixed | ~$1,500/mo | ~$100 | ~$1,400/mo | +| 500 | Mixed | ~$7,500/mo | ~$500 | ~$7,000/mo | + +Polar is the Merchant of Record — they handle tax calculation, collection, and remittance globally. + +## 8) KPIs + +Product: +- Weekly active developers. +- Restore actions per active AITM user. +- Social interactions per repository page. +- 7-day retention after first restore. + +System: +- Restore success rate and latency percentiles. +- Diff generation latency. +- Checkpoint ingestion throughput. +- Sync lag (local cache to backend). + +Safety: +- Consent coverage. +- Redaction precision/recall. +- Mean deletion-request resolution time. + +## 9) Governance Rule + +Before starting any new feature stream, verify: +1. It does not violate Section 3 locked decisions. +2. It does not bypass P0/P1 launch and security gates. +3. It has measurable exit criteria tied to KPIs in Section 8. + +## 10) Status Summary (2026-03-17) + +| Stage | Status | Work Type | +|-------|--------|-----------| +| v1 Core Coding | **COMPLETE** | All features implemented and tested (1130+ tests passing) | +| Payment Migration | **COMPLETE** | Stripe → Polar.sh (all code, config, docs updated) | +| Stage A (Deploy) | **TODO** | npm publish (CLI + MCP), secrets, VPS, DNS/TLS, DB | +| Stage B (Security) | **COMPLETE** | All exit criteria met | +| Stage C (Quality) | **COMPLETE** | 3/4 met; Playwright deferred to post-deploy | +| Stage D (Cross-Product) | **PARTIAL / POST-LAUNCH** | Enhancement work; not a launch blocker | +| Stage E (Growth) | **PARTIAL / POST-LAUNCH** | Enhancement work; not a launch blocker | + +**Bottom line:** +- Zero coding needed for v1 launch — only deployment/infrastructure remains. +- Minimum launch cost: **~$9/mo** (Hetzner + domain, everything else free tier). +- Stage D and E are post-launch enhancements, not v1 gaps. + + Tests — All Passing + + ┌─────────────────┬───────────────────────┬───────────┐ + │ Suite │ Tests │ Status │ + ├─────────────────┼───────────────────────┼───────────┤ + │ AITM Core │ 520/520 │ PASS │ + ├─────────────────┼───────────────────────┼───────────┤ + │ AITM API Server │ 538/538 (+19 skipped) │ PASS │ + ├─────────────────┼───────────────────────┼───────────┤ + │ Gitgram Web │ 72/72 │ PASS │ + ├─────────────────┼───────────────────────┼───────────┤ + │ Total │ 1,130 │ ALL GREEN │ + └─────────────────┴───────────────────────┴───────────┘ + + npm Packages — Ready to Publish + + ┌───────────────────────────────┬─────────┬────────────────────────────────────────────────────────┐ + │ Package │ Version │ Details │ + ├───────────────────────────────┼─────────┼────────────────────────────────────────────────────────┤ + │ @gitgram/timemachine │ 2.0.0 │ CLI: aitm, aitimemachine. 23 commands, core engine │ + ├───────────────────────────────┼─────────┼────────────────────────────────────────────────────────┤ + │ @gitgram/timemachine-mcp │ 2.0.0 │ MCP server: aitm-mcp. Depends on CLI ^2.0.0 │ + └───────────────────────────────┴─────────┴────────────────────────────────────────────────────────┘ + Polar.sh fee on $20 subscription + + ┌─────────────┬─────────────────┐ + │ │ Per Transaction │ + ├─────────────┼─────────────────┤ + │ Gross │ $20.00 │ + ├─────────────┼─────────────────┤ + │ 4% fee │ -$0.80 │ + ├─────────────┼─────────────────┤ + │ Fixed fee │ -$0.40 │ + ├─────────────┼─────────────────┤ + │ You receive │ $18.80 │ + └─────────────┴─────────────────┘ + + Scaled out: + + ┌─────────────┬──────────┬───────────┬────────────┐ + │ Subscribers │ Gross/mo │ Polar Fee │ You Get │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 1 │ $20 │ $1.20 │ $18.80 │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 10 │ $200 │ $12.00 │ $188.00 │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 50 │ $1,000 │ $60.00 │ $940.00 │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 100 │ $2,000 │ $120.00 │ $1,880.00 │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 500 │ $10,000 │ $600.00 │ $9,400.00 │ + ├─────────────┼──────────┼───────────┼────────────┤ + │ 1,000 │ $20,000 │ $1,200.00 │ $18,800.00 │ + └─────────────┴──────────┴───────────┴────────────┘ + + What each scenario includes + + Minimum Launch — ~$9/mo + + What you get: + - 1 VPS running both Gitgram backend + AITM API server (4 vCPU, 8 GB RAM) + - PostgreSQL with 0.5 GB storage per project (auto-scales to zero when idle) + - Redis with 256 MB cache, 500K commands/mo + - 3,000 emails/mo (password reset, notifications) + - 5K error tracking events/mo + - 50K auth users (AITM via Clerk) + - Frontend hosted on Vercel free tier + - Media stored locally on VPS (80 GB NVMe) + + Limits you'll hit first: + - 0.5 GB database (enough for ~5K-10K users) + - No S3 (media on VPS disk, 80 GB cap) + - 3K emails/mo (enough for ~500 active users) + - No CDN, no custom email domain + - Vercel hobby = non-commercial only + + Good for: Development, beta testing, first users + + --- + Small Scale — ~$20/mo (100 users) + + What you get (on top of minimum): + - Neon Launch x2 ($5 each) — real production databases, auto-scaling compute, more storage + - AWS S3 for media uploads (10 GB) — proper file storage, not VPS disk + - Everything else still on free tiers + + Limits: + - S3 egress: first 100 GB/mo free + - 3K emails/mo still (upgrade to Resend Pro at $20/mo if needed) + - Single VPS (no redundancy) + + Good for: Soft launch, early adopters, getting first paying subscribers + + --- + Growth — ~$163/mo (1,000 users) + + What you get (on top of small): + - Bigger VPS (8 vCPU, 16 GB RAM) — handles concurrent WebSocket connections, video processing + - Neon Scale x2 — higher connection limits, read replicas, compliance features + - Vercel Pro — commercial use allowed, 1 TB bandwidth, team collaboration + - Clerk Pro — SSO, enterprise auth features, unlimited orgs + - Resend Pro — 50K emails/mo + - Sentry Team — 100K errors/mo, multiple users, performance monitoring + - Upstash paid — more Redis capacity for rate limiting, sessions, caching + - AWS S3 100 GB + egress + + Limits: + - Single VPS still (for HA, add second VPS +$18/mo) + - No dedicated DB (Neon Scale is still serverless) + + Good for: Active community, real revenue, hundreds of daily active users + + --- + Break-even math + + With $9/mo minimum infra and $18.80 net per subscriber: + + ┌─────────────────────────────┬────────────────────┐ + │ │ Subscribers needed │ + ├─────────────────────────────┼────────────────────┤ + │ Cover minimum infra ($9/mo) │ 1 subscriber │ + ├─────────────────────────────┼────────────────────┤ + │ Cover small scale ($20/mo) │ 2 subscribers │ + ├─────────────────────────────┼────────────────────┤ + │ Cover growth ($163/mo) │ 9 subscribers │ + └─────────────────────────────┴────────────────────┘ + + ┌─────────────────┬───────────────────────────────┬────────┐ + │ Product │ Check │ Result │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM CLI │ Build (tsc) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM CLI │ Tests (520/520) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM CLI │ Runtime (--version → 2.0.0) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM CLI │ Help (23 commands listed) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM MCP │ Build (tsc) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM MCP │ Runtime (stdio server starts) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM API Server │ Type-check │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM API Server │ Tests (538/538, +19 skip) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM VS Code │ Tests (62/62) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ AITM Website │ Next.js build │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ Gitgram Backend │ Type-check (0 errors) │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ Gitgram Web │ Next.js build │ PASS │ + ├─────────────────┼───────────────────────────────┼────────┤ + │ Gitgram Web │ Tests (72/72) │ PASS │ + └─────────────────┴───────────────────────────────┴────────┘ + + we can fork bd https://github.com/upstash/context7.git https://github.com/BeehiveInnovations/pal-mcp-server.git for aitm mcp and aitm stuff and for + gitgram we can clone some openourse social media stuff becuase gitgram is social media layer for github repositories we can fork bd https://github.com/upstash/context7.git https://github.com/BeehiveInnovations/pal-mcp-server.git for aitm mcp and aitm stuff and for + gitgram we can clone some openourse social media stuff becuase gitgram is social media layer for github repositories vvv + --- + Repos to Fork/Clone + + For AITM MCP + + ┌───────────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Repo │ What to Adapt │ + ├───────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ upstash/context7 │ Streamable HTTP transport (AITM is stdio-only), AsyncLocalStorage per-request context, tool description │ + │ │ engineering, multi-agent setup CLI (aitm setup --claude --cursor), SDK client pattern with retry │ + ├───────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ BeehiveInnovations/pal-mcp-server │ ToolAnnotations (readOnlyHint/destructiveHint) for all 40 tools, modular tool class architecture (extract from │ + │ │ monolithic index.ts), DISABLED_TOOLS env var, standardized output envelope, path validation for restore safety │ + ├───────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ abhigyanpatwari/GitNexus │ Zero-server knowledge graph + Graph RAG agent over repos. Adapt for AITM code-context indexing, checkpoint │ + │ → external/gitnexus │ knowledge graphs, and Gitgram repo explore/visualisation surface. Cloned 2026-06-21. │ + ├───────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ gastownhall/beads │ Git-native DAG issue tracker for AI agents. `bd ready` shows dependency-unblocked work with hash-based IDs. │ + │ → external/beads │ Reference for AITM task/issue tracking integration and multi-agent coordination. Cloned 2026-06-21. │ + └───────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + + For Gitgram (Social Layer) + + ┌─────┬──────────────────────────────────┬───────┬─────────┬───────────────────────────────────────────────────────────────────────────────────────────┐ + │ # │ Repo │ Stars │ License │ Why │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 1 │ codinginflow/nextjs-15-social- │ 417 │ -- │ Exact same stack (Next.js + Prisma + PG). Prisma schema maps 1:1 to Gitgram. Infinite │ + │ │ media-app → external/nextjs-15- │ │ │ scroll, optimistic updates, notifications. Cloned 2026-06-21. │ + │ │ social-media-app │ │ │ │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 2 │ leandronorcio/munia │ 293 │ MIT │ Most complete features: GitHub OAuth, @mentions, hashtags, nested comments, drag-drop │ + │ │ → external/munia │ │ │ media, S3 uploads, accessibility. Cloned 2026-06-21. │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 3 │ TomDoesTech/t3-twitter-clone │ 189 │ MIT │ Only one with Express branch — matches Gitgram's Express backend. Type-safe tRPC │ + │ │ → external/t3-twitter-clone │ │ │ patterns. Cloned 2026-06-21. │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 4 │ karanpratapsingh/Proximity │ 360 │ GPL-3 │ Best React Native social app with Prisma. Real-time messaging via GraphQL subscriptions. │ + │ │ → external/proximity │ │ │ Reference only (GPL). Cloned 2026-06-21. │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 5 │ bluesky-social/social-app │ 17.8K │ MIT │ Production-scale feed algorithms, notifications, moderation. Architecture reference. │ + │ │ → external/bluesky-social-app │ │ │ Cloned 2026-06-21. │ + ├─────┼──────────────────────────────────┼───────┼─────────┼───────────────────────────────────────────────────────────────────────────────────────────┤ + │ 6 │ abhigyanpatwari/GitNexus │ -- │ -- │ Client-side knowledge graph + Graph RAG over any repo. Reference for Gitgram repo │ + │ │ → external/gitnexus │ │ │ explore/visualisation. Also useful for AITM. Cloned 2026-06-21. │ + └─────┴──────────────────────────────────┴───────┴─────────┴───────────────────────────────────────────────────────────────────────────────────────────┘ + + Quick-win bonus + + - amirfakhrullah/gh-social (MIT) — The only "GitHub Social Platform" found. Syncs GitHub follows/stars to a social feed. Next.js + tRPC + Clerk + + Drizzle + Neon. Closest concept to Gitgram. Cloned 2026-06-21 → external/gh-social. + - gastownhall/beads (MIT) — Git-native DAG issue tracker by Steve Yegge for AI agents. Cloned 2026-06-21 → external/beads. + + --- + Top Actionable Items from Research + + AITM MCP (high priority, small effort): + 1. Add ToolAnnotations to all 40 tools — readOnlyHint: true for read tools, destructiveHint: true for restore + 2. Add Streamable HTTP transport (from context7 pattern) — enables web/VS Code clients + 3. Extract tools from monolithic index.ts into modular files + + Gitgram (reference patterns to study): + 1. Prisma schema design from nextjs-15-social-media-app + 2. @mentions + hashtag system from munia + 3. Feed pagination (cursor-based infinite scroll) from both + 4. GitHub API integration from gh-social + + │ BloopAI/vibe-kanban │ 24.6k │ Kanban for AI agents, MCP server+client, git worktree isolation, diff review. Best task board │ + │ │ │ foundation. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ gastownhall/beads (bd!) │ 20.4k │ Git-native DAG issue tracker made for AI agents. This is what bd is. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ a2aproject/a2a-js │ 509 │ Official A2A TypeScript SDK. Needed for standards compliance. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ agentgateway/agentgateway │ 2.3k │ Rust proxy for A2A + MCP traffic. Security/observability built in. │ + + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ GongRzhe/A2A-MCP-Server │ 147 │ Bridges MCP ↔ A2A so Claude can talk to A2A agents. │ + + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ Open-ACP/OpenACP │ 58 │ Connects 28+ AI agents to Telegram, Discord, Slack. Real-time messaging. │ + └───────────────────────────┴───────┴───────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + Key insight about bd (Beads) + + + Beads by Steve Yegge is exactly what you're already using for task tracking! It's a git-native graph issue tracker designed for AI agents. + + bd ready shows unblocked work, dependency DAGs, hash-based IDs for concurrent agents. 20k+ stars, MIT. + + │ BloopAI/vibe-kanban │ 24.6k │ Kanban for AI agents, MCP server+client, git worktree isolation, diff review. Best task board │ + │ │ │ foundation. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ gastownhall/beads (bd!) │ 20.4k │ Git-native DAG issue tracker made for AI agents. This is what bd is. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ a2aproject/a2a-js │ 509 │ Official A2A TypeScript SDK. Needed for standards compliance. │ + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ agentgateway/agentgateway │ 2.3k │ Rust proxy for A2A + MCP traffic. Security/observability built in. │ + + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ GongRzhe/A2A-MCP-Server │ 147 │ Bridges MCP ↔ A2A so Claude can talk to A2A agents. │ + + ├───────────────────────────┼───────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ + + │ Open-ACP/OpenACP │ 58 │ Connects 28+ AI agents to Telegram, Discord, Slack. Real-time messaging. │ + └───────────────────────────┴───────┴───────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + Key insight about bd (Beads) + + + Beads by Steve Yegge is exactly what you're already using for task tracking! It's a git-native graph issue tracker designed for AI agents. + + bd ready shows unblocked work, dependency DAGs, hash-based IDs for concurrent agents. 20k+ stars, MIT. + + # Unified Manual Launch Plan for Gitgram + AITM + + ## Summary + + Prepare both products for a parallel launch using a single server / operator-managed deployment model as the source of truth. The plan should treat the + existing repo as mostly implementation-complete, but should not trust the docs blindly: first reconcile deployment artifacts, environment contracts, and + stale references, then execute a manual launch path that does not depend on unfinished CI/CD automation. + + Success means: + + - AITM npm packages are published and installable. + - Gitgram backend/web and AITM API are deployable from documented manual commands. + - Production secrets/env contracts are complete and validated. + - Health checks, rollback steps, and operator runbooks match the actual repo layout. + - Stale or conflicting deployment guidance is removed or explicitly deprecated. + + ## Implementation Changes + + ### 1. Establish one canonical launch topology + + - Make the manual-ops path the primary launch contract: single operator-managed host, Docker/PM2/systemd as applicable, no dependency on unfinished + GitHub Actions deployment flows. + - Standardize the production target as: + - Gitgram backend: long-running Node service on the server. + - Gitgram web: long-running Next.js service on the server. + - AITM API: long-running Node service on the server. + - Shared infra: PostgreSQL + Redis on the same host or tightly coupled managed services, but documented as one operational unit. + - Demote GitHub Actions deploy workflows to “future automation” unless they fully match the manual launch path. + + ### 2. Reconcile repo truth with docs before launch execution + + - Audit and fix contradictions between SINGLE_SOURCE_PLAN.md, docs/deploy-pipeline.md, docs/ops-quickref.md, docs/runbook.md, compose files, and actual + workflow filenames. + - Remove or clearly mark stale deployment variants that conflict with the chosen topology. + - Resolve broken references in docs, especially any paths/services that do not exist in the tree. + - Normalize naming so the same service names, ports, health endpoints, workflow names, and env var conventions are used everywhere. + - Treat docs/can be fork.md as the maintained reference-clone inventory; use SINGLE_SOURCE_PLAN.md as the canonical execution plan for launch sequencing. + + ### 3. Lock the manual launch path per product + + - AITM: + - Verify package publish order and prerequisites for @gitgram/timemachine and @gitgram/timemachine-mcp. + - Define the exact manual publish checklist, post-publish verification, and rollback/version bump procedure. + - Define the exact API deployment procedure, including build, env validation, migration application, service restart, and health verification. + - Gitgram: + - Define the exact backend deployment procedure, including Prisma generation/migration, build, restart, and health verification. + - Define the exact web deployment procedure, including build, runtime env injection, restart, and smoke verification. + - Shared: + - Choose one restart/control mechanism per service and document it as canonical. If both PM2 and systemd are kept, assign one as primary and one as + fallback, not equal-status alternatives. + + ### 4. Harden secrets and env contracts for launch + + - Use docs/secrets-contract.md and docs/environment-matrix.md as the starting point, but reconcile them with actual runtime/package needs. + - Produce one canonical required-vars list for: + - Gitgram backend + - Gitgram web + - AITM API + - AITM package publish/release flow + - Validate that production-only integrations needed for launch are present and named consistently, especially database, Redis, JWT, Polar, S3/storage, + email, and any internal auth secrets. + - Ensure validation scripts and runbooks match the chosen unprefixed/prefixed env convention and the manual deployment path. + - Explicitly document which secrets are required before first launch vs optional post-launch integrations. + + ### 5. Define launch-day operations and rollback + + - Convert the runbook into an operator-focused launch sequence with: + - preflight checks + - ordered deploy steps + - health gates + - smoke checks + - rollback commands + - post-launch monitoring window + - Rollback should be manual-first and concrete: + - previous package version for npm artifacts + - previous app build/image/tag for server services + - DB migration rollback policy or “restore from backup only” policy where down-migrations are not supported + - Require a backup/checkpoint step before applying production migrations. + + ## Public Interfaces / Contracts + + - No product API redesign is planned. + - Public-facing contract changes are limited to: + - canonical deployment/runbook docs + - explicit deprecation of stale deployment paths and references + - If any current docs mention missing components such as non-existent service paths, update the documentation contract rather than inventing new + components for launch. + + ## Test Plan + + - Verify repo-level build/test/typecheck commands for the launch-critical services still pass. + - Dry-run the manual deployment commands in a staging-like environment for: + - Gitgram backend + - Gitgram web + - AITM API + - AITM package publish verification flow + - Validate health endpoints after each service restart. + - Run minimal smoke scenarios: + - Gitgram auth and basic API reachability + - Gitgram web load and API connectivity + - AITM API health/readiness and one representative core flow + - npm install plus basic CLI/MCP invocation for published AITM packages + - Test rollback instructions at least once in staging-like conditions. + + ## Assumptions and Defaults + + - Launch is for both products in parallel, not a staggered product strategy. + - The delivery path is manual operations first, with CI/CD treated as non-blocking unless brought into alignment. + - The hosting model is single server / operator-managed, not a multi-platform redesign. + - Existing code is assumed largely complete; the main launch risk is operational inconsistency, stale docs, and deployment drift. + - Any workflow or doc that conflicts with the chosen manual launch path should be updated, deprecated, or ignored rather than preserved for symmetry. diff --git a/V1_90_DAY_BACKLOG.md b/V1_90_DAY_BACKLOG.md new file mode 100644 index 00000000..29323092 --- /dev/null +++ b/V1_90_DAY_BACKLOG.md @@ -0,0 +1,112 @@ +# V1 90-Day Backlog (Actionable) + +Canonical product reference: `NORTH_STAR_PRD_ARCHITECTURE_V1.md` +Canonical schema reference: `CANONICAL_SQL_SCHEMA_V1.sql` + +## Milestone A (Days 1-30) - Core Reliability + +### Epic A1 - Canonical Data Layer +- `A1-1` Define migration baseline from `CANONICAL_SQL_SCHEMA_V1.sql`. + - Acceptance: migrations apply cleanly in dev/staging; rollback tested. +- `A1-2` Implement repository/service layer for `checkpoints`, `checkpoint_files`, `messages`, `tool_calls`, `contexts`. + - Acceptance: CRUD + listing APIs pass integration tests. +- `A1-3` Add idempotency for action capture (`before_action`, `after_action`). + - Acceptance: duplicate requests do not create duplicate checkpoints. + +### Epic A2 - Deterministic Restore +- `A2-1` Implement `POST /api/time/restore` for project/file/folder scope. + - Acceptance: restore supports exact checkpoint IDs and time-based selection. +- `A2-2` Implement `POST /api/time/undo` using linked before/after checkpoints. + - Acceptance: undo returns previous good state in one operation. +- `A2-3` Add restore preview endpoint with diff summary. + - Acceptance: clients can fetch preview before applying restore. + +### Epic A3 - Capture Pipeline +- `A3-1` Implement `POST /api/actions/before` and `POST /api/actions/after`. + - Acceptance: every successful action writes linked checkpoints. +- `A3-2` Compute/store unified diffs and file-level stats. + - Acceptance: `GET /api/time/diff` returns precomputed diff + stats. +- `A3-3` Add sync worker for local cache to backend reconciliation. + - Acceptance: sync lag metric reported; retries with backoff enabled. + +### Epic A4 - Governance Baseline +- `A4-1` Implement consent model (`operational_capture`, `training_export`) with org/project scope. + - Acceptance: capture/export endpoints enforce consent checks. +- `A4-2` Implement retention policy execution job. + - Acceptance: expired records are soft-deleted then hard-deleted per policy. +- `A4-3` Implement audit logging for restore/export/delete actions. + - Acceptance: every governed action emits immutable audit event. + +### Epic A5 - Gitgram Baseline Social +- `A5-1` Implement `POST /api/repos/import` for public GitHub repos. + - Acceptance: imported repo has profile and feed shell. +- `A5-2` Implement posts/comments/reactions/follows APIs. + - Acceptance: end-to-end create/read flows work for authenticated users. +- `A5-3` Implement trending scorer v1 (GitHub + internal engagement). + - Acceptance: `GET /api/trending` returns ranked repositories with reasons. + +## Milestone B (Days 31-60) - Social + Time Integration + +### Epic B1 - Checkpoint-Linked Content +- `B1-1` Add post type `change_story` referencing checkpoint IDs. + - Acceptance: posts can render before/after diff summary. +- `B1-2` Add media upload pipeline for screenshots/videos. + - Acceptance: signed upload + moderation scan before publish. +- `B1-3` Add repository timeline widget consuming AITM checkpoints. + - Acceptance: repo profile shows ordered technical change events. + +### Epic B2 - Timeline UX and Reliability +- `B2-1` Ship timeline endpoint pagination/filtering. + - Acceptance: filters by actor, tool, file, and date range. +- `B2-2` Add optimistic restore flow with server-side conflict checks. + - Acceptance: conflict paths return actionable error payloads. +- `B2-3` Add latency budgets and SLO dashboards for restore/diff/sync. + - Acceptance: SLO alerts trigger on threshold breaches. + +### Epic B3 - Org and Access Controls +- `B3-1` Implement organizations + memberships + role checks. + - Acceptance: owner/admin/member role matrix enforced in APIs. +- `B3-2` Implement scoped API keys and key rotation. + - Acceptance: read/write/admin scopes fully enforced and auditable. +- `B3-3` Add project-level policy binding (capture/export/retention). + - Acceptance: policy inheritance and overrides are deterministic. + +## Milestone C (Days 61-90) - Intelligence + Growth + +### Epic C1 - Semantic Search +- `C1-1` Add embedding worker for code/message chunks. + - Acceptance: embeddings persisted for eligible checkpoints/messages. +- `C1-2` Implement `POST /api/search/code` and `POST /api/search/chats`. + - Acceptance: relevance and latency meet benchmark targets. +- `C1-3` Add hybrid search (keyword + vector rerank). + - Acceptance: quality improves over keyword baseline in eval set. + +### Epic C2 - Training Export Pipeline +- `C2-1` Implement export jobs with consent gate and policy checks. + - Acceptance: exports fail closed when consent/policy is missing. +- `C2-2` Implement redaction pass for secrets/PII/license-sensitive text. + - Acceptance: sampled audits meet precision/recall targets. +- `C2-3` Implement signed export artifacts + audit traceability. + - Acceptance: each artifact maps to consent ID, job ID, and actor ID. + +### Epic C3 - Creator Growth Features +- `C3-1` Add reels/stories authoring flows for repo owners. + - Acceptance: owners can publish media content tied to repo/project. +- `C3-2` Add follow recommendations (repo + developer graph). + - Acceptance: recommendation CTR beats random baseline. +- `C3-3` Add anti-abuse moderation queue and enforcement actions. + - Acceptance: flagged content lifecycle is fully auditable. + +## Cross-Cutting Tickets + +- `X-1` Test strategy: unit + integration + e2e suites for capture/restore/social/governance APIs. +- `X-2` Observability: metrics, tracing, logs with request and checkpoint correlation IDs. +- `X-3` Security: threat model review and tenant-isolation verification tests. +- `X-4` Documentation: API reference + runbooks + incident handling guides. + +## KPI Gates (Must Hit) + +- `KPI-1` Restore success rate >= 99%. +- `KPI-2` P95 restore latency <= 2s (single file), <= 10s (medium project). +- `KPI-3` >= 25% of active repos receive at least one social interaction. +- `KPI-4` 100% training export jobs traceable to consent and audit records. diff --git a/V1_90_DAY_GITHUB_ISSUES.md b/V1_90_DAY_GITHUB_ISSUES.md new file mode 100644 index 00000000..20ace147 --- /dev/null +++ b/V1_90_DAY_GITHUB_ISSUES.md @@ -0,0 +1,474 @@ +# V1 90-Day Backlog -> GitHub Issues Format + +Reference docs: +- `NORTH_STAR_PRD_ARCHITECTURE_V1.md` +- `V1_90_DAY_BACKLOG.md` +- `CANONICAL_SQL_SCHEMA_V1.sql` + +Label conventions: +- Milestone: `m:A`, `m:B`, `m:C`, `m:X` +- Area: `area:data`, `area:time-machine`, `area:social`, `area:governance`, `area:search`, `area:platform` +- Priority: `p0`, `p1`, `p2` +- Type: `type:feature`, `type:infra`, `type:security`, `type:docs` + +Use this per issue: +```bash +gh issue create --title "" --label "<comma-separated-labels>" --body-file /tmp/issue.md +``` + +## Milestone A (Days 1-30) + +### [A1-1] Define migration baseline from canonical SQL +- Labels: `m:A,area:data,p0,type:infra` +- Depends on: none +- Body: +```md +## Objective +Create a stable migration baseline from `CANONICAL_SQL_SCHEMA_V1.sql`. + +## Scope +- Establish migration ordering and rollback strategy. +- Validate extensions (`pgcrypto`, `vector`) and compatibility in staging. + +## Acceptance Criteria +- Migrations apply cleanly in dev/staging. +- Rollback path is tested and documented. +``` + +### [A1-2] Implement repositories/services for core time-context tables +- Labels: `m:A,area:data,p0,type:feature` +- Depends on: `A1-1` +- Body: +```md +## Objective +Implement data access services for checkpoints, checkpoint_files, messages, tool_calls, contexts. + +## Acceptance Criteria +- CRUD + list APIs pass integration tests. +- Pagination/filtering support for timeline use cases. +``` + +### [A1-3] Add idempotency for before/after action capture +- Labels: `m:A,area:time-machine,p0,type:infra` +- Depends on: `A1-2` +- Body: +```md +## Objective +Prevent duplicate checkpoints for repeated action submissions. + +## Acceptance Criteria +- Duplicate requests do not create duplicate checkpoints. +- Idempotency key behavior is documented and tested. +``` + +### [A2-1] Implement restore endpoint for file/folder/project scope +- Labels: `m:A,area:time-machine,p0,type:feature` +- Depends on: `A1-2` +- Body: +```md +## Objective +Implement `POST /api/time/restore` for project, folder, and file scopes. + +## Acceptance Criteria +- Supports checkpoint-id and time-based restore selectors. +- Returns deterministic restore summary payload. +``` + +### [A2-2] Implement undo-last via linked before/after checkpoints +- Labels: `m:A,area:time-machine,p0,type:feature` +- Depends on: `A2-1,A3-1` +- Body: +```md +## Objective +Implement `POST /api/time/undo` based on linked before/after checkpoints. + +## Acceptance Criteria +- One-call rollback to previous good state. +- Handles missing links gracefully with actionable errors. +``` + +### [A2-3] Add restore preview endpoint with diff summary +- Labels: `m:A,area:time-machine,p1,type:feature` +- Depends on: `A3-2` +- Body: +```md +## Objective +Show preview diff and file impact before applying restore. + +## Acceptance Criteria +- Preview includes added/modified/deleted counts and key file list. +- Clients can block restore when preview indicates risk. +``` + +### [A3-1] Implement before/after action APIs +- Labels: `m:A,area:time-machine,p0,type:feature` +- Depends on: `A1-2` +- Body: +```md +## Objective +Implement `POST /api/actions/before` and `POST /api/actions/after`. + +## Acceptance Criteria +- Successful actions write linked checkpoints. +- Message/tool-call linkage is persisted. +``` + +### [A3-2] Compute/store unified diffs and file-level stats +- Labels: `m:A,area:time-machine,p0,type:feature` +- Depends on: `A3-1` +- Body: +```md +## Objective +Persist precomputed diffs for fast timeline and restore previews. + +## Acceptance Criteria +- `GET /api/time/diff` reads precomputed diffs. +- Diff stats are persisted per file and per action. +``` + +### [A3-3] Add local-cache to backend sync worker +- Labels: `m:A,area:platform,p1,type:infra` +- Depends on: `A3-1` +- Body: +```md +## Objective +Implement resilient push/retry pipeline from local cache to canonical backend. + +## Acceptance Criteria +- Backoff + retry enabled. +- Sync lag metric emitted. +``` + +### [A4-1] Implement consent model enforcement +- Labels: `m:A,area:governance,p0,type:security` +- Depends on: `A1-1` +- Body: +```md +## Objective +Enforce `operational_capture` and `training_export` consent by org/project scope. + +## Acceptance Criteria +- Capture/export endpoints enforce consent checks. +- No export allowed without explicit valid consent. +``` + +### [A4-2] Implement retention policy execution job +- Labels: `m:A,area:governance,p1,type:security` +- Depends on: `A4-1` +- Body: +```md +## Objective +Enforce retention windows for governed data classes. + +## Acceptance Criteria +- Expired records are processed with soft/hard delete policy. +- Job outputs audit trail. +``` + +### [A4-3] Implement immutable audit logging for governed actions +- Labels: `m:A,area:governance,p0,type:security` +- Depends on: `A4-1` +- Body: +```md +## Objective +Audit restore/export/delete/security actions with actor and target metadata. + +## Acceptance Criteria +- Every governed action writes an audit row. +- Query API supports actor/date/action filters. +``` + +### [A5-1] Import public GitHub repos into social profiles +- Labels: `m:A,area:social,p1,type:feature` +- Depends on: none +- Body: +```md +## Objective +Implement `POST /api/repos/import` and create repo social profile shell. + +## Acceptance Criteria +- Imported repo appears with basic profile/feed. +- Duplicate import is idempotent. +``` + +### [A5-2] Implement posts/comments/reactions/follows APIs +- Labels: `m:A,area:social,p1,type:feature` +- Depends on: `A5-1` +- Body: +```md +## Objective +Ship social primitives for interaction on repository content. + +## Acceptance Criteria +- Authenticated users can create/read interactions. +- Ownership and visibility checks enforced. +``` + +### [A5-3] Implement trending scorer v1 +- Labels: `m:A,area:social,p2,type:feature` +- Depends on: `A5-2` +- Body: +```md +## Objective +Compute `trend_scores` using GitHub and in-app engagement signals. + +## Acceptance Criteria +- `GET /api/trending` returns ranked repositories. +- Signal contribution is included in debug metadata. +``` + +## Milestone B (Days 31-60) + +### [B1-1] Add `change_story` post type linked to checkpoints +- Labels: `m:B,area:social,p1,type:feature` +- Depends on: `A2-3,A5-2` +- Body: +```md +## Objective +Allow posts that reference checkpoint IDs and render change summaries. + +## Acceptance Criteria +- `change_story` posts resolve checkpoint + diff preview. +- Invalid checkpoint references are rejected. +``` + +### [B1-2] Add media upload pipeline for screenshots/videos +- Labels: `m:B,area:social,p1,type:feature` +- Depends on: `A5-2` +- Body: +```md +## Objective +Support signed uploads and moderation checks before publish. + +## Acceptance Criteria +- Image/video uploads complete through signed URL flow. +- Moderation status gates publishing. +``` + +### [B1-3] Add repository timeline widget from AITM checkpoints +- Labels: `m:B,area:social,p2,type:feature` +- Depends on: `A3-2,B1-1` +- Body: +```md +## Objective +Display technical change timeline on repo profile pages. + +## Acceptance Criteria +- Timeline endpoint includes actor/tool/file summaries. +- UI consumes paginated timeline API. +``` + +### [B2-1] Timeline API pagination/filtering +- Labels: `m:B,area:time-machine,p1,type:feature` +- Depends on: `A3-2` +- Body: +```md +## Objective +Add timeline filters by actor, tool, file, and date range. + +## Acceptance Criteria +- Stable pagination cursors. +- Filtered results verified by integration tests. +``` + +### [B2-2] Optimistic restore flow with conflict checks +- Labels: `m:B,area:time-machine,p1,type:feature` +- Depends on: `A2-1` +- Body: +```md +## Objective +Improve restore UX while preventing unsafe apply when local state diverges. + +## Acceptance Criteria +- Conflict responses include actionable metadata. +- Client can retry with explicit override. +``` + +### [B2-3] SLO dashboards and alerts for restore/diff/sync +- Labels: `m:B,area:platform,p1,type:infra` +- Depends on: `A2-1,A3-3` +- Body: +```md +## Objective +Track restore latency/success, diff latency, and sync lag SLOs. + +## Acceptance Criteria +- Dashboards and alert thresholds exist. +- On-call runbook references these alerts. +``` + +### [B3-1] Implement organizations, memberships, role matrix +- Labels: `m:B,area:governance,p1,type:feature` +- Depends on: `A1-1` +- Body: +```md +## Objective +Ship owner/admin/member authorization checks for org-scoped resources. + +## Acceptance Criteria +- Role matrix enforced on protected endpoints. +- Membership management endpoints are tested. +``` + +### [B3-2] Scoped API keys and rotation +- Labels: `m:B,area:governance,p1,type:security` +- Depends on: `B3-1` +- Body: +```md +## Objective +Enforce read/write/admin key scopes and key rotation policy. + +## Acceptance Criteria +- Scope violations return 403 with traceable audit events. +- Rotation flow supports zero-downtime cutover. +``` + +### [B3-3] Project-level policy binding and inheritance +- Labels: `m:B,area:governance,p1,type:feature` +- Depends on: `A4-1,B3-1` +- Body: +```md +## Objective +Bind capture/export/retention policies to org/project with deterministic inheritance. + +## Acceptance Criteria +- Effective policy resolution API implemented. +- Override precedence tested. +``` + +## Milestone C (Days 61-90) + +### [C1-1] Embedding worker for code/message chunks +- Labels: `m:C,area:search,p1,type:feature` +- Depends on: `A1-2` +- Body: +```md +## Objective +Generate embeddings asynchronously for eligible checkpoints/messages/docs. + +## Acceptance Criteria +- Worker persists embeddings with retry logic. +- Backfill mode supports historical checkpoints. +``` + +### [C1-2] Semantic search APIs for code and chats +- Labels: `m:C,area:search,p1,type:feature` +- Depends on: `C1-1` +- Body: +```md +## Objective +Implement `POST /api/search/code` and `POST /api/search/chats`. + +## Acceptance Criteria +- Returns ranked results with relevance score. +- Meets baseline latency target in staging. +``` + +### [C1-3] Hybrid search (keyword + vector rerank) +- Labels: `m:C,area:search,p2,type:feature` +- Depends on: `C1-2` +- Body: +```md +## Objective +Improve quality by combining lexical retrieval with vector reranking. + +## Acceptance Criteria +- Evaluation set shows improved relevance over keyword baseline. +- Explainability metadata included for debugging. +``` + +### [C2-1] Training export jobs with consent/policy gates +- Labels: `m:C,area:governance,p0,type:security` +- Depends on: `A4-1,B3-3` +- Body: +```md +## Objective +Fail-closed export flow requiring valid consent and effective policy. + +## Acceptance Criteria +- Jobs are blocked when gates fail. +- Gate decisions are logged in audit records. +``` + +### [C2-2] Redaction pipeline for secrets/PII/license-sensitive text +- Labels: `m:C,area:governance,p0,type:security` +- Depends on: `C2-1` +- Body: +```md +## Objective +Redact sensitive data before export artifact creation. + +## Acceptance Criteria +- Precision/recall quality gates defined and met on sampled audits. +- Redaction report attached to export job summary. +``` + +### [C2-3] Signed export artifacts with full traceability +- Labels: `m:C,area:governance,p1,type:feature` +- Depends on: `C2-2` +- Body: +```md +## Objective +Ship signed artifact output and trace links to consent/job/actor. + +## Acceptance Criteria +- Artifact metadata includes consent_id, job_id, actor_id. +- Download access enforces authorization. +``` + +### [C3-1] Reels/stories authoring flow for repo owners +- Labels: `m:C,area:social,p2,type:feature` +- Depends on: `B1-2` +- Body: +```md +## Objective +Creator workflow for publishing reels/stories tied to repositories. + +## Acceptance Criteria +- Repo owners can draft, preview, publish. +- Visibility rules enforced. +``` + +### [C3-2] Follow recommendations (repo + developer graph) +- Labels: `m:C,area:social,p2,type:feature` +- Depends on: `A5-2` +- Body: +```md +## Objective +Recommend repositories/developers to follow based on activity graph. + +## Acceptance Criteria +- Recommendation endpoint returns scored candidates. +- CTR beats random baseline. +``` + +### [C3-3] Anti-abuse moderation queue and enforcement actions +- Labels: `m:C,area:social,p1,type:security` +- Depends on: `B1-2` +- Body: +```md +## Objective +Moderation queue for flagged social content with auditable actions. + +## Acceptance Criteria +- Flag -> review -> action lifecycle is tracked. +- Moderator actions are auditable. +``` + +## Cross-Cutting + +### [X-1] End-to-end test strategy for capture/restore/social/governance +- Labels: `m:X,area:platform,p0,type:infra` +- Depends on: none + +### [X-2] Observability with correlation IDs and tracing +- Labels: `m:X,area:platform,p1,type:infra` +- Depends on: `A3-1` + +### [X-3] Security threat model and tenant-isolation verification +- Labels: `m:X,area:governance,p0,type:security` +- Depends on: `A4-1` + +### [X-4] API/runbook/incident documentation +- Labels: `m:X,area:platform,p1,type:docs` +- Depends on: `A1-1` diff --git a/aitimemachine b/aitimemachine new file mode 160000 index 00000000..937722ab --- /dev/null +++ b/aitimemachine @@ -0,0 +1 @@ +Subproject commit 937722abe0db76e8f3915fb9861d9479b5902dc2 diff --git a/deploy/DEPLOYMENT.md b/deploy/DEPLOYMENT.md new file mode 100644 index 00000000..454eb27e --- /dev/null +++ b/deploy/DEPLOYMENT.md @@ -0,0 +1,18 @@ +# Deprecated Production Compose Guide + +This file is retained only as a historical reference for the old full-docker single-VPS experiment. + +Do not use it as the launch source of truth. + +Use `docs/runbook.md` instead. + +Why: + +- launch is manual-first and PM2-driven +- `docker-compose.prod.yml` includes paths and services that are not the canonical launch contract +- the canonical launch scope is Gitgram backend + Gitgram web + AITM API on one operator-managed host, with PostgreSQL and Redis treated as shared infra + +What remains useful here: + +- the repository still contains `docker-compose.prod.yml` if you want a non-canonical all-docker rehearsal +- the root `.env.*.example` files are still the operator-facing env templates diff --git a/deploy/backup.sh b/deploy/backup.sh new file mode 100755 index 00000000..d1b00194 --- /dev/null +++ b/deploy/backup.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Deprecated full-docker backup helper +# Usage: bash deploy/backup.sh +# Canonical backup procedure: docs/runbook.md +# ============================================================================= + +COMPOSE_FILE="docker-compose.prod.yml" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +BACKUP_DIR="$PROJECT_DIR/backups" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) + +cd "$PROJECT_DIR" + +mkdir -p "$BACKUP_DIR" + +echo "=========================================" +echo " Deprecated Docker Backup Helper — $TIMESTAMP" +echo "=========================================" +echo "This script only applies to the non-canonical docker-compose.prod.yml path." +echo "Use docs/runbook.md for the launch backup gate." +echo "" + +# --- PostgreSQL --- +echo "[1/3] Backing up PostgreSQL..." +docker compose -f "$COMPOSE_FILE" exec -T postgres \ + pg_dumpall -U "${POSTGRES_USER:-gitgram}" \ + | gzip > "$BACKUP_DIR/postgres_$TIMESTAMP.sql.gz" +echo " Saved: postgres_$TIMESTAMP.sql.gz ($(du -h "$BACKUP_DIR/postgres_$TIMESTAMP.sql.gz" | cut -f1))" + +# --- Redis --- +echo "[2/3] Backing up Redis..." +docker compose -f "$COMPOSE_FILE" exec -T redis redis-cli BGSAVE > /dev/null 2>&1 || true +sleep 2 +docker compose -f "$COMPOSE_FILE" cp redis:/data/dump.rdb "$BACKUP_DIR/redis_$TIMESTAMP.rdb" 2>/dev/null || echo " Redis backup skipped (no dump.rdb)" + +# --- Uploads --- +echo "[3/3] Backing up uploads..." +if [ -d "$PROJECT_DIR/gitgram/backend/uploads" ]; then + tar -czf "$BACKUP_DIR/uploads_$TIMESTAMP.tar.gz" -C "$PROJECT_DIR/gitgram/backend" uploads + echo " Saved: uploads_$TIMESTAMP.tar.gz" +else + echo " No uploads directory found, skipping." +fi + +# --- Cleanup old backups (keep last 7 days) --- +echo "" +echo "Cleaning backups older than 7 days..." +find "$BACKUP_DIR" -name "*.gz" -mtime +7 -delete 2>/dev/null || true +find "$BACKUP_DIR" -name "*.rdb" -mtime +7 -delete 2>/dev/null || true + +echo "" +echo "=========================================" +echo " Backup Complete!" +echo "=========================================" +ls -lh "$BACKUP_DIR"/*_$TIMESTAMP.* 2>/dev/null diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 00000000..87b07cd1 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Deprecated full-docker rehearsal script +# Run from project root: bash deploy/deploy.sh +# Canonical launch path: docs/runbook.md +# ============================================================================= + +COMPOSE_FILE="docker-compose.prod.yml" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +cd "$PROJECT_DIR" + +echo "=========================================" +echo " Deprecated Docker Rehearsal Deploy" +echo "=========================================" +echo "This script is not the canonical launch path." +echo "Use docs/runbook.md for the PM2-based manual launch contract." +echo "" + +# Pull latest code +if [ -d .git ]; then + echo "[1/4] Pulling latest code..." + git pull --ff-only +else + echo "[1/4] Not a git repo, skipping pull." +fi + +# Build all images +echo "[2/4] Building Docker images..." +docker compose -f "$COMPOSE_FILE" build + +# Start/restart services +echo "[3/4] Starting services..." +docker compose -f "$COMPOSE_FILE" up -d + +# Cleanup old images +echo "[4/4] Cleaning up old images..." +docker image prune -f + +echo "" +echo "=========================================" +echo " Deployment Complete!" +echo "=========================================" +echo "" +docker compose -f "$COMPOSE_FILE" ps diff --git a/deploy/logs.sh b/deploy/logs.sh new file mode 100755 index 00000000..6a578fa8 --- /dev/null +++ b/deploy/logs.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Deprecated full-docker log helper +# Usage: bash deploy/logs.sh [service-name] +# Canonical runtime logs: pm2 logs ... per docs/runbook.md +# ============================================================================= + +COMPOSE_FILE="docker-compose.prod.yml" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +cd "$PROJECT_DIR" + +SERVICE="${1:-}" + +if [ -n "$SERVICE" ]; then + echo "This script targets the non-canonical docker-compose.prod.yml path." + echo "Tailing logs for: $SERVICE" + docker compose -f "$COMPOSE_FILE" logs -f --tail=100 "$SERVICE" +else + echo "This script targets the non-canonical docker-compose.prod.yml path." + echo "Tailing logs for all services" + docker compose -f "$COMPOSE_FILE" logs -f --tail=50 +fi diff --git a/deploy/nginx/sites.conf b/deploy/nginx/sites.conf new file mode 100644 index 00000000..5698622a --- /dev/null +++ b/deploy/nginx/sites.conf @@ -0,0 +1,136 @@ +# ============================================================================= +# Nginx Configuration for Gitgram + AITimeMachine +# Behind Cloudflare Free Tier (Full Strict SSL) +# ============================================================================= + +# Cloudflare IP ranges for real IP restoration +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/13; +set_real_ip_from 104.24.0.0/14; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +# IPv6 +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; +real_ip_header CF-Connecting-IP; + +# HTTP -> HTTPS redirect +server { + listen 80; + listen [::]:80; + server_name gitgram.app www.gitgram.app timemachine.gitgram.app timemachine-api.gitgram.app; + return 301 https://$host$request_uri; +} + +# ============================================================================= +# gitgram.app — Gitgram Web + Backend +# ============================================================================= +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name gitgram.app www.gitgram.app; + + # Cloudflare Origin Certificate + ssl_certificate /etc/ssl/cloudflare/gitgram.app.pem; + ssl_certificate_key /etc/ssl/cloudflare/gitgram.app.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # Proxy headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # API routes -> Gitgram Backend (port 4000) + location /api/ { + proxy_pass http://127.0.0.1:4000; + proxy_http_version 1.1; + + # File uploads + client_max_body_size 100M; + } + + # WebSocket (Socket.IO) -> Gitgram Backend + location /socket.io/ { + proxy_pass http://127.0.0.1:4000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + } + + # Everything else -> Gitgram Web (port 3000) + location / { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + } +} + +# ============================================================================= +# timemachine.gitgram.app — AITM Website +# ============================================================================= +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name timemachine.gitgram.app; + + # Cloudflare Origin Certificate (reuses gitgram.app wildcard) + ssl_certificate /etc/ssl/cloudflare/gitgram.app.pem; + ssl_certificate_key /etc/ssl/cloudflare/gitgram.app.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + location / { + proxy_pass http://127.0.0.1:3002; + proxy_http_version 1.1; + } +} + +# ============================================================================= +# timemachine-api.gitgram.app — AITM API Server +# ============================================================================= +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name timemachine-api.gitgram.app; + + # Cloudflare Origin Certificate (reuses gitgram.app wildcard) + ssl_certificate /etc/ssl/cloudflare/gitgram.app.pem; + ssl_certificate_key /etc/ssl/cloudflare/gitgram.app.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + client_max_body_size 50M; + + location / { + proxy_pass http://127.0.0.1:3001; + proxy_http_version 1.1; + } +} diff --git a/deploy/setup-hetzner.sh b/deploy/setup-hetzner.sh new file mode 100755 index 00000000..c9e02127 --- /dev/null +++ b/deploy/setup-hetzner.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# Hetzner VPS bootstrap helper +# Run as root on a fresh Ubuntu 22.04+ VPS +# This is not the full launch procedure; use docs/runbook.md for launch. +# ============================================================================= + +echo "=========================================" +echo " Hetzner VPS Bootstrap for Gitgram + AITM" +echo "=========================================" + +# --- System Updates --- +echo "[1/7] Updating system packages..." +apt-get update && apt-get upgrade -y + +# --- Install Node 20 + PM2 --- +echo "[2/7] Installing Node 20 runtime and PM2..." +apt-get install -y curl ca-certificates gnupg build-essential + +if ! command -v node &> /dev/null || ! node -v | grep -q '^v20\.'; then + mkdir -p /etc/apt/keyrings + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ + | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" \ + > /etc/apt/sources.list.d/nodesource.list + apt-get update + apt-get install -y nodejs +fi + +npm install -g pm2 + +# --- Install Nginx --- +echo "[3/7] Installing Nginx..." +apt-get install -y nginx +systemctl enable nginx + +# --- Configure UFW Firewall --- +echo "[4/7] Configuring firewall..." +apt-get install -y ufw +ufw default deny incoming +ufw default allow outgoing +ufw allow 22/tcp # SSH +ufw allow 80/tcp # HTTP (redirect to HTTPS) +ufw allow 443/tcp # HTTPS +echo "y" | ufw enable +ufw status + +# --- Create deploy user --- +echo "[5/7] Creating deploy user..." +if ! id "deploy" &>/dev/null; then + useradd -m -s /bin/bash deploy + echo " Created user 'deploy'. Set a password with: passwd deploy" + echo " Add SSH key: ssh-copy-id deploy@<server-ip>" +else + echo " User 'deploy' already exists." +fi + +# --- Setup Nginx config --- +echo "[6/7] Installing Nginx configuration..." +mkdir -p /etc/ssl/cloudflare + +# Copy the nginx config +if [ -f "$(dirname "$0")/nginx/sites.conf" ]; then + cp "$(dirname "$0")/nginx/sites.conf" /etc/nginx/conf.d/sites.conf + echo " Nginx config installed to /etc/nginx/conf.d/sites.conf" +else + echo " WARNING: nginx/sites.conf not found. Copy it manually." +fi + +# --- Create project directory & env files --- +echo "[7/7] Setting up project directory..." +PROJECT_DIR="/home/deploy/gitgram-ai" +BOOTSTRAP_ENV_DIR="$PROJECT_DIR/bootstrap-env" +mkdir -p "$PROJECT_DIR" +mkdir -p "$BOOTSTRAP_ENV_DIR" + +# Copy env templates if they exist +for env_file in .env.postgres.example .env.gitgram.example .env.gitgram-web.example .env.aitm-api.example .env.aitm-website.example; do + src="$(dirname "$0")/../$env_file" + dest_name="${env_file%.example}" + if [ -f "$src" ] && [ ! -f "$BOOTSTRAP_ENV_DIR/$dest_name" ]; then + cp "$src" "$BOOTSTRAP_ENV_DIR/$dest_name" + echo " Staged $dest_name in $BOOTSTRAP_ENV_DIR" + fi +done + +chown -R deploy:deploy "$PROJECT_DIR" + +# --- Print next steps --- +echo "" +echo "=========================================" +echo " Setup Complete!" +echo "=========================================" +echo "" +echo "NEXT STEPS:" +echo "" +echo "1. Upload Cloudflare Origin Certificates:" +echo " scp gitgram.app.pem root@<server>:/etc/ssl/cloudflare/" +echo " scp gitgram.app.key root@<server>:/etc/ssl/cloudflare/" +echo " (gitgram.app wildcard cert covers timemachine.gitgram.app subdomains)" +echo "" +echo "2. Configure Cloudflare DNS (A records):" +echo " gitgram.app -> <server-ip> (Proxied)" +echo " www.gitgram.app -> <server-ip> (Proxied)" +echo " timemachine.gitgram.app -> <server-ip> (Proxied)" +echo " timemachine-api.gitgram.app -> <server-ip> (Proxied)" +echo "" +echo "3. Set Cloudflare SSL/TLS mode to 'Full (Strict)'" +echo "" +echo "4. Edit environment files in $PROJECT_DIR:" +echo " Templates were staged in $BOOTSTRAP_ENV_DIR" +echo " Copy them into the repo root after cloning" +echo "" +echo "5. Clone repo and continue with the manual runbook:" +echo " su - deploy" +echo " cd $PROJECT_DIR" +echo " git clone <repo-url> ." +echo " cp bootstrap-env/.env.gitgram ." +echo " cp bootstrap-env/.env.gitgram-web ." +echo " cp bootstrap-env/.env.aitm-api ." +echo " cp bootstrap-env/.env.aitm-website . # optional" +echo " less docs/runbook.md" +echo " npm run verify:launch" +echo " bash scripts/deploy-launch.sh staging --gitgram-only" +echo " # Then deploy AITM and run the public verification script" +echo "" +echo "6. Test nginx config:" +echo " nginx -t && systemctl reload nginx" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..c44163c8 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,191 @@ +services: + # ========================================================================= + # Infrastructure + # ========================================================================= + postgres: + image: postgres:16-alpine + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + environment: + POSTGRES_USER: ${POSTGRES_USER:-gitgram} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB:-gitgram} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-gitgram}"] + interval: 10s + timeout: 5s + retries: 5 + deploy: + resources: + limits: + memory: 512M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + redis: + image: redis:7-alpine + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + deploy: + resources: + limits: + memory: 256M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + # ========================================================================= + # Gitgram + # ========================================================================= + gitgram-backend: + build: + context: ./gitgram/backend + dockerfile: Dockerfile + restart: unless-stopped + ports: + - "127.0.0.1:4000:4000" + env_file: + - .env.gitgram + environment: + NODE_ENV: production + PORT: 4000 + DATABASE_URL: postgresql://${POSTGRES_USER:-gitgram}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-gitgram} + REDIS_URL: redis://redis:6379 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:4000/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + deploy: + resources: + limits: + memory: 512M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + gitgram-web: + build: + context: ./gitgram/web + dockerfile: Dockerfile + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + env_file: + - .env.gitgram-web + environment: + NODE_ENV: production + PORT: 3000 + depends_on: + gitgram-backend: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + deploy: + resources: + limits: + memory: 256M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + # ========================================================================= + # AITimeMachine + # ========================================================================= + aitm-api: + build: + context: ./aitimemachine/api-server + dockerfile: Dockerfile + restart: unless-stopped + ports: + - "127.0.0.1:3001:3001" + env_file: + - .env.aitm-api + environment: + NODE_ENV: production + PORT: 3001 + # DATABASE_URL is intentionally provided by .env.aitm-api (or the host environment), + # so production can use Neon without compose overriding it. + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3001/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + deploy: + resources: + limits: + memory: 512M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + aitm-website: + build: + context: ./aitimemachine/website + dockerfile: Dockerfile + restart: unless-stopped + ports: + - "127.0.0.1:3002:3002" + env_file: + - .env.aitm-website + environment: + NODE_ENV: production + PORT: 3002 + depends_on: + aitm-api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3002/"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + deploy: + resources: + limits: + memory: 256M + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +volumes: + postgres_data: + redis_data: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..54987043 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,338 @@ +# ============================================================================= +# Local Development — Docker Compose +# ============================================================================= +# Runs infrastructure + all application services for local development. +# +# Usage: +# docker compose up # start everything +# docker compose up postgres redis # infrastructure only +# docker compose --profile tools up # include GitNexus + Context7 +# docker compose down -v # tear down and remove volumes +# +# Services (default profile): +# postgres - Shared PostgreSQL 16 (port 5432) +# redis - Shared Redis 7 (port 6379) +# gitgram-backend - Gitgram Express API (port 4000) +# gitgram-web - Gitgram Next.js UI (port 3000) +# aitm-api - AITM REST API (port 3001) +# aitm-website - AITM Next.js UI (port 3002) +# aitm-mcp - AITM MCP server (port 3003) +# +# Services (tools profile): +# gitnexus - Code intelligence MCP (port 3010) +# context7 - Live library docs MCP (port 3100) +# +# Notes: +# - AITM can point to Neon or local Postgres; local postgres is the default. +# - aitm-mcp is an stdio/HTTP MCP server; adjust port if you expose it. +# - All secrets in environment blocks are dev-only placeholders — never use +# these values in production. +# ============================================================================= + +networks: + gitgram-net: + driver: bridge + +volumes: + postgres_data: + driver: local + redis_data: + driver: local + gitgram_uploads: + driver: local + gitgram_logs: + driver: local + +services: + + # --------------------------------------------------------------------------- + # Infrastructure + # --------------------------------------------------------------------------- + + postgres: + image: postgres:16-alpine + container_name: gitgram-postgres + networks: + - gitgram-net + environment: + POSTGRES_USER: gitgram + POSTGRES_PASSWORD: gitgram_dev + POSTGRES_DB: gitgram + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + # Optional init scripts: place *.sql or *.sh files here to seed the DB. + # - ./deploy/postgres/init:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U gitgram -d gitgram"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: gitgram-redis + networks: + - gitgram-net + ports: + - "6379:6379" + volumes: + - redis_data:/data + # Persistence (AOF) + sensible memory cap for local dev. + command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Gitgram Backend (Express API — Node 20) + # --------------------------------------------------------------------------- + + gitgram-backend: + build: + context: ./gitgram/backend + dockerfile: Dockerfile + container_name: gitgram-backend + networks: + - gitgram-net + ports: + - "4000:4000" + env_file: + - path: .env.gitgram + required: false # falls back to inline environment below + environment: + NODE_ENV: development + PORT: "4000" + DATABASE_URL: postgresql://gitgram:gitgram_dev@postgres:5432/gitgram + REDIS_URL: redis://redis:6379 + JWT_SECRET: local-dev-jwt-secret-not-for-production-use-replace-me + JWT_REFRESH_SECRET: local-dev-refresh-secret-not-for-production-replace + JWT_EXPIRES_IN: 15m + REFRESH_TOKEN_EXPIRES_IN: 7d + FRONTEND_URL: http://localhost:3000 + CORS_ORIGIN: http://localhost:3000,http://localhost:3001,http://localhost:3002,http://localhost:8081 + STORAGE_TYPE: local + UPLOAD_DIR: /app/uploads + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - gitgram_uploads:/app/uploads + - gitgram_logs:/app/logs + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:4000/health', (r) => { process.exit(r.statusCode === 200 ? 0 : 1) })\""] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + # Run migrations then start the compiled server. + command: sh -c "npx prisma migrate deploy && node dist/index.js" + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Gitgram Web (Next.js — Node 20) + # --------------------------------------------------------------------------- + + gitgram-web: + build: + context: ./gitgram/web + dockerfile: Dockerfile + container_name: gitgram-web + networks: + - gitgram-net + ports: + - "3000:3000" + env_file: + - path: .env.gitgram-web + required: false + environment: + NODE_ENV: development + PORT: "3000" + NEXT_PUBLIC_API_URL: http://localhost:4000/api + NEXT_PUBLIC_WS_URL: http://localhost:4000 + NEXT_PUBLIC_AITM_API_URL: http://localhost:3001/v1 + depends_on: + gitgram-backend: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3000', (r) => { process.exit(r.statusCode < 500 ? 0 : 1) })\""] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + # --------------------------------------------------------------------------- + # AITM API Server (Hono/Node — Node 20) + # --------------------------------------------------------------------------- + # Connects to local Postgres by default. Override DATABASE_URL in + # .env.aitm-api to point at Neon for hybrid dev. + + aitm-api: + build: + context: ./aitimemachine/api-server + dockerfile: Dockerfile + container_name: aitm-api + networks: + - gitgram-net + ports: + - "3001:3001" + env_file: + - path: .env.aitm-api + required: false + environment: + NODE_ENV: development + HOST: "0.0.0.0" + PORT: "3001" + DATABASE_URL: postgresql://gitgram:gitgram_dev@postgres:5432/gitgram + REDIS_URL: redis://redis:6379 + CORS_ORIGIN: http://localhost:3000,http://localhost:3002 + WEBSITE_URL: http://localhost:3002 + INTERNAL_SECRET: local-dev-internal-secret-replace-in-prod + GITGRAM_API_URL: http://gitgram-backend:4000 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3001/health', (r) => { process.exit(r.statusCode === 200 ? 0 : 1) })\""] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + # --------------------------------------------------------------------------- + # AITM Website (Next.js — Node 20) + # --------------------------------------------------------------------------- + + aitm-website: + build: + context: ./aitimemachine/website + dockerfile: Dockerfile + container_name: aitm-website + networks: + - gitgram-net + ports: + - "3002:3002" + env_file: + - path: .env.aitm-website + required: false + environment: + NODE_ENV: development + PORT: "3002" + NEXT_PUBLIC_API_URL: http://localhost:3001 + API_URL: http://aitm-api:3001 + INTERNAL_SECRET: local-dev-internal-secret-replace-in-prod + NEXT_PUBLIC_APP_URL: http://localhost:3002 + depends_on: + aitm-api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3002', (r) => { process.exit(r.statusCode < 500 ? 0 : 1) })\""] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + # --------------------------------------------------------------------------- + # AITM MCP Server (stdio / HTTP bridge — Node 20) + # --------------------------------------------------------------------------- + # The MCP server communicates over stdio by default; this container wraps it + # so Claude Code and other editors can reach it over HTTP on port 3003. + + aitm-mcp: + build: + context: ./aitimemachine/mcp + dockerfile: Dockerfile + container_name: aitm-mcp + networks: + - gitgram-net + ports: + - "3003:3003" + env_file: + - path: .env.aitm-api + required: false + environment: + NODE_ENV: development + PORT: "3003" + AITM_API_URL: http://aitm-api:3001 + GITNEXUS_URL: http://gitnexus:3010 + AITM_GITNEXUS_ENABLED: "0" + CONTEXT7_URL: http://context7:3100 + AITM_CONTEXT7_ENABLED: "0" + depends_on: + aitm-api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3003/health', (r) => { process.exit(r.statusCode < 500 ? 0 : 1) }).on('error', () => process.exit(0))\""] + interval: 20s + timeout: 5s + retries: 5 + start_period: 20s + restart: unless-stopped + + # --------------------------------------------------------------------------- + # GitNexus — code intelligence MCP server (profile: tools) + # --------------------------------------------------------------------------- + + gitnexus: + build: + context: ./aitimemachine/gitnexus + dockerfile: Dockerfile + container_name: gitnexus + networks: + - gitgram-net + ports: + - "3010:3010" + environment: + NODE_ENV: development + PORT: "3010" + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3010/health', (r) => { process.exit(r.statusCode < 500 ? 0 : 1) }).on('error', () => process.exit(0))\""] + interval: 20s + timeout: 5s + retries: 5 + start_period: 15s + profiles: + - tools + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Context7 — live library docs MCP server (profile: tools) + # --------------------------------------------------------------------------- + + context7: + image: node:20-alpine + container_name: context7 + working_dir: /app + networks: + - gitgram-net + volumes: + - ./aitimemachine/context7/packages/mcp:/app:ro + command: sh -c "npm install --if-present && node dist/index.js" + ports: + - "3100:3100" + environment: + NODE_ENV: development + PORT: "3100" + healthcheck: + test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:3100/health', (r) => { process.exit(r.statusCode < 500 ? 0 : 1) }).on('error', () => process.exit(0))\""] + interval: 20s + timeout: 5s + retries: 5 + start_period: 20s + profiles: + - tools + restart: unless-stopped diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 00000000..9e3804e2 --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,591 @@ +# AI Tools — Architecture & Reference + +This document is the single-source reference for every AI tool integrated into the +`gitgram-ai` monorepo. All tools live under `aitimemachine/` and are wired together +through the AITM MCP server. + +--- + +## Table of Contents + +1. [Architecture overview](#1-architecture-overview) +2. [Starting each tool](#2-starting-each-tool) +3. [Analyzing codebases with GitNexus](#3-analyzing-codebases-with-gitnexus) +4. [A2A protocol — how it works and how to register agents](#4-a2a-protocol) +5. [MCP tools reference](#5-mcp-tools-reference) +6. [Quick-start guide](#6-quick-start-guide) + +--- + +## 1. Architecture Overview + +``` +gitgram-ai/ +├── aitimemachine/ +│ ├── mcp/ # AITM MCP server — central hub, exposes all tools +│ ├── api-server/ # REST API for TimeMachine state (Drizzle/Neon) +│ ├── gitnexus/ # Graph-powered code intelligence, 17 MCP tools +│ ├── a2a/ # A2A protocol types & JSON-RPC helpers +│ ├── context7/packages/mcp/ # Context7 MCP — up-to-date library docs +│ ├── beads/ # Go CLI for distributed graph issue tracking (bd) +│ ├── pal-mcp-server/ # PAL MCP — multi-model AI orchestration +│ └── tools/ +│ ├── repo2txt/ # React/Vite UI: convert repos to LLM-readable text +│ └── repo-to-text/ # Python CLI: same conversion, CLI variant +└── scripts/ + ├── setup-aitm-tools.sh # One-shot build of all companion tools + └── start-all-dev.sh # Start core dev services with concurrently +``` + +### How the pieces connect + +``` +Claude Code / Cursor / any MCP client + │ + ▼ + ┌───────────────────────────────────────┐ + │ AITM MCP server │ + │ aitimemachine/mcp (port: stdio) │ + │ │ + │ Tool groups: │ + │ • tm_* — TimeMachine core │ + │ • gn_* proxy — GitNexus tools │ + │ • tm_docs_* — Context7 tools │ + │ • a2a_* — A2A agent tools │ + │ • before/after_action │ + └──────────┬──────────────┬────────────┘ + │ │ + ▼ ▼ + ┌──────────────┐ ┌──────────────────┐ + │ GitNexus │ │ Context7 MCP │ + │ :3001/msg │ │ :3100/msg │ + └──────────────┘ └──────────────────┘ +``` + +The AITM MCP server is the single entry point for AI clients. It embeds the +TimeMachine engine directly and proxies GitNexus and Context7 over HTTP/JSON-RPC. +PAL MCP and Beads run as independent services alongside it. + +### Tool inventory + +| Tool | Location | Language | Purpose | +|------|----------|----------|---------| +| AITM MCP server | `aitimemachine/mcp` | TypeScript | Central MCP hub; checkpoint, restore, session, file, worktree, S3, Docker capsule tools | +| GitNexus | `aitimemachine/gitnexus` | TypeScript | Code knowledge graph: query, impact, trace, rename, route/tool maps | +| PAL MCP Server | `aitimemachine/pal-mcp-server` | Python | Multi-model AI orchestration (chat, code review, debug, plan, consensus) | +| Context7 | `aitimemachine/context7/packages/mcp` | TypeScript | Up-to-date library documentation injected into LLM context | +| Beads (`bd`) | `aitimemachine/beads` | Go | Distributed graph issue tracker for AI agents (Dolt-powered) | +| A2A protocol | `aitimemachine/a2a` | TypeScript | JSON-RPC 2.0 agent-to-agent protocol types and SDK | +| repo2txt | `aitimemachine/tools/repo2txt` | TypeScript/React | UI to convert a GitHub repo or local directory to LLM-readable text | +| repo-to-text | `aitimemachine/tools/repo-to-text` | Python | CLI to convert a directory to structured XML/text for LLM prompts | + +--- + +## 2. Starting Each Tool + +All commands below are root-level npm scripts. Run them from the monorepo root +(`/Users/arhansubasi/gitgram-ai/` or wherever you cloned the repo). + +### AITM MCP server + +```bash +npm run dev:aitm-mcp +# Starts aitimemachine/mcp in watch mode (stdio transport) +``` + +Configure your MCP client to launch it via the `mcp.json` at +`aitimemachine/mcp/mcp.json`. The `aitm` server entry uses `node dist/index.js`. + +Environment variables recognised by the server: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `GITNEXUS_URL` | `http://localhost:3001` | Where to proxy GitNexus tool calls | +| `CONTEXT7_URL` | `http://localhost:3100` | Where to proxy Context7 tool calls | +| `AITM_A2A_AGENTS` | `[]` | JSON array of `{ name, url }` to register A2A agents | +| `AITM_EXPERIMENTAL_TOOLS` | `0` | Set to `1` to enable experimental tools | +| `DISABLED_TOOLS` | — | Comma-separated list of tool names to disable | + +### AITM API server + +```bash +npm run dev:aitm-api # standard dev mode +npm run dev:aitm-api:clean # dev mode with a clean database +``` + +Serves the TimeMachine REST API at `http://localhost:3001` by default. + +### GitNexus + +```bash +npm run dev:gitnexus # starts the GitNexus HTTP server at :3001 +npm run dev:gitnexus-web # starts the GitNexus web UI +``` + +GitNexus must have at least one repo indexed before it can answer queries. +See [Section 3](#3-analyzing-codebases-with-gitnexus) for indexing instructions. + +### PAL MCP Server + +```bash +npm run dev:pal-mcp +# Runs aitimemachine/pal-mcp-server/run-server.sh (activates Python venv, starts server) +``` + +Configure providers by creating a `.env` in `aitimemachine/pal-mcp-server/` based +on the `requirements.txt` instructions. At minimum set one of: +`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or `GEMINI_API_KEY`. + +### Context7 + +```bash +npm run dev:context7 +# Starts aitimemachine/context7/packages/mcp/dist/index.js +# (falls back to node directly if the package has no dev script) +``` + +Listens on port `3100` by default. No API key required for public libraries; +set `CONTEXT7_API_KEY` for higher rate limits and private repositories. + +### Beads (`bd`) + +```bash +npm run build:beads +# Compiles Go binary to aitimemachine/beads/bd +``` + +After building, use the binary directly: + +```bash +aitimemachine/beads/bd init # initialise in any project +aitimemachine/beads/bd setup claude # install Claude Code hooks +aitimemachine/beads/bd ready # list unblocked tasks +aitimemachine/beads/bd prime # print agent workflow context +``` + +### repo2txt (React/Vite UI) + +```bash +npm run dev:repo2txt +# Vite dev server for the repo2txt web UI +``` + +### repo-to-text (Python CLI) + +```bash +npm run dev:repo-to-text +# python -m repo_to_text from aitimemachine/tools/repo-to-text +``` + +Install the Python package once before first use: + +```bash +pip install repo-to-text +``` + +### Start all core services at once + +```bash +bash scripts/start-all-dev.sh +# Requires: npm install -g concurrently +# Starts: gitgram/backend (:3000), gitgram/web (:3002), aitimemachine/api-server (:3001) +``` + +Port overrides: + +```bash +GITGRAM_BACKEND_PORT=3000 GITGRAM_WEB_PORT=3002 AITM_API_PORT=3001 \ + bash scripts/start-all-dev.sh +``` + +--- + +## 3. Analyzing Codebases with GitNexus + +GitNexus builds a queryable knowledge graph (nodes = symbols, edges = calls/imports/ +inheritance) from a TypeScript or JavaScript codebase. You must index a repo before +the graph tools return useful results. + +### Using the convenience script + +```bash +# From monorepo root +bash scripts/analyze-with-gitnexus.sh <REPO> +``` + +Supported `<REPO>` aliases: + +| Alias | Path | +|-------|------| +| `gitgram-backend` | `gitgram/backend` | +| `gitgram-web` | `gitgram/web` | +| `aitm-api` | `aitimemachine/api-server` | +| `aitm-mcp` | `aitimemachine/mcp` | + +The script auto-builds the GitNexus CLI if `dist/cli/index.js` is missing, then +calls `gitnexus analyze <path> --name <name>`. + +### Using the CLI directly + +```bash +cd aitimemachine/gitnexus + +# Index a repo (run from its root, or pass --root) +node dist/cli/index.js analyze /absolute/path/to/repo --name my-repo + +# Start the query server +npm run serve # or: node dist/cli/index.js server +# Default: http://localhost:3001 + +# Query via CLI +node dist/cli/index.js query "authentication flow" +node dist/cli/index.js context validateUser +node dist/cli/index.js impact AuthService +``` + +### Using GitNexus through the AITM MCP server + +Once the GitNexus server is running and `GITNEXUS_URL` points at it, all `gn_*` +tools become available inside the AITM MCP server automatically. Example in Claude +Code: + +``` +list all indexed repos → calls gn_list_repos +query "how does checkout work" → calls gn_query +impact analysis for PaymentService → calls gn_impact +``` + +### Using GitNexus with MCP directly (without AITM) + +```bash +# Claude Code +claude mcp add gitnexus -- npx -y gitnexus@latest mcp + +# or add to .mcp.json +{ + "mcpServers": { + "gitnexus": { + "command": "node", + "args": ["aitimemachine/gitnexus/dist/cli/index.js", "mcp"] + } + } +} +``` + +--- + +## 4. A2A Protocol + +The A2A (Agent-to-Agent) protocol is a Google-authored open standard for agents to +delegate tasks to other agents over HTTP using JSON-RPC 2.0. The implementation in +`aitimemachine/a2a` provides TypeScript types and the `a2a_*` MCP tools. + +### How it works + +1. Every A2A agent exposes a `POST /a2a` endpoint that accepts JSON-RPC 2.0 requests. +2. The caller sends a `tasks/send` request with a `message` (array of `Part` objects). +3. The agent executes the task and returns a `Task` object with a `status` field + (`submitted`, `working`, `completed`, `failed`, or `canceled`). +4. The caller can poll with `tasks/get` using the same `id` until the task completes. + +``` +Caller Agent + │ │ + │ POST /a2a │ + │ { method: "tasks/send", │ + │ params: { id, message } } ──► │ + │ │ (executes) + │ ◄─── { result: Task } │ + │ │ + │ POST /a2a │ + │ { method: "tasks/get", │ + │ params: { id } } ──► │ + │ ◄─── { result: Task{status} } │ +``` + +### Registering agents + +Set the `AITM_A2A_AGENTS` environment variable before starting the AITM MCP server. +It must be a JSON array of `{ name, url }` objects: + +```bash +export AITM_A2A_AGENTS='[ + { "name": "code-reviewer", "url": "http://localhost:4001" }, + { "name": "planner", "url": "http://localhost:4002" } +]' +npm run dev:aitm-mcp +``` + +Or set it in `aitimemachine/mcp/mcp.json`: + +```json +{ + "mcpServers": { + "aitm": { + "command": "node", + "args": ["dist/index.js"], + "env": { + "AITM_A2A_AGENTS": "[{\"name\":\"code-reviewer\",\"url\":\"http://localhost:4001\"}]" + } + } + } +} +``` + +### Using A2A tools inside Claude Code + +``` +# List registered agents +a2a_list_agents + +# Dispatch a task to an agent +a2a_send_task + agentUrl: "http://localhost:4001" + taskId: "review-pr-42" + message: "Review the auth module for security issues" + +# Poll for completion +a2a_get_task + agentUrl: "http://localhost:4001" + taskId: "review-pr-42" +``` + +### Building an A2A-compatible agent + +Any HTTP server that implements `POST /a2a` accepting JSON-RPC 2.0 qualifies. Import +the shared types from the monorepo package: + +```typescript +import type { Task, TaskStatus, Part, TextPart } from 'aitimemachine/a2a'; +``` + +The `aitimemachine/a2a` package also exports `AGENT_CARD_PATH` (`/.well-known/agent.json`) +and the `AgentCard` schema for agent discovery. + +--- + +## 5. MCP Tools Reference + +The AITM MCP server registers tools in eight groups. All tools are available to any +MCP-compatible client pointing at `aitimemachine/mcp`. + +### 5.1 TimeMachine core tools + +| Tool | Description | +|------|-------------| +| `tm_project_init` | Initialize or register a project for time tracking | +| `tm_set_project` | Set the current working project | +| `tm_checkpoint_create` | Create a checkpoint (snapshot of current state) | +| `tm_checkpoint_list` | List all checkpoints/timepoints | +| `tm_checkpoint_restore` | Restore project to a checkpoint | +| `tm_checkpoint_preview` | Preview what restore would change | +| `tm_diff` | Show diff between two checkpoints or checkpoint and current state | +| `tm_diff_checkpoints` | Detailed diff between two checkpoints (files + chat + terminal) | +| `tm_event_append` | Append an event to the project timeline | +| `tm_timeline` | Get timeline of events for a project | +| `tm_search` | Search across conversations and checkpoints | +| `tm_find_restore` | Search conversations and find the best checkpoint to restore | +| `tm_log_message` | Log a conversation message | +| `tm_stats` | Get TimeMachine statistics | +| `tm_tag` | Tag a checkpoint with a label | +| `tm_ready` | Check if the TimeMachine daemon is ready | +| `before_action` | Call before making file changes — creates an automatic checkpoint | +| `after_action` | Call after making file changes — records what was done | + +### 5.2 Session tools + +| Tool | Description | +|------|-------------| +| `tm_session_capture` | Capture current state as a session (used by the daemon) | +| `tm_session_list` | List all captured sessions | +| `tm_bisect` | Binary search to find when a test started failing | + +### 5.3 Integration/environment tools + +| Tool | Description | +|------|-------------| +| `tm_container_state` | Get Docker/container state for a project | +| `tm_database_state` | Get database state for a project | +| `tm_gemini_sessions` | List Gemini CLI sessions | +| `tm_git_state` | Get git state for a project | +| `tm_env_state` | Get environment state (runtimes, package managers, dependencies) | +| `tm_aider_sessions` | List Aider sessions | +| `tm_cursor_workspaces` | List Cursor workspaces | +| `tm_claude_projects` | List Claude Code projects | +| `tm_claude_conversations` | Get Claude Code conversations for a project | + +### 5.4 Storage / S3 / Docker capsule tools + +| Tool | Description | +|------|-------------| +| `tm_s3_config` | Configure S3 backup credentials | +| `tm_s3_backup` | Backup all TimeMachine data to S3 | +| `tm_s3_restore` | Restore all TimeMachine data from S3 | +| `tm_capsule_create` | Create a Docker capsule for deterministic execution | +| `tm_capsule_exec` | Execute a command inside a capsule container | +| `tm_capsule_stop` | Stop and remove a capsule container | + +### 5.5 File tools + +| Tool | Description | +|------|-------------| +| `tm_restore_files` | Restore specific files from a session (glob patterns supported) | +| `tm_get_file_at` | Get file content at a specific checkpoint | +| `tm_file_history` | Get history of a file across checkpoints | +| `tm_prompt_debug_regression` | Generate a prompt to debug regressions between two checkpoints | +| `tm_prompt_write_postmortem` | Generate a prompt to write an incident postmortem | +| `tm_prompt_generate_repro_steps` | Generate reproduction steps from checkpoint history | +| `tm_repo_to_text` | Convert a repository or directory to a single text block for LLM context | + +### 5.6 Worktree tools + +| Tool | Description | +|------|-------------| +| `tm_worktree_list` | List active AITM worktrees for the current project | +| `tm_worktree_restore` | Restore a worktree to a checkpoint | +| `tm_worktree_cleanup` | Remove all AITM worktrees for the current project | + +### 5.7 GitNexus proxy tools (prefix: none, routed to GitNexus server) + +These tools forward calls to the GitNexus server at `GITNEXUS_URL`. + +| Tool | Description | +|------|-------------| +| `list_repos` | List indexed repositories (paginated) | +| `query` | Query the code knowledge graph for execution flows related to a concept | +| `cypher` | Execute a Cypher query directly against the knowledge graph | +| `context` | 360-degree view of a single symbol: incoming/outgoing references, callers, callees | +| `detect_changes` | Analyze uncommitted git changes and find affected execution flows | +| `check` | Run read-only structural checks (circular imports, etc.) | +| `rename` | Multi-file coordinated rename using the knowledge graph | +| `impact` | Analyze the blast radius of changing a code symbol | +| `explain` | Explain persisted taint findings from `gitnexus analyze --pdg` | +| `pdg_query` | Query the Program Dependence Graph (control + data flow) | +| `route_map` | Show API route mappings: which components fetch which endpoints | +| `tool_map` | Show MCP/RPC tool definitions and their handlers | +| `shape_check` | Check API response shapes against their consumers | +| `api_impact` | Pre-change impact report for an API route handler | +| `group_list` | List configured repository groups | +| `group_sync` | Rebuild the Contract Registry for a group | +| `trace` | Find the shortest directed path between two symbols | + +### 5.8 Context7 documentation tools + +| Tool | Description | +|------|-------------| +| `tm_docs_resolve` | Resolve a library name (e.g. `"react"`) to its Context7 library ID | +| `tm_docs_query` | Query documentation for a resolved Context7 library ID | +| `tm_docs_for_imports` | Detect imports in a source file and fetch docs for each library | + +### 5.9 A2A tools + +| Tool | Description | +|------|-------------| +| `a2a_send_task` | Send a task to an A2A agent via `tasks/send` | +| `a2a_get_task` | Get the current status of an A2A task via `tasks/get` | +| `a2a_list_agents` | List agents registered in `AITM_A2A_AGENTS` | + +--- + +## 6. Quick-Start Guide + +### Step 1 — Build companion tools + +Run this once after cloning the repo or pulling major updates: + +```bash +bash scripts/setup-aitm-tools.sh +``` + +This script builds four tools in sequence (a failure in one does not abort the others): + +| Tool | What it does | +|------|--------------| +| `beads` | `go build ./cmd/bd/...` → binary at `aitimemachine/beads/bd` | +| `gitnexus` | `npm install && npm run build` in `aitimemachine/gitnexus/gitnexus` | +| `pal-mcp-server` | Creates Python venv and installs `requirements.txt` | +| `context7-mcp` | `pnpm/npm install && build` in `aitimemachine/context7/packages/mcp` | + +Prerequisites: +- Node.js 18+ and npm +- Python 3.10+ (for PAL MCP server) +- Go 1.21+ (for Beads; skipped gracefully if missing) +- pnpm (preferred for context7; falls back to npm) + +### Step 2 — Start core dev services + +```bash +bash scripts/start-all-dev.sh +``` + +Services started: + +| Service | Default port | +|---------|-------------| +| `gitgram/backend` | 3000 | +| `aitimemachine/api-server` | 3001 | +| `gitgram/web` | 3002 | + +Requires `concurrently` globally installed (`npm install -g concurrently`). Without it +the script prints manual commands for three separate terminals. + +### Step 3 — Start the AI tool layer + +In a separate terminal (or add to your `concurrently` invocation): + +```bash +# AITM MCP server (required — hub for all AI tools) +npm run dev:aitm-mcp + +# GitNexus server (required for gn_* tools) +npm run dev:gitnexus + +# Context7 (optional — for tm_docs_* tools) +npm run dev:context7 + +# PAL MCP (optional — for multi-model AI) +npm run dev:pal-mcp +``` + +### Step 4 — Index your codebase with GitNexus + +```bash +bash scripts/analyze-with-gitnexus.sh gitgram-backend +bash scripts/analyze-with-gitnexus.sh aitm-mcp +``` + +### Step 5 — Point your MCP client at the AITM server + +Add the following to your MCP client config (e.g. `~/.claude/mcp.json` or Claude +Code's `claude mcp add` command): + +```json +{ + "mcpServers": { + "aitm": { + "command": "node", + "args": ["/absolute/path/to/aitimemachine/mcp/dist/index.js"], + "env": { + "GITNEXUS_URL": "http://localhost:3001", + "AITM_A2A_AGENTS": "[]" + } + } + } +} +``` + +Or use the pre-built `mcp.json` at `aitimemachine/mcp/mcp.json`, which also defines +standalone entries for `gitnexus`, `context7`, and `pal`. + +### Step 6 — Verify everything is running + +```bash +# Check the AITM API server +curl http://localhost:3001/health + +# Check the GitNexus server +curl http://localhost:3001/health + +# List indexed repos (from inside Claude Code) +# > list_repos +``` diff --git a/docs/can be fork.md b/docs/can be fork.md new file mode 100644 index 00000000..ff0cedf2 --- /dev/null +++ b/docs/can be fork.md @@ -0,0 +1,59 @@ +# Actionable Reference Clones + +Last updated: 2026-06-21 + +This file replaces the old starred-repos dump. + +Purpose: +- keep a small set of upstream reference repos inside this workspace +- clone only repos that map to current Gitgram or AITM work +- avoid cloning unrelated agent/social projects just because they look interesting + +Clone root: +- `/Users/arhansubasi/gitgram-ai/external` + +## Cloned Now + +| Repo | Local path | Why it matters now | Status | +|---|---|---|---| +| `abinthomasonline/repo2txt` | `external/repo2txt` | Direct reference for repo-to-text ingest/export UX and URL handling patterns. | `cloned 2026-05-12` | +| `kirill-markin/repo-to-text` | `external/repo-to-text` | Simpler alternative repo-to-text implementation to compare parsing, file selection, and output shaping. | `cloned 2026-05-12` | +| `a2aproject/a2a-js` | `external/a2a-js` | Official A2A SDK. Needed to align AITM transport and agent-card behavior with the spec instead of maintaining only hand-rolled protocol types. | `cloned 2026-05-12` | +| `upstash/context7` | `external/context7` | Reference for production MCP HTTP patterns, transport wiring, and request-scoped context handling. | `cloned 2026-05-12` | +| `BeehiveInnovations/pal-mcp-server` | `external/pal-mcp-server` | Reference for MCP tool organization, annotations, output envelopes, and tool disabling patterns. | `cloned 2026-05-12` | +| `abhigyanpatwari/GitNexus` | `external/gitnexus` | Client-side code intelligence engine — zero-server knowledge graph + Graph RAG agent over any repo. Reference for Gitgram repo explore/visualisation surface and AITM code-context indexing. | `cloned 2026-06-21` | +| `gastownhall/beads` | `external/beads` | Steve Yegge's git-native DAG issue tracker designed for AI agents. Reference for AITM task/issue tracking integration and multi-agent coordination patterns. `bd ready` shows unblocked work with dependency DAGs and hash-based IDs. | `cloned 2026-06-21` | +| `codinginflow/nextjs-15-social-media-app` | `external/nextjs-15-social-media-app` | Exact same stack as Gitgram (Next.js 15 + Prisma + PG). Prisma schema maps 1:1 to Gitgram. Reference for infinite scroll, optimistic updates, and notification patterns. | `cloned 2026-06-21` | +| `leandronorcio/munia` | `external/munia` | Most complete open-source social features: GitHub OAuth, @mentions, hashtags, nested comments, drag-drop media, S3 uploads, accessibility. | `cloned 2026-06-21` | +| `TomDoesTech/t3-twitter-clone` | `external/t3-twitter-clone` | Only open-source Twitter clone with an Express backend branch — matches Gitgram's Express backend. Type-safe tRPC patterns. | `cloned 2026-06-21` | +| `karanpratapsingh/Proximity` | `external/proximity` | Best React Native social app with Prisma. Real-time messaging via GraphQL subscriptions. Reference only (GPL-3). | `cloned 2026-06-21` | +| `amirfakhrullah/gh-social` | `external/gh-social` | GitHub Social Platform — the closest open-source concept to Gitgram. Syncs GitHub follows/stars to a social feed. Next.js + tRPC + Clerk + Drizzle + Neon. | `cloned 2026-06-21` | +| `bluesky-social/social-app` | `external/bluesky-social-app` | Production-scale feed algorithms, notifications, and moderation patterns. Architecture reference for Gitgram at scale. | `cloned 2026-06-21` | + +## Watch Later + +| Repo | Why not cloning now | +|---|---| +| `browserbase/stagehand` | Browser automation is not on the current launch-critical path. | +| `daytonaio/daytona` | Infra-heavy and not needed for the current single-host launch. | +| `mem0ai/mem0` | Memory layer work is post-launch, not a deploy blocker. | +| `vercel-labs/open-agents` | Broad cloud-agent template, but not a direct gap in the current repos. | +| `badlogic/sitegeist` | Useful browser-collaboration reference, but not tied to current shipping tasks. | +| `agentgateway/agentgateway` | Worth reviewing later for A2A/MCP proxying, but not needed before the first launch. | +| `GongRzhe/A2A-MCP-Server` | Useful bridge reference after the official A2A SDK pass. | +| `Open-ACP/OpenACP` | Messaging-channel expansion is post-launch work. | + +## Current Adoption Targets + +| Reference | Product | Current status | What is already used | Next planned use in project | +|---|---|---|---|---| +| `repo2txt` + `repo-to-text` | Gitgram | `adopted` | Gitgram now ships a real repo-to-text export flow at `/api/v1/repositories/:owner/:name/export/text`, a public short-path text view at `/:owner/:name/text`, a raw plain-text shortcut at `/:owner/:name/text/raw`, queryable export sizing via `maxFiles` / `maxFileBytes`, and direct `Text` / `Raw` entry points from the repository page plus discovery cards in search, explore, and trending. | No launch-critical follow-up remains. Reuse only if Gitgram adds deeper compose/share/import workflows around repo text. | +| `a2a-js` | AITM MCP | `adopted` | AITM now uses an `a2a-js`-derived contract layer in `aitimemachine/mcp/src/lib/a2a-contract.ts`, a canonical `message/send` JSON-RPC flow in `aitimemachine/mcp/src/http-transport.ts`, spec-shaped task responses, protocol-versioned agent cards, and a repeatable smoke gate in `aitimemachine/scripts/a2a-smoke-test.sh`. | No launch-critical follow-up remains beyond future expansion into real streaming or additional A2A task methods if the product needs them. | +| `context7` | AITM MCP | `adopted` | AITM already exposes streamable HTTP transport and now also uses request-scoped HTTP context, stable `X-Request-Id` response headers, structured parser/protocol error mapping, per-request logging around `/mcp`, `/a2a`, `/.well-known/agent-card.json`, and `/health`, plus nested request-ID propagation from A2A into the internal `/mcp` bridge. | No launch-critical follow-up remains. Revisit only if AITM later needs deeper user/auth context propagation across remote downstream calls. | +| `pal-mcp-server` | AITM MCP | `adopted` | AITM MCP now enforces a real tool registry with stable-vs-experimental exposure, `DISABLED_TOOLS` env support, aligned A2A/HTTP capability reporting, category-based tool registration modules, shared tool-annotation helpers, shared tool response helpers, and declarative `createToolRegistrar` / `defineTool` registration helpers instead of repeated `server.tool(...)` scaffolding. | No launch-critical follow-up remains. Revisit only if the tool surface grows enough to justify deeper output-schema or capability-family generation. | + +## Refresh Policy + +- Keep clones shallow unless a deeper history is required. +- Treat `external/*` as upstream references, not as code to edit directly for product work. +- When a reference repo becomes genuinely integrated, copy or adapt the needed pattern into `gitgram` or `aitimemachine` with explicit attribution in the relevant PR or commit. diff --git a/docs/deploy-pipeline.md b/docs/deploy-pipeline.md new file mode 100644 index 00000000..0cf0d2ea --- /dev/null +++ b/docs/deploy-pipeline.md @@ -0,0 +1,95 @@ +# Deployment Pipeline Status + +Last updated: 2026-05-09 + +## Canonical Launch Contract + +Launch is manual-first. + +- Canonical deploy path: `docs/runbook.md` +- Canonical control plane: `pm2` +- Canonical rollback model: operator-managed rollback to a previous checkout or release directory, plus database restore from backup when migrations are involved + +No GitHub Actions workflow is required for launch readiness. + +## Workflow Inventory + +### Shared CI + +Current root workflows: + +- `.github/workflows/ci.yml` +- `.github/workflows/tests.yml` +- `.github/workflows/integration-tests.yml` +- `.github/workflows/smoke-tests.yml` +- `.github/workflows/migrations.yml` +- `.github/workflows/openapi-validation.yml` +- `.github/workflows/verify-production-db.yml` + +These are build and verification workflows, not the manual launch source of truth. + +### Gitgram Automation + +Workflow files present in the repo: + +- `gitgram/.github/workflows/deploy.yml` +- `gitgram/.github/workflows/deploy-production.yml` +- `gitgram/.github/workflows/deploy-staging.yml` +- `gitgram/.github/workflows/production-deploy.yml` + +Status: + +- `deploy.yml` is the newest automation candidate. +- `deploy-production.yml`, `deploy-staging.yml`, and `production-deploy.yml` are legacy or incomplete variants. +- None of the Gitgram deploy workflows should be treated as required for launch until they match the manual PM2 process-control path. + +### AITM Automation + +Workflow files present in the repo: + +- `aitimemachine/.github/workflows/deploy.yml` +- `aitimemachine/.github/workflows/ci.yml` +- `aitimemachine/.github/workflows/pr-check.yml` + +Status: + +- `deploy.yml` is an automation candidate, not the launch source of truth. +- `ci.yml` currently contains a tag-driven npm release step for `@gitgram/timemachine`. +- The MCP package publish is still manual. + +## Automation-Only Variables + +These names exist in workflow files but are not part of the manual launch contract: + +- `CURRENT_DEPLOY_TAG` +- `GITGRAM_DATABASE_URL` +- `GITGRAM_REDIS_URL` +- `AITM_DATABASE_URL` +- other workflow-scoped `GITGRAM_*` / `AITM_*` secrets + +They are acceptable for future CI/CD, but the operator-managed launch path uses the root `.env.*` files and runtime process restarts described in `docs/runbook.md`. + +## Rollback Policy + +Rollback is manual-first. + +Trigger rollback when any of these happen: + +1. A service health gate fails after restart. +2. A smoke test fails. +3. A migration partially applies or produces broken behavior. +4. A package publish is broken after public verification. + +Manual rollback actions: + +1. Revert to the previous known-good tag, commit, or release directory. +2. Rebuild the affected service. +3. Restart the affected PM2 process with `--update-env`. +4. Re-run health gates. +5. Restore the database from the pre-deploy backup when schema state is unsafe. + +For npm packages: + +1. Deprecate the bad version. +2. Bump the version. +3. Rebuild and republish. diff --git a/docs/environment-matrix.md b/docs/environment-matrix.md new file mode 100644 index 00000000..45ea8393 --- /dev/null +++ b/docs/environment-matrix.md @@ -0,0 +1,172 @@ +# Environment Matrix + +Last updated: 2026-05-09 + +This is the canonical launch-time environment contract for the manual single-host deployment path. + +Rules: + +- Runtime env files live at repo root: `.env.gitgram`, `.env.gitgram-web`, `.env.aitm-api`, optional `.env.aitm-website`. +- Service-local `.env.example` files remain the code-level reference for development. +- `scripts/validate-prod-secrets.sh` accepts both unprefixed runtime names and `GITGRAM_*` / `AITM_*` aliases for CI-style validation. + +## Gitgram Backend + +Required before first launch: + +- `DATABASE_URL` +- `REDIS_URL` +- `JWT_SECRET` +- `JWT_REFRESH_SECRET` +- `FRONTEND_URL` +- `CORS_ORIGIN` +- `GITHUB_CLIENT_ID` +- `GITHUB_CLIENT_SECRET` +- `POLAR_ACCESS_TOKEN` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PREMIUM_PRODUCT_ID` +- `POLAR_CREATOR_PRODUCT_ID` +- `POLAR_MODE` +- `RESEND_API_KEY` +- `SENTRY_DSN` +- `STORAGE_TYPE` + +Conditional requirements: + +- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_S3_BUCKET`, `AWS_REGION` when `STORAGE_TYPE=s3` + +Optional after launch: + +- `GOOGLE_CLIENT_ID` +- `GOOGLE_CLIENT_SECRET` +- `GITHUB_WEBHOOK_SECRET` +- `GITHUB_TOKEN` +- `CDN_URL` +- `AITM_API_URL` +- `AITM_ENCRYPTION_SECRET` +- analytics and feature-flag keys + +Code reference: + +- `gitgram/backend/ENV_MATRIX.md` +- `gitgram/backend/.env.example` +- `gitgram/backend/src/config/env-contract.ts` + +## Gitgram Web + +Required before first launch: + +- `NEXT_PUBLIC_API_URL` +- `NEXT_PUBLIC_WS_URL` or legacy `NEXT_PUBLIC_SOCKET_URL` +- `NEXT_PUBLIC_GITHUB_CLIENT_ID` + +Optional after launch: + +- `NEXT_PUBLIC_GOOGLE_CLIENT_ID` +- `NEXT_PUBLIC_AITM_API_URL` +- `NEXT_PUBLIC_SENTRY_DSN` +- `NEXT_PUBLIC_GA_ID` +- `NEXT_PUBLIC_MAPBOX_TOKEN` + +Operational note: + +- `NEXT_PUBLIC_*` values are build-time inputs. Rebuild the web app whenever they change. + +Code reference: + +- `gitgram/web/.env.example` +- `gitgram/web/next.config.js` + +## AITM API + +Required before first launch: + +- `DATABASE_URL` +- `INTERNAL_SECRET` +- `CORS_ORIGIN` +- `WEBSITE_URL` +- `POLAR_ACCESS_TOKEN` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PRO_PRODUCT_ID` +- `POLAR_TEAM_PRODUCT_ID` +- `RESEND_API_KEY` +- `SENTRY_DSN` + +Optional after launch: + +- `CLERK_WEBHOOK_SECRET` +- `REDIS_URL` +- `OPENAI_API_KEY` +- `ANTHROPIC_API_KEY` +- `VOYAGE_API_KEY` +- `BRAVE_SEARCH_API_KEY` +- `SERPAPI_KEY` +- `SERPER_API_KEY` +- `FIRECRAWL_API_KEY` +- `KB_S3_BUCKET` +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- worker toggles and cron expressions + +Operational notes: + +- `DATABASE_URL` must be valid before the API starts in `staging` or `production`. +- Without LLM/search keys, AI-heavy routes degrade gracefully instead of blocking startup. +- Worker env vars are only required if you enable the managed scheduler on launch day. + +Code reference: + +- `aitimemachine/api-server/.env.example` +- `aitimemachine/api-server/src/lib/validateEnv.ts` +- `aitimemachine/api-server/src/config/env-validation.ts` + +## AITM npm Publish / Release Flow + +Manual publish requires: + +- authenticated npm session via `npm login` or `npm whoami` + +Optional CI automation: + +- GitHub secret `NPM_TOKEN` + +Operational notes: + +- The current CI release job publishes `@gitgram/timemachine`. +- `@gitgram/timemachine-mcp` still requires manual publish. + +## Optional Self-Hosted AITM Website + +Only required if you decide to run `aitimemachine/website` on the launch host. + +Required for self-host: + +- `NEXT_PUBLIC_API_URL` +- `API_URL` +- `INTERNAL_SECRET` +- `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` +- `CLERK_SECRET_KEY` +- `NEXT_PUBLIC_APP_URL` +- `POLAR_ACCESS_TOKEN` +- `NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID` +- `NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID` + +Code reference: + +- `.env.aitm-website.example` +- `aitimemachine/website/.env.example` + +## Validation Commands + +```bash +bash scripts/validate-prod-secrets.sh staging +bash scripts/validate-prod-secrets.sh production +``` + +These checks: + +- validate the launch-critical variable set +- support both prefixed and unprefixed naming +- auto-load the root `.env.*` files when present +- fail on placeholder values +- enforce `POLAR_MODE=production` for a production Gitgram launch diff --git a/docs/launch-secrets-checklist.md b/docs/launch-secrets-checklist.md new file mode 100644 index 00000000..04672c26 --- /dev/null +++ b/docs/launch-secrets-checklist.md @@ -0,0 +1,133 @@ +# Launch Secrets Checklist + +Last updated: 2026-05-12 + +This file is the operator-facing checklist for filling the launch env files before deploy. + +Preferred file layout: +- staging: `.env.gitgram.staging`, `.env.gitgram-web.staging`, `.env.aitm-api.staging` +- production: `.env.gitgram.production`, `.env.gitgram-web.production`, `.env.aitm-api.production` + +Fallback layout: +- `.env.gitgram` +- `.env.gitgram-web` +- `.env.aitm-api` + +Current gate: +- `npm run verify:launch` for deployable core readiness +- `npm run verify:launch:full` for all third-party integrations live + +Polar reference: +- `docs/polar-payment-setup.md` + +Current state: +- deploy/readiness scripts are in place +- AITM package dry-run verification passes +- dedicated Neon databases now exist for `gitgram_staging`, `gitgram_production`, `aitm_staging`, and `aitm_production` +- root staging/production env files now contain real database URLs and generated app-owned secrets +- full-integration launch is still blocked by provider-issued values, not local infra placeholders + +## Gitgram Backend Env + +App-owned values now set: +- `DATABASE_URL` +- `JWT_SECRET` +- `JWT_REFRESH_SECRET` + +Still provider-issued: +- `REDIS_URL` +- `FRONTEND_URL` +- `CORS_ORIGIN` +- `GITHUB_CLIENT_ID` +- `GITHUB_CLIENT_SECRET` +- `POLAR_ACCESS_TOKEN` +- `POLAR_MODE` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PREMIUM_PRODUCT_ID` +- `POLAR_CREATOR_PRODUCT_ID` +- `RESEND_API_KEY` +- `SENTRY_DSN` + +Optional depending on storage mode: +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- `AWS_S3_BUCKET` +- `AWS_REGION` + +Files: +- staging: `.env.gitgram.staging` +- production fallback: `.env.gitgram` or `.env.gitgram.production` + +## Gitgram Web Env + +Confirm or fill: +- `NEXT_PUBLIC_API_URL` +- `NEXT_PUBLIC_WS_URL` +- `NEXT_PUBLIC_GITHUB_CLIENT_ID` +- `NEXT_PUBLIC_GOOGLE_CLIENT_ID` +- `NEXT_PUBLIC_AITM_API_URL` +- `NEXT_PUBLIC_SENTRY_DSN` + +Files: +- staging: `.env.gitgram-web.staging` +- production fallback: `.env.gitgram-web` or `.env.gitgram-web.production` + +## AITM API Env + +App-owned values now set: +- `DATABASE_URL` +- `INTERNAL_SECRET` + +Still provider-issued: +- `CORS_ORIGIN` +- `WEBSITE_URL` +- `POLAR_ACCESS_TOKEN` +- `POLAR_MODE` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PRO_PRODUCT_ID` +- `POLAR_TEAM_PRODUCT_ID` +- `RESEND_API_KEY` +- `SENTRY_DSN` + +Optional: +- `CLERK_WEBHOOK_SECRET` +- `REDIS_URL` +- `OPENAI_API_KEY` +- `ANTHROPIC_API_KEY` +- `VOYAGE_API_KEY` +- `BRAVE_SEARCH_API_KEY` + +Files: +- staging: `.env.aitm-api.staging` +- production fallback: `.env.aitm-api` or `.env.aitm-api.production` + +## Provider Mapping + +GitHub OAuth: +- `GITHUB_CLIENT_ID` +- `GITHUB_CLIENT_SECRET` +- `NEXT_PUBLIC_GITHUB_CLIENT_ID` + +Polar: +- Gitgram: `POLAR_ACCESS_TOKEN`, `POLAR_MODE`, `POLAR_WEBHOOK_SECRET`, `POLAR_PREMIUM_PRODUCT_ID`, `POLAR_CREATOR_PRODUCT_ID` +- AITM: `POLAR_ACCESS_TOKEN`, `POLAR_MODE`, `POLAR_WEBHOOK_SECRET`, `POLAR_PRO_PRODUCT_ID`, `POLAR_TEAM_PRODUCT_ID` + +Email: +- `RESEND_API_KEY` + +Monitoring: +- `SENTRY_DSN` +- `NEXT_PUBLIC_SENTRY_DSN` + +Core infra: +- Gitgram: `DATABASE_URL`, `REDIS_URL`, `JWT_SECRET`, `JWT_REFRESH_SECRET` +- AITM: `DATABASE_URL`, `INTERNAL_SECRET` + +## Deploy Sequence + +1. Run `npm run verify:launch` for the core deploy gate. +2. Run `bash scripts/deploy-launch.sh staging --gitgram-only`. +3. Run `bash scripts/deploy-launch.sh staging --aitm-only`. +4. Configure nginx and DNS if not already live. +5. Run `npm run verify:public-launch`. +6. Once OAuth, Polar, Resend, and Sentry are configured, run `npm run verify:launch:full`. diff --git a/docs/ops-quickref.md b/docs/ops-quickref.md new file mode 100644 index 00000000..96af7348 --- /dev/null +++ b/docs/ops-quickref.md @@ -0,0 +1,150 @@ +# Operations Quick Reference + +Full source of truth: `docs/runbook.md` + +## Validate Launch Env + +```bash +bash scripts/validate-prod-secrets.sh staging +bash scripts/validate-prod-secrets.sh production +``` + +## Gitgram Deploy + +```bash +cd gitgram/backend +npm ci --legacy-peer-deps +npx prisma generate +npx prisma migrate deploy +npm run build +``` + +```bash +( + set -a + source ../../.env.gitgram + set +a + pm2 restart gitgram-api --update-env || pm2 start ../ecosystem.config.js --only gitgram-api +) +``` + +```bash +( + set -a + source .env.gitgram + source .env.gitgram-web + set +a + cd gitgram/web + npm ci --legacy-peer-deps + npm run build + cd .. + pm2 restart gitgram-web --update-env || pm2 start ecosystem.config.js --only gitgram-web +) +``` + +## AITM API Deploy + +```bash +cd aitimemachine/api-server +npm ci +npm run db:migrate +npm run build +``` + +```bash +( + set -a + source ../../.env.aitm-api + set +a + pm2 restart aitm-api --update-env || pm2 start ../deploy/ecosystem.config.cjs --only aitm-api +) +``` + +## AITM Package Publish + +```bash +cd aitimemachine +npm ci +npm run build +npm test +bash scripts/verify-npm-packages.sh +npm publish --access public +``` + +```bash +cd aitimemachine/mcp +npm ci +npm run build +npm publish --access public +``` + +## Health Checks + +```bash +curl -sf http://127.0.0.1:4000/health | jq . +curl -sf http://127.0.0.1:4000/health/ready | jq . +curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1:3000 + +curl -sf http://127.0.0.1:3001/health | jq . +curl -sf http://127.0.0.1:3001/health/ready | jq . +``` + +Worker status when enabled: + +```bash +( + set -a + source .env.aitm-api + set +a + curl -sf -H "X-Internal-Secret: ${INTERNAL_SECRET}" \ + http://127.0.0.1:3001/v1/workers/status | jq . +) +``` + +## Backups + +```bash +( + set -a + source .env.gitgram + set +a + cd gitgram + ./scripts/backup-database.sh +) +``` + +```bash +( + set -a + source .env.aitm-api + set +a + pg_dump "$DATABASE_URL" | gzip > "/var/backups/aitm/aitm_$(date +%Y%m%d_%H%M%S).sql.gz" +) +``` + +## Rollback + +```bash +git checkout <previous-tag-or-commit> +``` + +```bash +pm2 restart gitgram-api --update-env +pm2 restart gitgram-web --update-env +pm2 restart aitm-api --update-env +``` + +Database rollback policy: + +- Gitgram: restore from pre-migration backup or provider PITR. +- AITM: restore from pre-migration backup or provider PITR. + +## Useful PM2 Commands + +```bash +pm2 ls +pm2 logs gitgram-api --lines 200 +pm2 logs gitgram-web --lines 100 +pm2 logs aitm-api --lines 200 +pm2 save +``` diff --git a/docs/polar-payment-setup.md b/docs/polar-payment-setup.md new file mode 100644 index 00000000..3c224479 --- /dev/null +++ b/docs/polar-payment-setup.md @@ -0,0 +1,112 @@ +# Polar Payment Setup + +Last updated: 2026-05-12 + +This guide is based on the locally cloned Polar source in [external/polar](/Users/arhansubasi/gitgram-ai/external/polar). + +Primary upstream references: +- [sandbox.mdx](/Users/arhansubasi/gitgram-ai/external/polar/docs/integrate/sandbox.mdx) +- [introduction.mdx](/Users/arhansubasi/gitgram-ai/external/polar/docs/integrate/oauth2/introduction.mdx) +- [delivery.mdx](/Users/arhansubasi/gitgram-ai/external/polar/docs/integrate/webhooks/delivery.mdx) +- [typescript.mdx](/Users/arhansubasi/gitgram-ai/external/polar/docs/integrate/sdk/typescript.mdx) +- [security.yml](/Users/arhansubasi/gitgram-ai/external/polar/sdk/overlays/security.yml) + +## What To Use + +Use an **Organization Access Token** for Gitgram and AITM service-to-Polar integration. + +Do not use OAuth/OpenID Connect unless you are building a third-party app that acts on behalf of other Polar users. + +From Polar’s own docs: +- service integrations should use an organization access token +- sandbox has its own separate token and separate organization +- the TS SDK should be configured with `server: 'sandbox'` or `server: 'production'` + +This repo also now supports an optional `POLAR_SERVER_URL` override for Gitgram and AITM. Use it only if you are running a self-hosted/local Polar instance from the cloned source instead of hosted Polar. + +## Staging Values + +For staging/testing: +- set `POLAR_MODE=sandbox` +- create the token in the sandbox environment, not production +- use sandbox product IDs from the sandbox organization +- use the sandbox webhook secret from the sandbox endpoint config +- if you are running a local/self-hosted Polar fork, leave `POLAR_MODE` as documentation only and set `POLAR_SERVER_URL` to that instance instead + +Sandbox dashboard/API references from local Polar docs: +- dashboard: `https://sandbox.polar.sh/start` +- API base: `https://sandbox-api.polar.sh` + +Successful test card from Polar’s sandbox docs: +- `4242 4242 4242 4242` + +## Production Values + +For production: +- set `POLAR_MODE=production` +- use the live organization access token +- use live product IDs from the live organization +- use the live webhook secret from the production endpoint config + +## Where Each Value Comes From + +`POLAR_ACCESS_TOKEN` +- generate an organization access token from your organization settings + +`POLAR_SERVER_URL` +- optional override for self-hosted/local Polar from `external/polar` +- when unset, the SDK uses hosted `sandbox` or `production` based on `POLAR_MODE` + +`POLAR_MODE` +- `sandbox` for staging/test +- `production` for live deploy + +`POLAR_WEBHOOK_SECRET` +- copy the webhook secret from the webhook endpoint setup in Polar +- when verifying through the SDK, pass the raw secret value +- Polar’s SDK handles the Standard Webhooks base64 requirement internally + +`POLAR_*_PRODUCT_ID` +- copy the exact product IDs from the corresponding Polar environment +- sandbox and production product IDs are different because the environments are isolated + +## Repo Mapping + +Gitgram: +- `.env.gitgram.staging` or `.env.gitgram` +- `POLAR_ACCESS_TOKEN` +- `POLAR_MODE` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PREMIUM_PRODUCT_ID` +- `POLAR_CREATOR_PRODUCT_ID` + +AITM API: +- `.env.aitm-api.staging` or `.env.aitm-api` +- `POLAR_ACCESS_TOKEN` +- `POLAR_MODE` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_PRO_PRODUCT_ID` +- `POLAR_TEAM_PRODUCT_ID` + +AITM website: +- `.env.aitm-website` +- `POLAR_ACCESS_TOKEN` +- `POLAR_MODE` +- `NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID` +- `NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID` + +## Testing Flow + +1. Fill staging envs with sandbox values. +2. Run `npm run verify:launch:staging:full`. +3. Deploy with `bash scripts/deploy-launch.sh staging`. +4. Open the billing flow and complete a sandbox checkout with `4242 4242 4242 4242`. +5. Confirm webhook delivery updates the local subscription state. + +If you are only validating deployability before billing is wired, `npm run verify:launch:staging` is enough. Use the `:full` gate when you expect checkout and webhooks to work. + +## Current Code Alignment + +This repo now aligns to the Polar source in two important ways: +- AITM and Gitgram Polar clients honor `POLAR_MODE` +- AITM and Gitgram webhook verification use the SDK webhook validator instead of custom signature logic diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 00000000..1c551420 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,569 @@ +# Manual Launch Runbook + +Last updated: 2026-05-12 + +This file is the launch source of truth for Gitgram and AITM. + +- Primary topology: one operator-managed host. +- Primary process manager: `pm2`. +- AITM `systemd` unit: fallback only. +- `docker-compose.prod.yml`, `deploy/DEPLOYMENT.md`, and GitHub deploy workflows are not the canonical launch path. + +## Canonical Topology + +| Service | Product | Runtime | Control Plane | Port | Health Gate | +|---------|---------|---------|---------------|------|-------------| +| `gitgram-api` | Gitgram | Node / Express | `pm2` | `4000` | `GET /health`, `GET /health/ready` | +| `gitgram-web` | Gitgram | Node / Next.js | `pm2` | `3000` | `GET /` returns `<500` | +| `aitm-api` | AITM | Node / Express | `pm2` primary, `systemd` fallback | `3001` | `GET /health`, `GET /health/ready` | +| `postgres` | Shared infra | PostgreSQL 16 | operator-managed | `5432` | `pg_isready` | +| `redis` | Shared infra | Redis 7 | operator-managed | `6379` | `redis-cli ping` | + +Optional companion service: + +| Service | Product | Runtime | Port | Notes | +|---------|---------|---------|------|------| +| `aitm-website` | AITM | Node / Next.js | `3002` | Self-host only if you want the hosted dashboard on the same machine. Not part of the launch-critical path in this repo. | + +## Repo Paths + +| Artifact | Path | +|---------|------| +| Gitgram PM2 config | `gitgram/ecosystem.config.js` | +| AITM PM2 config | `aitimemachine/deploy/ecosystem.config.cjs` | +| AITM `systemd` fallback unit | `aitimemachine/deploy/aitm-api.service` | +| Gitgram backend env template | `.env.gitgram.example` | +| Gitgram web env template | `.env.gitgram-web.example` | +| AITM API env template | `.env.aitm-api.example` | +| Optional AITM website env template | `.env.aitm-website.example` | +| Gitgram backend schema | `gitgram/backend/prisma/schema.prisma` | +| AITM API schema | `aitimemachine/api-server/src/db/schema.ts` | + +## Preflight + +Run these before any staging-like dry run or production launch: + +```bash +node -v +npm -v +pm2 -v +``` + +Use Node 20.x for build and runtime consistency with the repo workflows. + +Copy and fill the operator env files at repo root. + +Preferred layout: + +```bash +cp .env.gitgram.example .env.gitgram.production +cp .env.gitgram-web.example .env.gitgram-web.production +cp .env.aitm-api.example .env.aitm-api.production + +cp .env.gitgram.example .env.gitgram.staging +cp .env.gitgram-web.example .env.gitgram-web.staging +cp .env.aitm-api.example .env.aitm-api.staging +# Optional: +cp .env.aitm-website.example .env.aitm-website +``` + +Fallback layout still works if you want one shared set of root files: + +```bash +cp .env.gitgram.example .env.gitgram +cp .env.gitgram-web.example .env.gitgram-web +cp .env.aitm-api.example .env.aitm-api +``` + +Validate launch-critical env contracts: + +```bash +bash scripts/validate-prod-secrets.sh staging core +bash scripts/validate-prod-secrets.sh production core +``` + +One-command readiness gate: + +```bash +npm run verify:launch +``` + +This is the deployable core profile: database, app secrets, routing URLs, storage mode, and PM2/deploy artifacts. + +Optional deeper gate: + +```bash +npm run verify:launch:full +``` + +This adds third-party integrations that are not required just to boot the services: +- GitHub OAuth +- Polar billing +- Resend +- Sentry + +Manual deploy helper: + +```bash +bash scripts/deploy-launch.sh staging +``` + +To require the full integrations gate before deploy: + +```bash +bash scripts/deploy-launch.sh staging --verify-profile full +``` + +Secret entry reference: + +- `docs/launch-secrets-checklist.md` + +Notes: + +- The validator accepts both unprefixed vars and `GITGRAM_*` / `AITM_*` aliases. +- It prefers `.env.gitgram.<env>`, `.env.gitgram-web.<env>`, `.env.aitm-api.<env>`, and `.env.aitm-website.<env>` when those files exist, and falls back to the root files otherwise. +- `verify:launch` uses the `core` profile by default so you can deploy the stack before every external provider account is wired. +- `verify:launch:full` and `validate-prod-secrets.sh <env> full` enforce the full launch contract, including third-party integrations. +- `verify:launch` also checks PM2 availability, required deploy artifacts, operator env file presence, and AITM package publishability. +- `deploy-launch.sh` is the scripted PM2/manual deploy flow. It is the automation companion to this runbook, not a Docker deploy. + +## Backup Gate + +Take backups before any production migration. + +Gitgram: + +```bash +( + set -a + source .env.gitgram + set +a + cd gitgram + ./scripts/backup-database.sh +) +``` + +AITM: + +```bash +( + set -a + source .env.aitm-api + set +a + mkdir -p /var/backups/aitm + pg_dump "$DATABASE_URL" | gzip > "/var/backups/aitm/aitm_$(date +%Y%m%d_%H%M%S).sql.gz" +) +``` + +Redis snapshot: + +```bash +redis-cli -u "$(grep '^REDIS_URL=' .env.gitgram | cut -d= -f2-)" BGSAVE +``` + +## AITM Package Release + +Publish order matters: + +1. `@gitgram/timemachine` +2. `@gitgram/timemachine-mcp` + +### Manual Publish Checklist + +Core package: + +```bash +cd aitimemachine +npm ci +npm run build +npm test +bash scripts/verify-npm-packages.sh +npm whoami +npm publish --access public +``` + +Post-publish verification: + +```bash +npm view @gitgram/timemachine version +npx -y @gitgram/timemachine --version + +tmpdir="$(mktemp -d)" +( + cd "$tmpdir" + npx -y @gitgram/timemachine init + npx -y @gitgram/timemachine checkpoint "launch-smoke" +) +rm -rf "$tmpdir" +``` + +MCP package: + +```bash +cd aitimemachine/mcp +npm ci +npm run build +npm publish --access public +``` + +Post-publish verification: + +```bash +npm view @gitgram/timemachine-mcp version +npx -y @gitgram/timemachine-mcp --help +``` + +### Package Rollback Policy + +Use manual deprecation plus a version bump. Do not rely on unpublish. + +```bash +npm deprecate @gitgram/timemachine@<bad-version> "broken release; use >=<fixed-version>" +npm deprecate @gitgram/timemachine-mcp@<bad-version> "broken release; use >=<fixed-version>" +``` + +Then publish a new patch or minor release: + +```bash +cd aitimemachine +npm version patch --no-git-tag-version + +cd mcp +npm version patch --no-git-tag-version +``` + +Automation status: + +- `aitimemachine/.github/workflows/ci.yml` has a tag-driven npm publish step. +- It currently publishes only `@gitgram/timemachine`. +- The MCP publish remains manual until the workflow is brought into line with the manual launch process. +- The GitHub secret name used there is `NPM_TOKEN`, passed to npm as `NODE_AUTH_TOKEN`. + +## Gitgram Deploy + +### Backend Build + Migrate + +```bash +cd gitgram/backend +npm ci --legacy-peer-deps +npx prisma generate +npx prisma migrate deploy +npm run build +``` + +### Backend Start / Restart + +Initial start: + +```bash +( + set -a + source ../../.env.gitgram + set +a + pm2 start ../ecosystem.config.js --only gitgram-api +) +pm2 save +``` + +Redeploy: + +```bash +( + set -a + source ../../.env.gitgram + set +a + pm2 restart gitgram-api --update-env +) +pm2 save +``` + +Scripted equivalent: + +```bash +bash scripts/deploy-launch.sh staging --gitgram-only +``` + +### Backend Health Gate + +```bash +curl -sf http://127.0.0.1:4000/health | jq . +curl -sf http://127.0.0.1:4000/health/ready | jq . +``` + +## Gitgram Web Deploy + +### Build + +`NEXT_PUBLIC_*` values are build-time inputs. Rebuild whenever they change. + +```bash +( + set -a + source .env.gitgram-web + set +a + cd gitgram/web + npm ci --legacy-peer-deps + npm run build +) +``` + +### Start / Restart + +Initial start: + +```bash +( + set -a + source .env.gitgram-web + set +a + cd gitgram + pm2 start ecosystem.config.js --only gitgram-web +) +pm2 save +``` + +Redeploy: + +```bash +( + set -a + source .env.gitgram-web + set +a + pm2 restart gitgram-web --update-env +) +pm2 save +``` + +Scripted equivalent: + +```bash +bash scripts/deploy-launch.sh staging --gitgram-only +``` + +### Web Health Gate + +```bash +curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1:3000 +``` + +## AITM API Deploy + +### Build + Migrate + +```bash +cd aitimemachine/api-server +npm ci +npm run db:migrate +npm run build +``` + +### Start / Restart With PM2 + +Initial start: + +```bash +( + set -a + source ../../.env.aitm-api + set +a + pm2 start ../deploy/ecosystem.config.cjs --only aitm-api +) +pm2 save +``` + +Redeploy: + +```bash +( + set -a + source ../../.env.aitm-api + set +a + pm2 restart aitm-api --update-env +) +pm2 save +``` + +Scripted equivalent: + +```bash +bash scripts/deploy-launch.sh staging --aitm-only +``` + +### `systemd` Fallback + +Use only if PM2 is unavailable or intentionally replaced. + +```bash +sudo cp aitimemachine/deploy/aitm-api.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable aitm-api +sudo systemctl restart aitm-api +``` + +The checked-in unit assumes the repo lives at `/opt/aitimemachine/api-server`. Adjust `WorkingDirectory`, `ExecStart`, and `EnvironmentFile` if your server layout differs. + +### API Health Gate + +```bash +curl -sf http://127.0.0.1:3001/health | jq . +curl -sf http://127.0.0.1:3001/health/ready | jq . +``` + +If `ENABLE_WORKERS=true`, also check worker status: + +```bash +( + set -a + source .env.aitm-api + set +a + curl -sf -H "X-Internal-Secret: ${INTERNAL_SECRET}" \ + http://127.0.0.1:3001/v1/workers/status | jq . +) +``` + +## Optional AITM Website + +This repo includes `aitimemachine/website`, but it is not part of the launch-critical path defined here. + +If you self-host it on the same machine: + +```bash +( + set -a + source .env.aitm-website + set +a + cd aitimemachine/website + npm ci + PORT=3002 npm run build + PORT=3002 npm start +) +``` + +## Launch-Day Order + +1. Freeze merges and confirm the intended commit or tag. +2. Run `bash scripts/validate-prod-secrets.sh production`. +3. Take Gitgram and AITM database backups. +4. Publish `@gitgram/timemachine`. +5. Verify install + smoke. +6. Publish `@gitgram/timemachine-mcp`. +7. Verify install + smoke. +8. Deploy Gitgram backend. +9. Wait for Gitgram backend health gates. +10. Deploy Gitgram web. +11. Wait for Gitgram web health gate. +12. Deploy AITM API. +13. Wait for AITM API health gates. +14. Run smoke checks. +15. Monitor for at least 30 minutes before declaring success. + +## Smoke Checks + +### Gitgram + +Required: + +- `curl -sf http://127.0.0.1:4000/health` +- `curl -sf http://127.0.0.1:4000/health/ready` +- `curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1:3000` + +Manual: + +- Load the web app. +- Confirm it can reach the API. +- Exercise one login path. +- Create one basic post. + +### AITM + +Required: + +- `curl -sf http://127.0.0.1:3001/health` +- `curl -sf http://127.0.0.1:3001/health/ready` +- `npx -y @gitgram/timemachine --version` +- `npx -y @gitgram/timemachine-mcp --help` + +Manual: + +- Create a temp project. +- Run `init`. +- Create a checkpoint. +- If workers are enabled, verify `/v1/workers/status`. + +Public-domain smoke: + +- `npm run verify:public-launch` + +## Rollback + +### npm Packages + +1. Deprecate the bad version. +2. Bump the version. +3. Rebuild and republish. +4. Re-run the install smokes. + +### Gitgram Backend / Web + +Rollback is manual-first. Use the previous known-good git tag, commit, or release directory already present on the server. + +Example with an in-place git checkout: + +```bash +git checkout <previous-tag-or-commit> + +cd gitgram/backend +npm ci --legacy-peer-deps +npx prisma generate +npm run build + +cd ../web +npm ci --legacy-peer-deps +npm run build + +pm2 restart gitgram-api --update-env +pm2 restart gitgram-web --update-env +``` + +### AITM API + +```bash +git checkout <previous-tag-or-commit> + +cd aitimemachine/api-server +npm ci +npm run build + +pm2 restart aitm-api --update-env +``` + +### Database Rollback Policy + +- Gitgram / Prisma: no automatic `down` migrations in this launch contract. +- AITM / Drizzle: no automatic `down` migrations in this launch contract. +- Production rollback policy is restore from backup or provider point-in-time restore unless you have already written and reviewed a reverse SQL plan. + +If a migration fails after partial apply: + +1. Stop the app deploy. +2. Do not keep rolling forward blindly. +3. Restore from the backup taken immediately before the migration, or use provider PITR. + +## Monitoring Window + +Watch for at least 30 minutes after launch: + +```bash +pm2 ls +pm2 logs gitgram-api --lines 200 +pm2 logs gitgram-web --lines 100 +pm2 logs aitm-api --lines 200 +``` + +Recommended external checks: + +- Sentry error volume +- Polar webhook delivery success +- PostgreSQL connection count +- Redis availability +- repeated `curl` probes against `/health` and `/health/ready` diff --git a/docs/secrets-contract.md b/docs/secrets-contract.md new file mode 100644 index 00000000..e5133d1b --- /dev/null +++ b/docs/secrets-contract.md @@ -0,0 +1,195 @@ +# Secrets Contract + +Last updated: 2026-05-09 + +This file describes secret ownership, storage, and rotation for the manual launch path. + +## Storage Rules + +Primary runtime storage: + +- server-side env files generated from `.env.gitgram.example`, `.env.gitgram-web.example`, `.env.aitm-api.example` +- optional `.env.aitm-website` if you self-host the AITM dashboard + +Optional storage for future automation: + +- GitHub Actions secrets and environment variables + +Rules: + +1. Do not commit populated `.env` files. +2. Do not rely on GitHub Actions secrets as the only production source of truth for a manual launch. +3. Use separate values for staging and production. +4. Use separate product-scoped credentials where providers allow it. + +## Launch-Critical Secrets + +### Gitgram + +| Secret | Purpose | Rotate | +|---|---|---| +| `DATABASE_URL` | PostgreSQL connection | on DB credential change | +| `REDIS_URL` | Redis connection | on Redis credential change | +| `JWT_SECRET` | access-token signing | every 90 days | +| `JWT_REFRESH_SECRET` | refresh-token signing | every 90 days | +| `GITHUB_CLIENT_SECRET` | GitHub OAuth | on compromise | +| `POLAR_ACCESS_TOKEN` | billing API | on compromise | +| `POLAR_WEBHOOK_SECRET` | billing webhook verification | on endpoint re-creation | +| `RESEND_API_KEY` | transactional email | on compromise | +| `SENTRY_DSN` | error reporting | when project changes | + +Conditional: + +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` + +These are required only when `STORAGE_TYPE=s3`. + +### Gitgram Web + +The launch-critical web vars are mostly public build inputs, not secrets. + +Non-secret public values: + +- `NEXT_PUBLIC_API_URL` +- `NEXT_PUBLIC_WS_URL` +- `NEXT_PUBLIC_GITHUB_CLIENT_ID` + +Optional public values: + +- `NEXT_PUBLIC_GOOGLE_CLIENT_ID` +- `NEXT_PUBLIC_AITM_API_URL` +- `NEXT_PUBLIC_SENTRY_DSN` + +### AITM API + +| Secret | Purpose | Rotate | +|---|---|---| +| `DATABASE_URL` | PostgreSQL / Neon connection | on DB credential change | +| `INTERNAL_SECRET` | internal website-to-API auth | every 90 days | +| `POLAR_ACCESS_TOKEN` | billing API | on compromise | +| `POLAR_WEBHOOK_SECRET` | billing webhook verification | on endpoint re-creation | +| `RESEND_API_KEY` | transactional email | on compromise | +| `SENTRY_DSN` | error reporting | when project changes | + +Optional after launch: + +- `CLERK_WEBHOOK_SECRET` +- `OPENAI_API_KEY` +- `ANTHROPIC_API_KEY` +- `VOYAGE_API_KEY` +- `BRAVE_SEARCH_API_KEY` +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` + +### AITM npm Release Flow + +Manual publish: + +- operator npm session from `npm login` + +CI release: + +- GitHub secret `NPM_TOKEN` + +Note: + +- the release workflow maps `NPM_TOKEN` into `NODE_AUTH_TOKEN` +- it currently covers `@gitgram/timemachine`, not the MCP package + +### Optional Self-Hosted AITM Website + +Only needed if you run `aitimemachine/website` yourself. + +Secrets: + +- `CLERK_SECRET_KEY` +- `INTERNAL_SECRET` +- `POLAR_ACCESS_TOKEN` + +Public build inputs: + +- `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` +- `NEXT_PUBLIC_API_URL` +- `API_URL` +- `NEXT_PUBLIC_APP_URL` +- `NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID` +- `NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID` + +## Shared Secret Policy + +### AWS + +Prefer separate IAM users: + +- Gitgram S3 user: media bucket only +- AITM S3 user: knowledge-base bucket only + +### Redis + +A shared Redis instance is acceptable. + +- Gitgram uses it for cache/session concerns. +- AITM uses it only when optional distributed rate limiting is enabled. + +### Resend + +Prefer separate API keys under the same account: + +- one key for Gitgram +- one key for AITM + +### Sentry + +Use separate projects and DSNs for Gitgram and AITM. + +## Rotation Checklist + +JWT-style secrets: + +```bash +openssl rand -base64 64 +``` + +Recommended process: + +1. rotate in staging first +2. redeploy staging and verify +3. rotate production +4. redeploy production +5. record the date and owner + +Database URLs: + +1. rotate credentials at the provider +2. update the server env file +3. restart the affected service +4. verify health endpoints + +Webhook secrets: + +1. recreate or rotate in the provider dashboard +2. update the matching env file +3. restart the service +4. verify one live webhook + +## What Can Wait Until After Launch + +These are explicitly not part of the first-launch blocker set: + +- Google OAuth for Gitgram +- AITM AI provider keys +- AITM web-search provider keys +- AITM Redis +- AITM S3 / knowledge-base offload +- optional self-hosted AITM website + +## Validation + +Use: + +```bash +bash scripts/validate-prod-secrets.sh production +``` + +That script is the repo-level preflight check for the manual launch contract. diff --git a/external/README.md b/external/README.md new file mode 100644 index 00000000..d196d809 --- /dev/null +++ b/external/README.md @@ -0,0 +1,22 @@ +# External Reference Repos + +This directory holds shallow upstream clones used as implementation references for Gitgram and AITM. + +Current expected clones: +- `repo2txt` +- `repo-to-text` +- `a2a-js` +- `context7` +- `pal-mcp-server` + +Rules: +- do not treat these repos as part of the product source tree +- do not make product changes inside these clones +- pull patterns from them into `gitgram/` or `aitimemachine/` deliberately + +Refresh example: + +```bash +git -C external/repo2txt fetch --depth 1 origin +git -C external/repo2txt reset --hard origin/main +``` diff --git a/external/a2a-js b/external/a2a-js new file mode 160000 index 00000000..dbaccb96 --- /dev/null +++ b/external/a2a-js @@ -0,0 +1 @@ +Subproject commit dbaccb964f9bb5d9ae7e993f797cebaeae52ac62 diff --git a/external/beads b/external/beads new file mode 160000 index 00000000..4d08edbe --- /dev/null +++ b/external/beads @@ -0,0 +1 @@ +Subproject commit 4d08edbeb77209d801daae42ba6de5484e7b95b8 diff --git a/external/bluesky-social-app b/external/bluesky-social-app new file mode 160000 index 00000000..dd452a43 --- /dev/null +++ b/external/bluesky-social-app @@ -0,0 +1 @@ +Subproject commit dd452a433638789618e875b96d511c9aeadb99ac diff --git a/external/context7 b/external/context7 new file mode 160000 index 00000000..61de754d --- /dev/null +++ b/external/context7 @@ -0,0 +1 @@ +Subproject commit 61de754d48e57d5c22dadd4540a74203fd55ecf1 diff --git a/external/gh-social b/external/gh-social new file mode 160000 index 00000000..e5ec48e8 --- /dev/null +++ b/external/gh-social @@ -0,0 +1 @@ +Subproject commit e5ec48e8e5efbc5140b2b023c08cc502525a01f8 diff --git a/external/gitnexus b/external/gitnexus new file mode 160000 index 00000000..dbd4e1c9 --- /dev/null +++ b/external/gitnexus @@ -0,0 +1 @@ +Subproject commit dbd4e1c9fb1d2fe6866c651873078029464663db diff --git a/external/munia b/external/munia new file mode 160000 index 00000000..577868b6 --- /dev/null +++ b/external/munia @@ -0,0 +1 @@ +Subproject commit 577868b63adf2b3d61e25809bd665f8c7687c141 diff --git a/external/nextjs-15-social-media-app b/external/nextjs-15-social-media-app new file mode 160000 index 00000000..2f407a5c --- /dev/null +++ b/external/nextjs-15-social-media-app @@ -0,0 +1 @@ +Subproject commit 2f407a5c8a6be333fdac54ac6d36b0c385348cb4 diff --git a/external/pal-mcp-server b/external/pal-mcp-server new file mode 160000 index 00000000..7afc7c1c --- /dev/null +++ b/external/pal-mcp-server @@ -0,0 +1 @@ +Subproject commit 7afc7c1cc96e23992c8f105f960132c657883bb1 diff --git a/external/polar b/external/polar new file mode 160000 index 00000000..c818d921 --- /dev/null +++ b/external/polar @@ -0,0 +1 @@ +Subproject commit c818d921dd6deee835410e065c78cf9365915876 diff --git a/external/proximity b/external/proximity new file mode 160000 index 00000000..768156dc --- /dev/null +++ b/external/proximity @@ -0,0 +1 @@ +Subproject commit 768156dc84cd4381a5ebdbfd589b2e4624e1ae76 diff --git a/external/repo-to-text b/external/repo-to-text new file mode 160000 index 00000000..eff5a544 --- /dev/null +++ b/external/repo-to-text @@ -0,0 +1 @@ +Subproject commit eff5a5449ff32f8c6e3bf8a071a2cbadb47f04d6 diff --git a/external/repo2txt b/external/repo2txt new file mode 160000 index 00000000..bba6c85c --- /dev/null +++ b/external/repo2txt @@ -0,0 +1 @@ +Subproject commit bba6c85c26e4273945fba6f6aaedf06607e18b6b diff --git a/external/t3-twitter-clone b/external/t3-twitter-clone new file mode 160000 index 00000000..b21c9199 --- /dev/null +++ b/external/t3-twitter-clone @@ -0,0 +1 @@ +Subproject commit b21c919986092f9c4e7cc6abf03648c74ba72d9b diff --git a/gitgram b/gitgram new file mode 160000 index 00000000..108517d1 --- /dev/null +++ b/gitgram @@ -0,0 +1 @@ +Subproject commit 108517d1608d13d82c670a2e4b3db9587b604214 diff --git a/package.json b/package.json new file mode 100644 index 00000000..a0a73323 --- /dev/null +++ b/package.json @@ -0,0 +1,80 @@ +{ + "name": "gitgram-ai", + "version": "1.0.0", + "private": true, + "description": "Gitgram AI monorepo — social platform for developers + AITimeMachine", + "workspaces": [ + "gitgram/web", + "gitgram/backend", + "gitgram/gh-social", + "gitgram/t3-base", + "aitimemachine", + "aitimemachine/api-server", + "aitimemachine/mcp", + "aitimemachine/gitnexus", + "aitimemachine/gitnexus-web", + "aitimemachine/gitnexus-shared", + "aitimemachine/a2a", + "aitimemachine/context7/packages/mcp", + "aitimemachine/context7/packages/cli", + "aitimemachine/context7/packages/sdk", + "aitimemachine/tools/repo2txt" + ], + "scripts": { + "test:all": "npm run test:gitgram-backend && npm run test:gitgram-web && npm run test:aitm", + "test:gitgram-backend": "cd gitgram/backend && npm test", + "test:gitgram-web": "cd gitgram/web && npm test", + "test:aitm": "cd aitimemachine/api-server && npm test", + + "dev:gitgram": "concurrently \"npm run dev:gitgram-backend\" \"npm run dev:gitgram-web\"", + "dev:gitgram-backend": "cd gitgram/backend && npm run dev", + "dev:gitgram-web": "cd gitgram/web && npm run dev", + "dev:gh-social": "cd gitgram/gh-social && npm run dev", + "dev:t3": "cd gitgram/t3-base && npm run dev", + "dev:proximity": "cd gitgram/mobile/proximity && npx expo start", + + "dev:aitm": "concurrently \"npm run dev:aitm-api\" \"npm run dev:aitm-mcp\" \"npm run dev:gitnexus\"", + "dev:aitm-api": "cd aitimemachine/api-server && npm run dev", + "dev:aitm-api:clean": "npm --prefix aitimemachine/api-server run dev:clean", + "dev:aitm-mcp": "cd aitimemachine/mcp && npm run dev", + "dev:gitnexus": "cd aitimemachine/gitnexus && npm run dev", + "dev:gitnexus-web": "cd aitimemachine/gitnexus-web && npm run dev", + "dev:a2a": "cd aitimemachine/a2a && npm run dev 2>/dev/null || echo 'a2a has no dev script — build with: cd aitimemachine/a2a && npm run build'", + "dev:context7": "cd aitimemachine/context7/packages/mcp && npm run dev 2>/dev/null || node aitimemachine/context7/packages/mcp/dist/index.js", + "dev:pal-mcp": "cd aitimemachine/pal-mcp-server && ./run-server.sh", + "dev:repo2txt": "cd aitimemachine/tools/repo2txt && npm run dev", + "dev:repo-to-text": "cd aitimemachine/tools/repo-to-text && python -m repo_to_text", + + "build:beads": "cd aitimemachine/beads && go build -o bd ./cmd/bd/ && echo 'beads bd binary built at aitimemachine/beads/bd'", + "build:all": "npm run build:gitgram-backend && npm run build:gitgram-web && npm run build:aitm && npm run build:gitnexus", + "build:gitgram-backend": "cd gitgram/backend && npm run build", + "build:gitgram-web": "cd gitgram/web && npm run build", + "build:aitm": "cd aitimemachine/api-server && npm run build", + "build:gitnexus": "cd aitimemachine/gitnexus && npm run build", + + "lint:all": "npm run lint:gitgram-backend && npm run lint:aitm", + "lint:gitgram-backend": "cd gitgram/backend && npm run lint", + "lint:aitm": "cd aitimemachine/api-server && npx tsc --noEmit", + + "typecheck:all": "npm run typecheck:gitgram && npm run typecheck:aitm", + "typecheck:gitgram": "npm run typecheck:gitgram-backend && npm run typecheck:gitgram-web", + "typecheck:gitgram-backend": "cd gitgram/backend && npx tsc --noEmit", + "typecheck:gitgram-web": "cd gitgram/web && npx tsc --noEmit", + "typecheck:aitm": "cd aitimemachine && npx tsc --noEmit && cd api-server && npx tsc --noEmit", + + "smoke:local-launch": "bash scripts/smoke-local-launch.sh", + "deploy:launch:staging": "bash scripts/deploy-launch.sh staging", + "deploy:launch:production": "bash scripts/deploy-launch.sh production", + "verify:aitm-packages": "cd aitimemachine && npm run verify:packages", + "verify:aitm-local": "bash scripts/validate-prod-secrets.sh local", + "verify:launch": "bash scripts/verify-launch-readiness.sh all core", + "verify:launch:staging": "bash scripts/verify-launch-readiness.sh staging core", + "verify:launch:production": "bash scripts/verify-launch-readiness.sh production core", + "verify:launch:full": "RUN_TYPECHECK=1 bash scripts/verify-launch-readiness.sh all full", + "verify:launch:staging:full": "RUN_TYPECHECK=1 bash scripts/verify-launch-readiness.sh staging full", + "verify:launch:production:full": "RUN_TYPECHECK=1 bash scripts/verify-launch-readiness.sh production full", + "verify:public-launch": "bash scripts/verify-public-launch.sh", + "verify:secrets:staging": "bash scripts/validate-prod-secrets.sh staging full", + "verify:secrets:production": "bash scripts/validate-prod-secrets.sh production full" + } +} diff --git a/scripts/analyze-with-gitnexus.sh b/scripts/analyze-with-gitnexus.sh new file mode 100755 index 00000000..5b3f0892 --- /dev/null +++ b/scripts/analyze-with-gitnexus.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# analyze-with-gitnexus.sh +# +# Usage: ./scripts/analyze-with-gitnexus.sh <REPO> +# +# REPO must be one of: +# gitgram-backend gitgram-web aitm-api aitm-mcp +# --------------------------------------------------------------------------- + +MONOREPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GITNEXUS_DIR="${MONOREPO_ROOT}/aitimemachine/gitnexus" +GITNEXUS_CLI="${GITNEXUS_DIR}/dist/cli/index.js" + +# --------------------------------------------------------------------------- +# Argument handling +# --------------------------------------------------------------------------- +if [[ $# -lt 1 ]]; then + echo "Usage: $0 <REPO>" + echo "" + echo "Available repos:" + echo " gitgram-backend -> gitgram/backend" + echo " gitgram-web -> gitgram/web" + echo " aitm-api -> aitimemachine/api-server" + echo " aitm-mcp -> aitimemachine/mcp" + exit 1 +fi + +REPO="$1" + +# --------------------------------------------------------------------------- +# Map repo name -> absolute path + display name +# --------------------------------------------------------------------------- +case "${REPO}" in + gitgram-backend) + REPO_PATH="${MONOREPO_ROOT}/gitgram/backend" + REPO_NAME="gitgram-backend" + ;; + gitgram-web) + REPO_PATH="${MONOREPO_ROOT}/gitgram/web" + REPO_NAME="gitgram-web" + ;; + aitm-api) + REPO_PATH="${MONOREPO_ROOT}/aitimemachine/api-server" + REPO_NAME="aitm-api" + ;; + aitm-mcp) + REPO_PATH="${MONOREPO_ROOT}/aitimemachine/mcp" + REPO_NAME="aitm-mcp" + ;; + *) + echo "ERROR: Unknown repo '${REPO}'" + echo "Valid options: gitgram-backend, gitgram-web, aitm-api, aitm-mcp" + exit 1 + ;; +esac + +if [[ ! -d "${REPO_PATH}" ]]; then + echo "ERROR: Repo path does not exist: ${REPO_PATH}" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Build gitnexus if dist/cli/index.js is missing +# --------------------------------------------------------------------------- +if [[ ! -f "${GITNEXUS_CLI}" ]]; then + echo "gitnexus CLI not built. Building now..." + echo " -> ${GITNEXUS_DIR}" + + if [[ ! -d "${GITNEXUS_DIR}/node_modules" ]]; then + echo "Installing gitnexus dependencies..." + (cd "${GITNEXUS_DIR}" && npm install) + fi + + (cd "${GITNEXUS_DIR}" && npm run build) + + if [[ ! -f "${GITNEXUS_CLI}" ]]; then + echo "ERROR: Build completed but ${GITNEXUS_CLI} still not found." + exit 1 + fi + + echo "Build complete." +else + echo "gitnexus CLI already built: ${GITNEXUS_CLI}" +fi + +# --------------------------------------------------------------------------- +# Run analysis +# --------------------------------------------------------------------------- +echo "" +echo "Analyzing '${REPO_NAME}' at:" +echo " ${REPO_PATH}" +echo "" + +node "${GITNEXUS_CLI}" analyze "${REPO_PATH}" --name "${REPO_NAME}" + +# --------------------------------------------------------------------------- +# Post-analysis instructions +# --------------------------------------------------------------------------- +echo "" +echo "---------------------------------------------------------------------" +echo "Analysis complete for '${REPO_NAME}'." +echo "" +echo "To start the gitnexus server and query the index:" +echo "" +echo " cd ${GITNEXUS_DIR}" +echo " npm run serve" +echo "" +echo "The server listens on http://localhost:3001 by default." +echo "" +echo "To use it with the AITM MCP server:" +echo "" +echo " cd ${MONOREPO_ROOT}/aitimemachine/mcp" +echo " npm run dev:with-gitnexus" +echo "---------------------------------------------------------------------" diff --git a/scripts/deploy-gitgram.sh b/scripts/deploy-gitgram.sh new file mode 100755 index 00000000..733844c5 --- /dev/null +++ b/scripts/deploy-gitgram.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "scripts/deploy-gitgram.sh now wraps the PM2/manual launch flow." +echo "For multi-service deploys use scripts/deploy-launch.sh directly." +exec "${SCRIPT_DIR}/deploy-launch.sh" "$@" --gitgram-only diff --git a/scripts/deploy-launch.sh b/scripts/deploy-launch.sh new file mode 100755 index 00000000..c2d89d1e --- /dev/null +++ b/scripts/deploy-launch.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -euo pipefail + +ENVIRONMENT="${1:-}" +if [[ -z "${ENVIRONMENT}" || ! "${ENVIRONMENT}" =~ ^(staging|production)$ ]]; then + echo "Usage: $0 <staging|production> [--gitgram-only] [--aitm-only] [--skip-verify] [--verify-profile core|full] [--skip-install] [--skip-typecheck] [--dry-run]" >&2 + exit 1 +fi +shift || true + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GITGRAM_DIR="${ROOT_DIR}/gitgram" +AITM_DIR="${ROOT_DIR}/aitimemachine" + +GITGRAM_ONLY=0 +AITM_ONLY=0 +SKIP_VERIFY=0 +VERIFY_PROFILE="${VERIFY_PROFILE:-core}" +SKIP_INSTALL=0 +SKIP_TYPECHECK=0 +DRY_RUN=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --gitgram-only) GITGRAM_ONLY=1; shift ;; + --aitm-only) AITM_ONLY=1; shift ;; + --skip-verify) SKIP_VERIFY=1; shift ;; + --verify-profile) + VERIFY_PROFILE="${2:-}" + if [[ "${VERIFY_PROFILE}" != "core" && "${VERIFY_PROFILE}" != "full" ]]; then + echo "Invalid verify profile: ${VERIFY_PROFILE}" >&2 + exit 1 + fi + shift 2 + ;; + --skip-install) SKIP_INSTALL=1; shift ;; + --skip-typecheck) SKIP_TYPECHECK=1; shift ;; + --dry-run) DRY_RUN=1; shift ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +if [[ "${GITGRAM_ONLY}" == "1" && "${AITM_ONLY}" == "1" ]]; then + echo "Use either --gitgram-only or --aitm-only, not both." >&2 + exit 1 +fi + +run_cmd() { + if [[ "${DRY_RUN}" == "1" ]]; then + printf 'DRY_RUN ' + printf '%q ' "$@" + printf '\n' + return 0 + fi + "$@" +} + +run_bash() { + local cmd="$1" + if [[ "${DRY_RUN}" == "1" ]]; then + printf 'DRY_RUN bash -lc %q\n' "${cmd}" + return 0 + fi + bash -lc "${cmd}" +} + +log() { + printf '%s\n' "$1" +} + +resolve_env_file() { + local base="$1" + local env_name="$2" + local candidate="${base}.${env_name}" + + if [[ -f "${candidate}" ]]; then + printf '%s' "${candidate}" + return 0 + fi + + printf '%s' "${base}" +} + +check_cmd() { + local cmd="$1" + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Missing required command: ${cmd}" >&2 + exit 1 + fi +} + +wait_for_http() { + local label="$1" + local url="$2" + local retries="${3:-20}" + local sleep_seconds="${4:-2}" + + if [[ "${DRY_RUN}" == "1" ]]; then + log "DRY_RUN health check ${label}: ${url}" + return 0 + fi + + for _ in $(seq 1 "${retries}"); do + if curl -fsS --max-time 8 "${url}" >/dev/null 2>&1; then + log "PASS ${label}: ${url}" + return 0 + fi + sleep "${sleep_seconds}" + done + + echo "Health check failed for ${label}: ${url}" >&2 + return 1 +} + +pm2_apply() { + local cwd="$1" + local ecosystem_file="$2" + local app_name="$3" + shift 3 + + local source_lines="" + local env_file + for env_file in "$@"; do + source_lines="${source_lines}source \"${env_file}\"\n" + done + + local script + script="$(cat <<EOF +set -euo pipefail +set -a +$(printf '%b' "${source_lines}") +set +a +cd "${cwd}" +if pm2 describe "${app_name}" >/dev/null 2>&1; then + pm2 restart "${app_name}" --update-env +else + pm2 start "${ecosystem_file}" --only "${app_name}" +fi +pm2 save +EOF +)" + run_bash "${script}" +} + +log "=== Manual Launch Deploy ===" +log "Environment: ${ENVIRONMENT}" +log "Verify profile: ${VERIFY_PROFILE}" +log "Root: ${ROOT_DIR}" + +GITGRAM_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.gitgram" "${ENVIRONMENT}")" +GITGRAM_WEB_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.gitgram-web" "${ENVIRONMENT}")" +AITM_API_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.aitm-api" "${ENVIRONMENT}")" + +check_cmd bash +check_cmd node +check_cmd npm +check_cmd pm2 +check_cmd curl + +if [[ "${SKIP_VERIFY}" != "1" ]]; then + log "" + log "1) Verifying launch readiness" + run_cmd bash "${ROOT_DIR}/scripts/verify-launch-readiness.sh" "${ENVIRONMENT}" "${VERIFY_PROFILE}" +else + log "" + log "1) Skipping launch readiness verification" +fi + +if [[ "${AITM_ONLY}" != "1" ]]; then + log "" + log "2) Deploying Gitgram backend" + log "Gitgram backend env: ${GITGRAM_ENV_FILE}" + run_cmd mkdir -p "${GITGRAM_DIR}/logs" + if [[ "${SKIP_INSTALL}" != "1" ]]; then + run_bash "cd \"${GITGRAM_DIR}/backend\" && npm ci --legacy-peer-deps" + fi + run_bash "set -a; source \"${GITGRAM_ENV_FILE}\"; set +a; cd \"${GITGRAM_DIR}/backend\" && npx prisma generate && npx prisma migrate deploy && npm run build" + if [[ "${SKIP_TYPECHECK}" != "1" ]]; then + run_bash "cd \"${GITGRAM_DIR}/backend\" && npx tsc --noEmit" + fi + pm2_apply "${GITGRAM_DIR}" "ecosystem.config.js" "gitgram-api" "${GITGRAM_ENV_FILE}" + + log "" + log "3) Deploying Gitgram web" + log "Gitgram web env: ${GITGRAM_WEB_ENV_FILE}" + if [[ "${SKIP_INSTALL}" != "1" ]]; then + run_bash "cd \"${GITGRAM_DIR}/web\" && npm ci --legacy-peer-deps" + fi + run_bash "set -a; source \"${GITGRAM_WEB_ENV_FILE}\"; set +a; cd \"${GITGRAM_DIR}/web\" && npm run build" + if [[ "${SKIP_TYPECHECK}" != "1" ]]; then + run_bash "cd \"${GITGRAM_DIR}/web\" && npx tsc --noEmit" + fi + pm2_apply "${GITGRAM_DIR}" "ecosystem.config.js" "gitgram-web" "${GITGRAM_WEB_ENV_FILE}" + + log "" + log "4) Verifying Gitgram local health" + wait_for_http "Gitgram backend health" "http://127.0.0.1:4000/health" + wait_for_http "Gitgram backend readiness" "http://127.0.0.1:4000/health/ready" + wait_for_http "Gitgram web" "http://127.0.0.1:3000/" +fi + +if [[ "${GITGRAM_ONLY}" != "1" ]]; then + log "" + log "5) Deploying AITM API" + log "AITM API env: ${AITM_API_ENV_FILE}" + if [[ "${SKIP_INSTALL}" != "1" ]]; then + run_bash "cd \"${AITM_DIR}/api-server\" && npm ci" + fi + run_bash "set -a; source \"${AITM_API_ENV_FILE}\"; set +a; cd \"${AITM_DIR}/api-server\" && npm run db:migrate && npm run build" + if [[ "${SKIP_TYPECHECK}" != "1" ]]; then + run_bash "cd \"${AITM_DIR}\" && npx tsc --noEmit && cd api-server && npx tsc --noEmit" + fi + pm2_apply "${AITM_DIR}" "deploy/ecosystem.config.cjs" "aitm-api" "${AITM_API_ENV_FILE}" + + log "" + log "6) Verifying AITM local health" + wait_for_http "AITM API health" "http://127.0.0.1:3001/health" + wait_for_http "AITM API readiness" "http://127.0.0.1:3001/health/ready" +fi + +log "" +log "Manual launch deploy flow completed." diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 00000000..7dc30f75 --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# ============================================================================= +# dev.sh — Developer onboarding script for the Gitgram + AITM monorepo +# ============================================================================= +# Usage: +# bash scripts/dev.sh # full setup (infra + deps + migrations) +# bash scripts/dev.sh --infra # infra only (postgres + redis) +# bash scripts/dev.sh --help # show this help +# ============================================================================= +set -euo pipefail + +# ----------------------------------------------------------------------------- +# Colours +# ----------------------------------------------------------------------------- +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +RESET='\033[0m' + +ok() { echo -e " ${GREEN}✔${RESET} $*"; } +info() { echo -e " ${CYAN}→${RESET} $*"; } +warn() { echo -e " ${YELLOW}⚠${RESET} $*"; } +fail() { echo -e " ${RED}✘${RESET} $*"; exit 1; } +step() { echo -e "\n${BOLD}${CYAN}[dev.sh]${RESET} ${BOLD}$*${RESET}"; } + +# ----------------------------------------------------------------------------- +# Helpers +# ----------------------------------------------------------------------------- +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +infra_only=false +for arg in "$@"; do + case "$arg" in + --infra) infra_only=true ;; + --help|-h) + echo "Usage: bash scripts/dev.sh [--infra] [--help]" + echo " --infra Start Postgres + Redis only, skip npm/migrations" + exit 0 + ;; + esac +done + +# ----------------------------------------------------------------------------- +# 1. Prerequisite checks +# ----------------------------------------------------------------------------- +step "Checking prerequisites" + +# Node >= 20 +if command -v node &>/dev/null; then + NODE_MAJOR=$(node -e "process.stdout.write(process.versions.node.split('.')[0])") + if [[ "$NODE_MAJOR" -ge 20 ]]; then + ok "node $(node --version)" + else + fail "Node.js 20+ required (found $(node --version)). Use nvm: nvm install 20" + fi +else + fail "node not found. Install Node.js 20+: https://nodejs.org" +fi + +# npm +if command -v npm &>/dev/null; then + ok "npm $(npm --version)" +else + fail "npm not found" +fi + +# Docker +if command -v docker &>/dev/null; then + if docker info &>/dev/null 2>&1; then + ok "docker $(docker --version | awk '{print $3}' | tr -d ',')" + else + fail "Docker daemon is not running. Start Docker Desktop (or your daemon) and retry." + fi +else + fail "docker not found. Install Docker Desktop: https://docs.docker.com/get-docker/" +fi + +# docker compose (v2 plugin) +if docker compose version &>/dev/null 2>&1; then + ok "docker compose $(docker compose version --short 2>/dev/null || echo 'v2')" +else + fail "docker compose (v2) not found. Update Docker Desktop or install the plugin." +fi + +# git +if command -v git &>/dev/null; then + ok "git $(git --version | awk '{print $3}')" +else + warn "git not found — version control features will be limited." +fi + +# ----------------------------------------------------------------------------- +# 2. Copy .env.example → .env (never overwrites existing files) +# ----------------------------------------------------------------------------- +step "Setting up environment files" + +copy_env() { + local example="$REPO_ROOT/$1" + local target="$REPO_ROOT/$2" + if [[ ! -f "$example" ]]; then + warn "Example not found: $1 (skipping)" + return + fi + if [[ -f "$target" ]]; then + ok "$2 already exists — skipping" + else + cp "$example" "$target" + ok "Created $2 from $1" + warn " Edit $2 and fill in any required secrets before running services." + fi +} + +copy_env ".env.gitgram.example" ".env.gitgram" +copy_env ".env.gitgram-web.example" ".env.gitgram-web" +copy_env ".env.aitm-api.example" ".env.aitm-api" +copy_env ".env.aitm-website.example" ".env.aitm-website" +copy_env ".env.postgres.example" ".env.postgres" + +# ----------------------------------------------------------------------------- +# 3. Start infrastructure (postgres + redis) +# ----------------------------------------------------------------------------- +step "Starting infrastructure (postgres + redis)" + +cd "$REPO_ROOT" + +docker compose up -d postgres redis + +info "Waiting for postgres to be healthy…" +RETRIES=20 +until docker compose exec -T postgres pg_isready -U gitgram -d gitgram &>/dev/null; do + RETRIES=$((RETRIES - 1)) + if [[ $RETRIES -le 0 ]]; then + fail "Postgres did not become healthy after 100 s. Check: docker compose logs postgres" + fi + sleep 5 +done +ok "Postgres is healthy" + +info "Waiting for redis to be healthy…" +RETRIES=10 +until docker compose exec -T redis redis-cli ping 2>/dev/null | grep -q PONG; do + RETRIES=$((RETRIES - 1)) + if [[ $RETRIES -le 0 ]]; then + fail "Redis did not become healthy after 50 s. Check: docker compose logs redis" + fi + sleep 5 +done +ok "Redis is healthy" + +if $infra_only; then + echo "" + echo -e "${GREEN}${BOLD}Infrastructure is up.${RESET}" + echo -e " Postgres → ${CYAN}postgresql://gitgram:gitgram_dev@localhost:5432/gitgram${RESET}" + echo -e " Redis → ${CYAN}redis://localhost:6379${RESET}" + exit 0 +fi + +# ----------------------------------------------------------------------------- +# 4. Install npm dependencies +# ----------------------------------------------------------------------------- +step "Installing npm dependencies" + +PACKAGES=( + "gitgram/backend" + "gitgram/web" + "aitimemachine/api-server" + "aitimemachine/mcp" +) + +for pkg in "${PACKAGES[@]}"; do + pkg_path="$REPO_ROOT/$pkg" + if [[ -f "$pkg_path/package.json" ]]; then + info "npm ci → $pkg" + npm ci --prefix "$pkg_path" --silent + ok "$pkg" + else + warn "$pkg/package.json not found — skipping" + fi +done + +# ----------------------------------------------------------------------------- +# 5. Run database migrations +# ----------------------------------------------------------------------------- +step "Running database migrations" + +# Gitgram backend — Prisma +GITGRAM_BACKEND="$REPO_ROOT/gitgram/backend" +if [[ -f "$GITGRAM_BACKEND/package.json" ]]; then + info "Prisma migrate deploy → gitgram/backend" + DATABASE_URL="postgresql://gitgram:gitgram_dev@localhost:5432/gitgram" \ + npx --prefix "$GITGRAM_BACKEND" prisma migrate deploy 2>&1 | tail -5 + ok "Gitgram migrations applied" +else + warn "gitgram/backend not found — skipping Prisma migration" +fi + +# AITM API — Drizzle +AITM_API="$REPO_ROOT/aitimemachine/api-server" +if [[ -f "$AITM_API/package.json" ]]; then + AITM_MIGRATE_SCRIPT=$(node -e "const p=require('$AITM_API/package.json'); process.stdout.write(p.scripts&&p.scripts['db:migrate']?'db:migrate':p.scripts&&p.scripts['migrate']?'migrate':'')" 2>/dev/null || true) + if [[ -n "$AITM_MIGRATE_SCRIPT" ]]; then + info "npm run $AITM_MIGRATE_SCRIPT → aitimemachine/api-server" + DATABASE_URL="postgresql://gitgram:gitgram_dev@localhost:5432/gitgram" \ + npm run "$AITM_MIGRATE_SCRIPT" --prefix "$AITM_API" 2>&1 | tail -5 + ok "AITM migrations applied" + else + warn "No migrate script found in aitimemachine/api-server/package.json — skipping" + fi +else + warn "aitimemachine/api-server not found — skipping Drizzle migration" +fi + +# ----------------------------------------------------------------------------- +# Done +# ----------------------------------------------------------------------------- +echo "" +echo -e "${GREEN}${BOLD}Dev environment is ready.${RESET}" +echo "" +echo -e " ${BOLD}Infrastructure${RESET}" +echo -e " Postgres → ${CYAN}postgresql://gitgram:gitgram_dev@localhost:5432/gitgram${RESET}" +echo -e " Redis → ${CYAN}redis://localhost:6379${RESET}" +echo "" +echo -e " ${BOLD}Start all services${RESET}" +echo -e " ${CYAN}docker compose up${RESET}" +echo -e " ${CYAN}docker compose --profile tools up${RESET} (+ GitNexus + Context7)" +echo "" +echo -e " ${BOLD}Start individual services (native, hot-reload)${RESET}" +echo -e " ${CYAN}npm run dev:gitgram-backend${RESET} → http://localhost:4000" +echo -e " ${CYAN}npm run dev:gitgram-web${RESET} → http://localhost:3000" +echo -e " ${CYAN}npm run dev:aitm-api${RESET} → http://localhost:3001" +echo -e " ${CYAN}npm run dev:aitm-mcp${RESET} → stdio / http://localhost:3003" +echo "" +echo -e " ${BOLD}Tear down${RESET}" +echo -e " ${CYAN}docker compose down -v${RESET} (removes volumes too)" +echo "" diff --git a/scripts/setup-aitm-tools.sh b/scripts/setup-aitm-tools.sh new file mode 100755 index 00000000..c502339a --- /dev/null +++ b/scripts/setup-aitm-tools.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# setup-aitm-tools.sh +# Builds and configures all AITM companion tools: +# - beads (Go binary, external/beads/cmd/bd) +# - gitnexus (npm, external/gitnexus/gitnexus) +# - pal-mcp-server (Python venv, external/pal-mcp-server) +# - context7 mcp (pnpm/npm, external/context7/packages/mcp) +# +# Designed to be defensive: a failure in one tool does NOT abort the others. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +EXTERNAL="$ROOT/external" + +# ── colours ────────────────────────────────────────────────────────────────── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +RESET='\033[0m' + +ok() { echo -e "${GREEN}[ok]${RESET} $*"; } +warn() { echo -e "${YELLOW}[warn]${RESET} $*"; } +fail() { echo -e "${RED}[fail]${RESET} $*"; } +step() { echo -e "\n${CYAN}==> $*${RESET}"; } + +ERRORS=() + +# ── helpers ─────────────────────────────────────────────────────────────────── +require_dir() { + local dir="$1" name="$2" + if [[ ! -d "$dir" ]]; then + fail "$name directory not found: $dir" + return 1 + fi + return 0 +} + +# Run a named build function; record failure but continue +run_step() { + local name="$1"; shift + if "$@"; then + ok "$name done" + else + fail "$name FAILED (exit $?)" + ERRORS+=("$name") + fi +} + +# ── 1. beads (Go) ───────────────────────────────────────────────────────────── +build_beads() { + step "beads — Go binary (cmd/bd)" + + if ! command -v go &>/dev/null; then + warn "Go not found — skipping beads build" + warn "Install from https://go.dev/dl/" + return 0 + fi + + local beads_dir="$EXTERNAL/beads" + require_dir "$beads_dir" "beads" || return 1 + + # Prefer cmd/bd; fall back to cmd/beads or root package + local cmd_path="" + if [[ -f "$beads_dir/cmd/bd/main.go" ]]; then + cmd_path="./cmd/bd" + elif [[ -f "$beads_dir/cmd/beads/main.go" ]]; then + cmd_path="./cmd/beads" + else + warn "No cmd/bd or cmd/beads found — attempting root package build" + cmd_path="." + fi + + local out_bin="$beads_dir/bd" + ( + cd "$beads_dir" + go build -o "$out_bin" "$cmd_path" + ) + ok "beads binary at $out_bin" + echo " run: $out_bin --help" +} + +# ── 2. gitnexus (npm) ───────────────────────────────────────────────────────── +build_gitnexus() { + step "gitnexus — npm install + build" + + local gnx_dir="$EXTERNAL/gitnexus" + require_dir "$gnx_dir" "gitnexus" || return 1 + + # The main CLI lives in the gitnexus/ sub-package + local pkg_dir="$gnx_dir/gitnexus" + if [[ ! -d "$pkg_dir" ]]; then + warn "gitnexus/gitnexus sub-directory not found; trying root" + pkg_dir="$gnx_dir" + fi + + if ! command -v npm &>/dev/null; then + fail "npm not found — cannot build gitnexus" + return 1 + fi + + ( + cd "$pkg_dir" + echo " Installing dependencies..." + npm install --prefer-offline --no-fund --no-audit 2>&1 | tail -3 + echo " Building..." + npm run build 2>&1 | tail -5 + ) + ok "gitnexus built in $pkg_dir" +} + +# ── 3. pal-mcp-server (Python venv) ─────────────────────────────────────────── +setup_pal_mcp() { + step "pal-mcp-server — Python venv + requirements" + + local pal_dir="$EXTERNAL/pal-mcp-server" + require_dir "$pal_dir" "pal-mcp-server" || return 1 + + if ! command -v python3 &>/dev/null; then + fail "python3 not found — cannot set up pal-mcp-server" + fail "Install Python 3.10+ from https://python.org" + return 1 + fi + + local python_version + python_version="$(python3 --version 2>&1)" + echo " Using $python_version" + + local venv_dir="$pal_dir/.venv" + + if [[ ! -d "$venv_dir" ]]; then + echo " Creating venv at $venv_dir ..." + python3 -m venv "$venv_dir" + else + echo " venv already exists, reusing" + fi + + local pip="$venv_dir/bin/pip" + + if [[ -f "$pal_dir/requirements.txt" ]]; then + echo " Installing requirements.txt ..." + "$pip" install --quiet -r "$pal_dir/requirements.txt" + else + warn "requirements.txt not found in $pal_dir" + fi + + if [[ -f "$pal_dir/requirements-dev.txt" ]]; then + echo " Installing requirements-dev.txt ..." + "$pip" install --quiet -r "$pal_dir/requirements-dev.txt" + fi + + ok "pal-mcp-server venv ready at $venv_dir" + echo " activate: source $venv_dir/bin/activate" + echo " run: $venv_dir/bin/python $pal_dir/server.py" +} + +# ── 4. context7 packages/mcp ────────────────────────────────────────────────── +build_context7_mcp() { + step "context7 — packages/mcp build" + + local ctx7_dir="$EXTERNAL/context7" + require_dir "$ctx7_dir" "context7" || return 1 + + local mcp_dir="$ctx7_dir/packages/mcp" + require_dir "$mcp_dir" "context7/packages/mcp" || return 1 + + # Prefer pnpm (the repo uses pnpm workspaces), fall back to npm + local pkg_cmd + if command -v pnpm &>/dev/null; then + pkg_cmd="pnpm" + elif command -v npm &>/dev/null; then + pkg_cmd="npm" + warn "pnpm not found; falling back to npm (workspace installs may be incomplete)" + else + fail "Neither pnpm nor npm found — cannot build context7" + return 1 + fi + + ( + cd "$mcp_dir" + echo " Installing dependencies (${pkg_cmd})..." + "$pkg_cmd" install --prefer-offline 2>&1 | tail -3 + echo " Building..." + "$pkg_cmd" run build 2>&1 | tail -5 + ) + ok "context7/packages/mcp built" + echo " run: node $mcp_dir/dist/index.js --transport http" +} + +# ── main ────────────────────────────────────────────────────────────────────── +echo -e "\n${CYAN}AITM companion tools setup${RESET}" +echo "Repository root: $ROOT" +echo "External tools: $EXTERNAL" + +run_step "beads" build_beads +run_step "gitnexus" build_gitnexus +run_step "pal-mcp-server" setup_pal_mcp +run_step "context7-mcp" build_context7_mcp + +echo "" +echo "----------------------------------------------" +if [[ ${#ERRORS[@]} -eq 0 ]]; then + echo -e "${GREEN}All tools set up successfully.${RESET}" +else + echo -e "${RED}Setup completed with errors in: ${ERRORS[*]}${RESET}" + echo "Review the output above, fix the issues, then re-run." + exit 1 +fi + +echo "" +echo "Quick reference:" +echo " beads: $EXTERNAL/beads/bd --help" +echo " gitnexus: cd $EXTERNAL/gitnexus/gitnexus && npm run serve" +echo " pal-mcp-server: source $EXTERNAL/pal-mcp-server/.venv/bin/activate && python $EXTERNAL/pal-mcp-server/server.py" +echo " context7-mcp: node $EXTERNAL/context7/packages/mcp/dist/index.js --transport http" diff --git a/scripts/smoke-local-launch.sh b/scripts/smoke-local-launch.sh new file mode 100755 index 00000000..653bd1c7 --- /dev/null +++ b/scripts/smoke-local-launch.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +GITGRAM_BACKEND_HEALTH_URL="${GITGRAM_BACKEND_HEALTH_URL:-http://localhost:4000/health}" +GITGRAM_WEB_URL="${GITGRAM_WEB_URL:-http://localhost:3000}" +AITM_API_HEALTH_URL="${AITM_API_HEALTH_URL:-http://localhost:3001/health}" + +SKIP_HTTP="${SKIP_HTTP:-0}" +RUN_AITM_CLI_SMOKE="${RUN_AITM_CLI_SMOKE:-0}" +RUN_AITM_MCP_SMOKE="${RUN_AITM_MCP_SMOKE:-0}" +AUTO_START="${AUTO_START:-0}" + +log() { + printf '%s\n' "$1" +} + +check_cmd() { + local cmd="$1" + if ! command -v "${cmd}" >/dev/null 2>&1; then + log "Missing required command: ${cmd}" + exit 1 + fi +} + +check_http() { + local label="$1" + local url="$2" + if curl -fsS --max-time 8 "${url}" >/dev/null; then + log "PASS ${label}: ${url}" + else + log "FAIL ${label}: ${url}" + return 1 + fi +} + +start_services_if_requested() { + if [[ "${AUTO_START}" != "1" ]]; then + return 0 + fi + + check_cmd docker + log "" + log "Starting local services via docker compose (AUTO_START=1)" + ( + cd "${ROOT_DIR}" + docker compose up -d postgres redis gitgram-backend gitgram-web aitm-api + ) +} + +wait_for_http() { + local label="$1" + local url="$2" + local retries="${3:-30}" + local sleep_seconds="${4:-2}" + + for _ in $(seq 1 "${retries}"); do + if curl -fsS --max-time 8 "${url}" >/dev/null 2>&1; then + return 0 + fi + sleep "${sleep_seconds}" + done + + log "FAIL ${label}: ${url} did not become ready after ${retries} attempts" + return 1 +} + +log "=== Stage A Local Smoke Check ===" +log "Repo: ${ROOT_DIR}" + +check_cmd npm +check_cmd curl + +log "" +log "1) Validating local AITM env minimums" +npm --prefix "${ROOT_DIR}" run verify:aitm-local + +log "" +log "2) Verifying AITM npm package tarballs" +npm --prefix "${ROOT_DIR}" run verify:aitm-packages + +if [[ "${SKIP_HTTP}" == "1" ]]; then + log "" + log "3) HTTP health checks skipped (SKIP_HTTP=1)" +else + start_services_if_requested + log "" + log "3) Checking live local service health endpoints" + if [[ "${AUTO_START}" == "1" ]]; then + wait_for_http "Gitgram backend health" "${GITGRAM_BACKEND_HEALTH_URL}" + wait_for_http "Gitgram web" "${GITGRAM_WEB_URL}" + wait_for_http "AITM API health" "${AITM_API_HEALTH_URL}" + fi + check_http "Gitgram backend health" "${GITGRAM_BACKEND_HEALTH_URL}" + check_http "Gitgram web" "${GITGRAM_WEB_URL}" + check_http "AITM API health" "${AITM_API_HEALTH_URL}" +fi + +if [[ "${RUN_AITM_CLI_SMOKE}" == "1" ]]; then + log "" + log "4) Running AITM CLI smoke test" + bash "${ROOT_DIR}/aitimemachine/scripts/cli-smoke-test.sh" +fi + +if [[ "${RUN_AITM_MCP_SMOKE}" == "1" ]]; then + log "" + log "5) Running AITM MCP smoke test" + bash "${ROOT_DIR}/aitimemachine/scripts/mcp-smoke-test.sh" +fi + +log "" +log "Stage A local smoke checks completed." diff --git a/scripts/start-all-dev.sh b/scripts/start-all-dev.sh new file mode 100755 index 00000000..54d5caf4 --- /dev/null +++ b/scripts/start-all-dev.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# start-all-dev.sh +# Starts all core dev services for the Gitgram-AI monorepo: +# +# Service Port / URL +# ──────────────────────── ───────────────────────────── +# gitgram/backend http://localhost:3000 +# gitgram/web (Next.js) http://localhost:3001 (next dev default = 3000, +# but backend already owns that, so web uses 3001 +# via PORT env — or the Next.js default if unset) +# aitimemachine/api-server http://localhost:3001 (its own default) +# +# NOTE: gitgram/backend defaults to PORT 3000. +# aitimemachine/api-server defaults to PORT 3001. +# gitgram/web (Next.js) defaults to PORT 3000; to avoid clash it is +# started on PORT 3002 here. Override with GITGRAM_WEB_PORT env var. +# +# Uses `concurrently` when available (npm install -g concurrently). +# Falls back to printing manual commands. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +# ── colours ────────────────────────────────────────────────────────────────── +CYAN='\033[0;36m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RESET='\033[0m' + +# ── configurable ports ──────────────────────────────────────────────────────── +BACKEND_PORT="${GITGRAM_BACKEND_PORT:-3000}" +WEB_PORT="${GITGRAM_WEB_PORT:-3002}" +AITM_PORT="${AITM_API_PORT:-3001}" + +# ── service directories ─────────────────────────────────────────────────────── +BACKEND_DIR="$ROOT/gitgram/backend" +WEB_DIR="$ROOT/gitgram/web" +AITM_DIR="$ROOT/aitimemachine/api-server" + +# ── banner ──────────────────────────────────────────────────────────────────── +echo -e "\n${CYAN}Gitgram-AI — dev services${RESET}" +echo "" +echo " Service Command URL" +echo " ─────────────────────────────────────────────────────────────────────" +echo " gitgram/backend npm run dev http://localhost:${BACKEND_PORT}" +echo " gitgram/web (Next.js) npm run dev http://localhost:${WEB_PORT}" +echo " aitimemachine/api-server npm run dev http://localhost:${AITM_PORT}" +echo "" + +# ── preflight checks ───────────────────────────────────────────────────────── +MISSING=() +for pair in "gitgram/backend:$BACKEND_DIR" "gitgram/web:$WEB_DIR" "aitimemachine/api-server:$AITM_DIR"; do + name="${pair%%:*}" + dir="${pair#*:}" + if [[ ! -d "$dir" ]]; then + echo -e "${YELLOW}[warn]${RESET} Directory not found: $dir (${name} will be skipped)" + MISSING+=("$name") + fi +done + +if [[ ${#MISSING[@]} -gt 0 ]]; then + echo "" +fi + +# Build the npm dev commands with explicit PORT assignments +BACKEND_CMD="cd \"$BACKEND_DIR\" && PORT=$BACKEND_PORT npm run dev" +WEB_CMD="cd \"$WEB_DIR\" && PORT=$WEB_PORT npm run dev" +AITM_CMD="cd \"$AITM_DIR\" && PORT=$AITM_PORT npm run dev" + +# ── try concurrently ───────────────────────────────────────────────────────── +if command -v concurrently &>/dev/null; then + echo -e "${GREEN}Starting all services with concurrently...${RESET}" + echo "(Press Ctrl+C to stop all)" + echo "" + + # Build argument list; skip missing service dirs + CONC_ARGS=() + CONC_NAMES=() + + [[ ! " ${MISSING[*]} " =~ " gitgram/backend " ]] && { + CONC_ARGS+=("$BACKEND_CMD") + CONC_NAMES+=("backend") + } + [[ ! " ${MISSING[*]} " =~ " gitgram/web " ]] && { + CONC_ARGS+=("$WEB_CMD") + CONC_NAMES+=("web") + } + [[ ! " ${MISSING[*]} " =~ " aitimemachine/api-server " ]] && { + CONC_ARGS+=("$AITM_CMD") + CONC_NAMES+=("aitm-api") + } + + if [[ ${#CONC_ARGS[@]} -eq 0 ]]; then + echo "No service directories found. Nothing to start." + exit 1 + fi + + # Build --names flag + NAMES_FLAG="$(IFS=','; echo "${CONC_NAMES[*]}")" + + concurrently \ + --kill-others-on-fail \ + --prefix "[{name}]" \ + --names "$NAMES_FLAG" \ + --prefix-colors "cyan,magenta,green" \ + "${CONC_ARGS[@]}" + +else + # ── fallback: print manual commands ──────────────────────────────────────── + echo -e "${YELLOW}concurrently not found.${RESET}" + echo "Install it globally to start all services in one terminal:" + echo " npm install -g concurrently" + echo "" + echo "To start services manually, open three terminal windows and run:" + echo "" + echo " # Terminal 1 — gitgram/backend" + if [[ ! " ${MISSING[*]} " =~ " gitgram/backend " ]]; then + echo " cd $BACKEND_DIR" + echo " PORT=$BACKEND_PORT npm run dev" + else + echo " (directory not found: $BACKEND_DIR)" + fi + echo "" + echo " # Terminal 2 — gitgram/web" + if [[ ! " ${MISSING[*]} " =~ " gitgram/web " ]]; then + echo " cd $WEB_DIR" + echo " PORT=$WEB_PORT npm run dev" + else + echo " (directory not found: $WEB_DIR)" + fi + echo "" + echo " # Terminal 3 — aitimemachine/api-server" + if [[ ! " ${MISSING[*]} " =~ " aitimemachine/api-server " ]]; then + echo " cd $AITM_DIR" + echo " PORT=$AITM_PORT npm run dev" + else + echo " (directory not found: $AITM_DIR)" + fi + echo "" + echo "Port overrides (set before running this script):" + echo " GITGRAM_BACKEND_PORT=3000 (default)" + echo " GITGRAM_WEB_PORT=3002 (default)" + echo " AITM_API_PORT=3001 (default)" +fi diff --git a/scripts/validate-openapi-routes.ts b/scripts/validate-openapi-routes.ts new file mode 100644 index 00000000..ddca39f0 --- /dev/null +++ b/scripts/validate-openapi-routes.ts @@ -0,0 +1,423 @@ +#!/usr/bin/env node +/** + * OpenAPI Route Coverage Validator + * + * Parses Express route registrations from source code, compares them against + * the OpenAPI spec, and reports coverage gaps. + * + * Usage: + * npx tsx scripts/validate-openapi-routes.ts + * + * Exit codes: + * 0 - All routes covered + * 1 - Undocumented routes found + */ + +import * as fs from "fs"; +import * as path from "path"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface RouteInfo { + method: string; + path: string; + file: string; +} + +interface SpecRoute { + method: string; + path: string; +} + +interface ProductConfig { + name: string; + routeDir: string; + entryFile: string; + specFile: string; + basePrefix: string; +} + +// --------------------------------------------------------------------------- +// Route extraction from source code +// --------------------------------------------------------------------------- + +/** + * Extract route registrations from an Express router file. + * Matches patterns like: router.get("/path", ...) or router.post("/path", ...) + */ +function extractRoutesFromFile(filePath: string): Array<{ method: string; subPath: string }> { + const routes: Array<{ method: string; subPath: string }> = []; + + if (!fs.existsSync(filePath)) { + return routes; + } + + const content = fs.readFileSync(filePath, "utf-8"); + + // Match router.METHOD("path" or router.METHOD( followed by newline + "path" + const routeRegex = /router\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/gi; + + let match: RegExpExecArray | null; + while ((match = routeRegex.exec(content)) !== null) { + const method = match[1].toUpperCase(); + const subPath = match[2]; + routes.push({ method, subPath }); + } + + // Also match multiline patterns like router.get(\n "/path", + const multilineRegex = /router\.(get|post|put|patch|delete)\(\s*\n\s*["'`]([^"'`]+)["'`]/gi; + while ((match = multilineRegex.exec(content)) !== null) { + const method = match[1].toUpperCase(); + const subPath = match[2]; + // Avoid duplicates + if (!routes.some((r) => r.method === method && r.subPath === subPath)) { + routes.push({ method, subPath }); + } + } + + return routes; +} + +/** + * Parse the entry file (index.ts) to find route mount points. + * Matches patterns like: app.use("/v1/sync", syncRoutes); + */ +function extractMountPoints(entryFile: string): Map<string, string> { + const mounts = new Map<string, string>(); + + if (!fs.existsSync(entryFile)) { + return mounts; + } + + const content = fs.readFileSync(entryFile, "utf-8"); + + // Match import statements to map variable names to file names + const importMap = new Map<string, string>(); + const importRegex = /import\s+(\w+)\s+from\s+["'`]\.\/routes\/([^"'`]+)["'`]/g; + let match: RegExpExecArray | null; + while ((match = importRegex.exec(content)) !== null) { + importMap.set(match[1], match[2].replace(/\.js$/, "")); + } + + // Also handle CommonJS requires + const requireRegex = /(?:const|let|var)\s+(\w+)\s*=\s*require\(\s*["'`]\.\/routes\/([^"'`]+)["'`]\s*\)/g; + while ((match = requireRegex.exec(content)) !== null) { + importMap.set(match[1], match[2].replace(/\.js$/, "")); + } + + // Match app.use("/path", variableName) + const useRegex = /app\.use\(\s*["'`]([^"'`]+)["'`]\s*,\s*(?:[^,)]*,\s*)*(\w+Routes?)\s*\)/g; + while ((match = useRegex.exec(content)) !== null) { + const mountPath = match[1]; + const varName = match[2]; + const routeFile = importMap.get(varName); + if (routeFile) { + // Use the first mount point for each route file (prefer v1 routes) + if (!mounts.has(routeFile) || mountPath.startsWith("/v1") || mountPath.startsWith("/api/v1")) { + mounts.set(routeFile, mountPath); + } + } + } + + return mounts; +} + +/** + * Build the full list of implemented routes for a product. + */ +function getImplementedRoutes(config: ProductConfig): RouteInfo[] { + const routes: RouteInfo[] = []; + const mounts = extractMountPoints(config.entryFile); + + // Also handle inline routes defined directly in the entry file + const entryContent = fs.existsSync(config.entryFile) + ? fs.readFileSync(config.entryFile, "utf-8") + : ""; + + // Match app.get/post directly in entry file + const inlineRegex = /app\.(get|post|put|patch|delete)\(\s*["'`]([^"'`]+)["'`]/gi; + let inlineMatch: RegExpExecArray | null; + while ((inlineMatch = inlineRegex.exec(entryContent)) !== null) { + const method = inlineMatch[1].toUpperCase(); + const fullPath = inlineMatch[2]; + routes.push({ method, path: fullPath, file: path.basename(config.entryFile) }); + } + + // Get route files + if (!fs.existsSync(config.routeDir)) { + console.error(`Route directory not found: ${config.routeDir}`); + return routes; + } + + const routeFiles = fs.readdirSync(config.routeDir).filter((f) => f.endsWith(".ts")); + + for (const file of routeFiles) { + const filePath = path.join(config.routeDir, file); + const routeName = file.replace(/\.ts$/, ""); + const mountPath = mounts.get(routeName); + + if (!mountPath) { + // Try to find mount via various naming conventions + continue; + } + + const fileRoutes = extractRoutesFromFile(filePath); + + for (const { method, subPath } of fileRoutes) { + // Build full path + let fullPath = mountPath; + if (subPath === "/") { + // Root of the mount + } else { + fullPath = mountPath + subPath; + } + + // Normalize: remove trailing slash, collapse double slashes + fullPath = fullPath.replace(/\/+/g, "/").replace(/\/$/, "") || "/"; + + routes.push({ method, path: fullPath, file }); + } + } + + return routes; +} + +// --------------------------------------------------------------------------- +// OpenAPI spec parsing +// --------------------------------------------------------------------------- + +/** + * Parse OpenAPI YAML spec to extract paths. + * Simple YAML parser for our specific use case (no external deps). + */ +function parseOpenAPIRoutes(specFile: string): SpecRoute[] { + if (!fs.existsSync(specFile)) { + console.error(`OpenAPI spec not found: ${specFile}`); + return []; + } + + const content = fs.readFileSync(specFile, "utf-8"); + const routes: SpecRoute[] = []; + const lines = content.split("\n"); + + let currentPath = ""; + let inPaths = false; + let pathIndent = -1; + let methodIndent = -1; + + for (const line of lines) { + const trimmed = line.trimEnd(); + + // Detect paths: section + if (/^paths:\s*$/.test(trimmed)) { + inPaths = true; + pathIndent = -1; + continue; + } + + if (!inPaths) continue; + + // If we hit a top-level key that's not a path, end paths section + if (/^\S/.test(line) && !line.startsWith(" ") && !line.startsWith("#")) { + inPaths = false; + continue; + } + + // Match path entries (indented with spaces) + const pathMatch = trimmed.match(/^(\s+)(\/[^:]*)\s*:\s*$/); + if (pathMatch) { + const indent = pathMatch[1].length; + if (pathIndent === -1 || indent <= pathIndent) { + pathIndent = indent; + currentPath = pathMatch[2].trim(); + methodIndent = -1; + continue; + } + } + + // Match HTTP methods under a path + if (currentPath) { + const methodMatch = trimmed.match(/^(\s+)(get|post|put|patch|delete)\s*:\s*$/); + if (methodMatch) { + const indent = methodMatch[1].length; + if (indent > pathIndent) { + routes.push({ + method: methodMatch[2].toUpperCase(), + path: currentPath, + }); + } + } + } + } + + return routes; +} + +// --------------------------------------------------------------------------- +// Normalization and comparison +// --------------------------------------------------------------------------- + +/** + * Normalize a route path for comparison. + * Express :param becomes {param}, wildcards are handled, etc. + */ +function normalizePath(routePath: string): string { + return routePath + .replace(/:(\w+)/g, "{$1}") // :param -> {param} + .replace(/\/\*/g, "/{path}") // wildcard + .replace(/\/+/g, "/") // collapse slashes + .replace(/\/$/, "") // remove trailing slash + .toLowerCase(); +} + +/** + * Create a route key for comparison (METHOD:normalizedPath) + */ +function routeKey(method: string, routePath: string): string { + return `${method}:${normalizePath(routePath)}`; +} + +// --------------------------------------------------------------------------- +// Report generation +// --------------------------------------------------------------------------- + +function printTable(title: string, rows: Array<{ method: string; path: string; detail?: string }>) { + if (rows.length === 0) return; + + console.log(`\n${"=".repeat(80)}`); + console.log(` ${title}`); + console.log(`${"=".repeat(80)}`); + console.log(` ${"Method".padEnd(10)} ${"Path".padEnd(55)} ${rows[0].detail ? "Detail" : ""}`); + console.log(` ${"-".repeat(10)} ${"-".repeat(55)} ${rows[0].detail ? "-".repeat(10) : ""}`); + + for (const row of rows) { + console.log(` ${row.method.padEnd(10)} ${row.path.padEnd(55)} ${row.detail || ""}`); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +function validateProduct(config: ProductConfig): { undocumented: number; stale: number; covered: number } { + console.log(`\n${"#".repeat(80)}`); + console.log(` Validating: ${config.name}`); + console.log(`${"#".repeat(80)}`); + + // Get implemented routes + const implemented = getImplementedRoutes(config); + console.log(` Found ${implemented.length} implemented routes`); + + // Get spec routes + const specRoutes = parseOpenAPIRoutes(config.specFile); + console.log(` Found ${specRoutes.length} routes in OpenAPI spec`); + + // Build lookup sets + const implKeys = new Set(implemented.map((r) => routeKey(r.method, r.path))); + const specKeys = new Set(specRoutes.map((r) => routeKey(r.method, r.path))); + + // Find undocumented routes (in code but not spec) + const undocumented = implemented.filter((r) => !specKeys.has(routeKey(r.method, r.path))); + + // Find stale routes (in spec but not code) + const stale = specRoutes.filter((r) => !implKeys.has(routeKey(r.method, r.path))); + + // Find covered routes (in both) + const covered = implemented.filter((r) => specKeys.has(routeKey(r.method, r.path))); + + // Print results + const total = new Set([...implKeys, ...specKeys]).size; + const coveragePercent = total > 0 ? ((covered.length / total) * 100).toFixed(1) : "0"; + + console.log(`\n Coverage: ${covered.length}/${total} routes (${coveragePercent}%)`); + + if (undocumented.length > 0) { + printTable( + `UNDOCUMENTED (${undocumented.length}) - Routes in code but NOT in spec`, + undocumented.map((r) => ({ method: r.method, path: r.path, detail: r.file })) + ); + } else { + console.log(`\n All implemented routes are documented in the OpenAPI spec.`); + } + + if (stale.length > 0) { + printTable( + `STALE (${stale.length}) - Routes in spec but NOT in code`, + stale.map((r) => ({ method: r.method, path: r.path })) + ); + } + + if (covered.length > 0) { + printTable( + `COVERED (${covered.length}) - Routes in both code and spec`, + covered.map((r) => ({ method: r.method, path: r.path })) + ); + } + + return { + undocumented: undocumented.length, + stale: stale.length, + covered: covered.length, + }; +} + +// --------------------------------------------------------------------------- +// Product configurations +// --------------------------------------------------------------------------- + +const rootDir = path.resolve(__dirname, ".."); + +const products: ProductConfig[] = [ + { + name: "AITimeMachine API", + routeDir: path.join(rootDir, "aitimemachine/api-server/src/routes"), + entryFile: path.join(rootDir, "aitimemachine/api-server/src/index.ts"), + specFile: path.join(rootDir, "aitimemachine/api-server/docs/openapi.yaml"), + basePrefix: "/v1", + }, + { + name: "Gitgram Backend", + routeDir: path.join(rootDir, "gitgram/backend/src/routes"), + entryFile: path.join(rootDir, "gitgram/backend/src/index.ts"), + specFile: path.join(rootDir, "gitgram/backend/docs/openapi.yaml"), + basePrefix: "/api/v1", + }, +]; + +// Run validation +let hasFailures = false; +const results: Array<{ name: string; undocumented: number; stale: number; covered: number }> = []; + +for (const product of products) { + const result = validateProduct(product); + results.push({ name: product.name, ...result }); + + if (result.undocumented > 0) { + hasFailures = true; + } +} + +// Print summary +console.log(`\n${"=".repeat(80)}`); +console.log(" SUMMARY"); +console.log(`${"=".repeat(80)}`); +console.log(` ${"Product".padEnd(30)} ${"Covered".padEnd(12)} ${"Undocumented".padEnd(15)} ${"Stale".padEnd(10)}`); +console.log(` ${"-".repeat(30)} ${"-".repeat(12)} ${"-".repeat(15)} ${"-".repeat(10)}`); + +for (const r of results) { + const total = r.covered + r.undocumented; + const pct = total > 0 ? `(${((r.covered / total) * 100).toFixed(0)}%)` : ""; + console.log( + ` ${r.name.padEnd(30)} ${(r.covered + " " + pct).padEnd(12)} ${String(r.undocumented).padEnd(15)} ${String(r.stale).padEnd(10)}` + ); +} + +if (hasFailures) { + console.log(`\n FAIL: Undocumented routes found. Update OpenAPI specs to achieve full coverage.\n`); + process.exit(1); +} else { + console.log(`\n PASS: All implemented routes are documented.\n`); + process.exit(0); +} diff --git a/scripts/validate-openapi-spec.ts b/scripts/validate-openapi-spec.ts new file mode 100644 index 00000000..2e203fe8 --- /dev/null +++ b/scripts/validate-openapi-spec.ts @@ -0,0 +1,498 @@ +#!/usr/bin/env node +/** + * OpenAPI Spec Validator + * + * Validates OpenAPI YAML files for: + * - Valid YAML syntax + * - Required OpenAPI structure (info, paths, etc.) + * - Common issues: missing descriptions, missing response codes + * - Consistent naming conventions + * - Resolvable schema references + * + * Usage: + * npx tsx scripts/validate-openapi-spec.ts + * + * Exit codes: + * 0 - All specs valid + * 1 - Validation errors found + */ + +import * as fs from "fs"; +import * as path from "path"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ValidationIssue { + severity: "error" | "warning"; + path: string; + message: string; +} + +interface ParsedSpec { + openapi?: string; + info?: { title?: string; version?: string; description?: string }; + paths?: Record<string, Record<string, any>>; + components?: { + schemas?: Record<string, any>; + securitySchemes?: Record<string, any>; + }; +} + +// --------------------------------------------------------------------------- +// Simple YAML parser (enough for OpenAPI validation without dependencies) +// --------------------------------------------------------------------------- + +function parseYAML(content: string): ParsedSpec { + const result: any = {}; + const lines = content.split("\n"); + const stack: Array<{ indent: number; obj: any; key?: string }> = [ + { indent: -1, obj: result }, + ]; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Skip empty lines and comments + if (/^\s*$/.test(line) || /^\s*#/.test(line)) continue; + + const indentMatch = line.match(/^(\s*)/); + const indent = indentMatch ? indentMatch[1].length : 0; + const trimmed = line.trim(); + + // Pop stack until we find the right parent + while (stack.length > 1 && indent <= stack[stack.length - 1].indent) { + stack.pop(); + } + + const parent = stack[stack.length - 1]; + + // Match key: value + const kvMatch = trimmed.match(/^([^:]+?):\s*(.*)$/); + if (kvMatch) { + const key = kvMatch[1].trim().replace(/^["']|["']$/g, ""); + const value = kvMatch[2].trim(); + + if (value === "" || value === "|" || value === ">") { + // Object or block scalar + const newObj: any = {}; + if (Array.isArray(parent.obj)) { + parent.obj.push({ [key]: newObj }); + } else { + parent.obj[key] = newObj; + } + stack.push({ indent, obj: newObj, key }); + } else { + // Simple value + let parsedValue: any = value; + if (value === "true") parsedValue = true; + else if (value === "false") parsedValue = false; + else if (/^\d+$/.test(value)) parsedValue = parseInt(value, 10); + else parsedValue = value.replace(/^["']|["']$/g, ""); + + if (Array.isArray(parent.obj)) { + parent.obj.push({ [key]: parsedValue }); + } else { + parent.obj[key] = parsedValue; + } + } + } else if (trimmed.startsWith("- ")) { + // Array item + const itemValue = trimmed.slice(2).trim(); + if (!Array.isArray(parent.obj)) { + // Convert parent's value to array + const parentParent = stack.length > 1 ? stack[stack.length - 2] : null; + if (parentParent && parent.key) { + const arr: any[] = []; + parentParent.obj[parent.key] = arr; + parent.obj = arr; + } + } + if (Array.isArray(parent.obj)) { + parent.obj.push(itemValue.replace(/^["']|["']$/g, "")); + } + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Validation rules +// --------------------------------------------------------------------------- + +function validateSpecStructure(content: string, filePath: string): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const fileName = path.basename(filePath); + + // Check YAML can be parsed without errors + let spec: ParsedSpec; + try { + spec = parseYAML(content); + } catch (err: any) { + issues.push({ + severity: "error", + path: fileName, + message: `YAML parse error: ${err.message}`, + }); + return issues; + } + + // Check required top-level fields + if (!spec.openapi) { + issues.push({ + severity: "error", + path: `${fileName}/openapi`, + message: "Missing required field: openapi (version string)", + }); + } else if (!spec.openapi.toString().startsWith("3.")) { + issues.push({ + severity: "warning", + path: `${fileName}/openapi`, + message: `OpenAPI version ${spec.openapi} may not be widely supported. Consider 3.0.x or 3.1.x`, + }); + } + + if (!spec.info) { + issues.push({ + severity: "error", + path: `${fileName}/info`, + message: "Missing required field: info", + }); + } else { + if (!spec.info.title) { + issues.push({ + severity: "error", + path: `${fileName}/info/title`, + message: "Missing required field: info.title", + }); + } + if (!spec.info.version) { + issues.push({ + severity: "error", + path: `${fileName}/info/version`, + message: "Missing required field: info.version", + }); + } + if (!spec.info.description) { + issues.push({ + severity: "warning", + path: `${fileName}/info/description`, + message: "Missing recommended field: info.description", + }); + } + } + + if (!spec.paths || Object.keys(spec.paths).length === 0) { + issues.push({ + severity: "error", + path: `${fileName}/paths`, + message: "No paths defined in spec", + }); + } + + return issues; +} + +function validatePaths(content: string, filePath: string): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const fileName = path.basename(filePath); + const lines = content.split("\n"); + + let inPaths = false; + let currentPath = ""; + let currentMethod = ""; + let pathIndent = -1; + let hasResponses = false; + let hasSummary = false; + let hasOperationId = false; + let methodStartLine = 0; + + const operationIds = new Set<string>(); + const pathsFound = new Set<string>(); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + + if (/^paths:\s*$/.test(trimmed)) { + inPaths = true; + pathIndent = -1; + continue; + } + + if (!inPaths) continue; + + // End of paths section + if (/^\S/.test(line) && !line.startsWith(" ") && !line.startsWith("#") && trimmed !== "") { + inPaths = false; + continue; + } + + // Detect path entry + const pathMatch = trimmed.match(/^(\/[^:]*)\s*:\s*$/); + if (pathMatch && line.match(/^\s{2}\//)) { + // Save previous method check + if (currentMethod && !hasResponses) { + issues.push({ + severity: "warning", + path: `${fileName}#${currentPath}/${currentMethod}`, + message: `Missing responses for ${currentMethod.toUpperCase()} ${currentPath}`, + }); + } + + currentPath = pathMatch[1].trim(); + currentMethod = ""; + hasResponses = false; + hasSummary = false; + hasOperationId = false; + + if (pathsFound.has(currentPath)) { + issues.push({ + severity: "error", + path: `${fileName}#${currentPath}`, + message: `Duplicate path: ${currentPath}`, + }); + } + pathsFound.add(currentPath); + + // Check path naming convention + if (/[A-Z]/.test(currentPath) && !currentPath.includes("{")) { + issues.push({ + severity: "warning", + path: `${fileName}#${currentPath}`, + message: `Path contains uppercase characters. Use lowercase with hyphens for consistency.`, + }); + } + + continue; + } + + // Detect HTTP method + const methodMatch = trimmed.match(/^(get|post|put|patch|delete)\s*:\s*$/); + if (methodMatch && currentPath) { + // Save previous method check + if (currentMethod && !hasResponses) { + issues.push({ + severity: "warning", + path: `${fileName}#${currentPath}/${currentMethod}`, + message: `Missing responses for ${currentMethod.toUpperCase()} ${currentPath}`, + }); + } + + currentMethod = methodMatch[1]; + methodStartLine = i; + hasResponses = false; + hasSummary = false; + hasOperationId = false; + continue; + } + + // Check for summary + if (currentMethod && trimmed.startsWith("summary:")) { + hasSummary = true; + const summary = trimmed.replace(/^summary:\s*/, "").replace(/^["']|["']$/g, ""); + if (summary.length === 0) { + issues.push({ + severity: "warning", + path: `${fileName}#${currentPath}/${currentMethod}`, + message: `Empty summary for ${currentMethod.toUpperCase()} ${currentPath}`, + }); + } + } + + // Check for operationId + if (currentMethod && trimmed.startsWith("operationId:")) { + hasOperationId = true; + const opId = trimmed.replace(/^operationId:\s*/, "").replace(/^["']|["']$/g, ""); + if (operationIds.has(opId)) { + issues.push({ + severity: "error", + path: `${fileName}#${currentPath}/${currentMethod}`, + message: `Duplicate operationId: ${opId}`, + }); + } + operationIds.add(opId); + } + + // Check for responses + if (currentMethod && trimmed === "responses:") { + hasResponses = true; + } + } + + // Check last method + if (currentMethod && !hasResponses) { + issues.push({ + severity: "warning", + path: `${fileName}#${currentPath}/${currentMethod}`, + message: `Missing responses for ${currentMethod.toUpperCase()} ${currentPath}`, + }); + } + + return issues; +} + +function validateSchemaRefs(content: string, filePath: string): ValidationIssue[] { + const issues: ValidationIssue[] = []; + const fileName = path.basename(filePath); + + // Find all $ref references + const refRegex = /\$ref:\s*["']?#\/components\/schemas\/(\w+)["']?/g; + let match: RegExpExecArray | null; + const referencedSchemas = new Set<string>(); + + while ((match = refRegex.exec(content)) !== null) { + referencedSchemas.add(match[1]); + } + + // Find all defined schemas + const definedSchemas = new Set<string>(); + const schemaDefRegex = /^\s{4}(\w+):\s*$/gm; + let inComponents = false; + let inSchemas = false; + + const lines = content.split("\n"); + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === "components:") { + inComponents = true; + continue; + } + if (inComponents && trimmed === "schemas:") { + inSchemas = true; + continue; + } + if (inSchemas && /^\s{4}\w+:/.test(line)) { + const schemaName = line.trim().replace(/:.*$/, ""); + definedSchemas.add(schemaName); + } + // End of schemas section + if (inSchemas && /^\s{2}\w+:/.test(line) && !line.trim().startsWith("-")) { + if (!line.match(/^\s{4}/)) { + inSchemas = false; + } + } + } + + // Check for unresolvable references + for (const ref of referencedSchemas) { + if (!definedSchemas.has(ref)) { + issues.push({ + severity: "error", + path: `${fileName}/components/schemas/${ref}`, + message: `Unresolvable schema reference: #/components/schemas/${ref}`, + }); + } + } + + // Check for unused schemas (warning only) + for (const schema of definedSchemas) { + if (!referencedSchemas.has(schema)) { + issues.push({ + severity: "warning", + path: `${fileName}/components/schemas/${schema}`, + message: `Schema "${schema}" is defined but never referenced`, + }); + } + } + + return issues; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +interface SpecConfig { + name: string; + file: string; +} + +const rootDir = path.resolve(__dirname, ".."); + +const specs: SpecConfig[] = [ + { + name: "AITimeMachine API", + file: path.join(rootDir, "aitimemachine/api-server/docs/openapi.yaml"), + }, + { + name: "Gitgram Backend", + file: path.join(rootDir, "gitgram/backend/docs/openapi.yaml"), + }, +]; + +let totalErrors = 0; +let totalWarnings = 0; + +for (const spec of specs) { + console.log(`\n${"=".repeat(80)}`); + console.log(` Validating: ${spec.name}`); + console.log(` File: ${spec.file}`); + console.log(`${"=".repeat(80)}`); + + if (!fs.existsSync(spec.file)) { + console.log(` ERROR: File not found: ${spec.file}`); + totalErrors++; + continue; + } + + const content = fs.readFileSync(spec.file, "utf-8"); + + // Run all validation checks + const allIssues: ValidationIssue[] = [ + ...validateSpecStructure(content, spec.file), + ...validatePaths(content, spec.file), + ...validateSchemaRefs(content, spec.file), + ]; + + const errors = allIssues.filter((i) => i.severity === "error"); + const warnings = allIssues.filter((i) => i.severity === "warning"); + + totalErrors += errors.length; + totalWarnings += warnings.length; + + if (errors.length > 0) { + console.log(`\n ERRORS (${errors.length}):`); + for (const issue of errors) { + console.log(` [ERROR] ${issue.path}`); + console.log(` ${issue.message}`); + } + } + + if (warnings.length > 0) { + console.log(`\n WARNINGS (${warnings.length}):`); + for (const issue of warnings) { + console.log(` [WARN] ${issue.path}`); + console.log(` ${issue.message}`); + } + } + + if (errors.length === 0 && warnings.length === 0) { + console.log(`\n All checks passed.`); + } + + // Count paths and methods + const pathRegex = /^\s{2}\/[^:]+:\s*$/gm; + const methodRegex = /^\s{4}(get|post|put|patch|delete):\s*$/gm; + const pathCount = (content.match(pathRegex) || []).length; + const methodCount = (content.match(methodRegex) || []).length; + + console.log(`\n Stats: ${pathCount} paths, ${methodCount} operations`); +} + +// Summary +console.log(`\n${"=".repeat(80)}`); +console.log(" VALIDATION SUMMARY"); +console.log(`${"=".repeat(80)}`); +console.log(` Errors: ${totalErrors}`); +console.log(` Warnings: ${totalWarnings}`); + +if (totalErrors > 0) { + console.log(`\n FAIL: ${totalErrors} error(s) found. Fix errors before merging.\n`); + process.exit(1); +} else { + console.log(`\n PASS: All OpenAPI specs are structurally valid.\n`); + process.exit(0); +} diff --git a/scripts/validate-prod-secrets.sh b/scripts/validate-prod-secrets.sh new file mode 100755 index 00000000..f935ffa1 --- /dev/null +++ b/scripts/validate-prod-secrets.sh @@ -0,0 +1,382 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_ENV="${1:-production}" +if [[ "${TARGET_ENV}" != "production" && "${TARGET_ENV}" != "staging" && "${TARGET_ENV}" != "local" ]]; then + echo "Usage: $0 [production|staging|local] [core|full]" >&2 + exit 1 +fi + +VALIDATION_PROFILE="${2:-full}" +if [[ "${VALIDATION_PROFILE}" != "core" && "${VALIDATION_PROFILE}" != "full" ]]; then + echo "Usage: $0 [production|staging|local] [core|full]" >&2 + exit 1 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +resolve_env_file() { + local base="$1" + local env_name="$2" + local candidate="${base}.${env_name}" + + if [[ -f "${candidate}" ]]; then + printf '%s' "${candidate}" + return 0 + fi + + printf '%s' "${base}" +} + +MISSING=() +INVALID=() + +if [[ "${TARGET_ENV}" == "local" ]]; then + AITM_ENV_FILE="${ROOT_DIR}/aitimemachine/api-server/.env" + if [[ -f "${AITM_ENV_FILE}" ]]; then + set -a + # shellcheck source=/dev/null + source "${AITM_ENV_FILE}" + set +a + fi +fi + +load_prefixed_env_file() { + local product_prefix="$1" + local file="$2" + + if [[ ! -f "${file}" ]]; then + return 0 + fi + + while IFS='=' read -r key _; do + if [[ ! "${key}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + continue + fi + + local prefixed="${product_prefix}_${key}" + if [[ -n "${!prefixed:-}" ]]; then + continue + fi + + local value + value="$( + bash -c ' + set -a + source "$1" + printf "%s" "${!2}" + ' _ "${file}" "${key}" + )" + + export "${prefixed}=${value}" + done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "${file}" || true) +} + +if [[ "${TARGET_ENV}" != "local" ]]; then + GITGRAM_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.gitgram" "${TARGET_ENV}")" + GITGRAM_WEB_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.gitgram-web" "${TARGET_ENV}")" + AITM_API_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.aitm-api" "${TARGET_ENV}")" + AITM_WEBSITE_ENV_FILE="$(resolve_env_file "${ROOT_DIR}/.env.aitm-website" "${TARGET_ENV}")" + + load_prefixed_env_file "GITGRAM" "${GITGRAM_ENV_FILE}" + load_prefixed_env_file "GITGRAM" "${GITGRAM_WEB_ENV_FILE}" + load_prefixed_env_file "AITM" "${AITM_API_ENV_FILE}" + load_prefixed_env_file "AITM" "${AITM_WEBSITE_ENV_FILE}" +fi + +add_missing() { + MISSING+=("$1") +} + +add_invalid() { + INVALID+=("$1") +} + +resolve_value() { + local product_prefix="$1" + local key="$2" + local prefixed="${product_prefix}_${key}" + + if [[ -n "${!prefixed:-}" ]]; then + printf '%s' "${!prefixed}" + return 0 + fi + + if [[ -n "${!key:-}" ]]; then + printf '%s' "${!key}" + return 0 + fi + + printf '' +} + +validate_not_placeholder() { + local label="$1" + local value="$2" + local lowered + lowered="$(printf '%s' "${value}" | tr '[:upper:]' '[:lower:]')" + + local patterns=( + "change_me" + "replace-with" + "your_" + "your-" + "example" + "placeholder" + "xxx" + "todo" + "user:password@" + "project.neon.tech" + ) + + for pattern in "${patterns[@]}"; do + if [[ "${lowered}" == *"${pattern}"* ]]; then + add_invalid "${label} looks like a placeholder value" + return + fi + done +} + +assert_allowed_value() { + local product_prefix="$1" + local key="$2" + shift 2 + local allowed=("$@") + local label="${product_prefix}_${key}" + local value + value="$(resolve_value "${product_prefix}" "${key}")" + + if [[ -z "${value}" ]]; then + return 0 + fi + + for candidate in "${allowed[@]}"; do + if [[ "${value}" == "${candidate}" ]]; then + return 0 + fi + done + + add_invalid "${label} must be one of: ${allowed[*]} (found ${value})" +} + +assert_required() { + local product_prefix="$1" + local key="$2" + local label="${product_prefix}_${key}" + local value + value="$(resolve_value "${product_prefix}" "${key}")" + + if [[ -z "${value}" ]]; then + add_missing "${label} (or unprefixed ${key})" + return + fi + + validate_not_placeholder "${label}" "${value}" +} + +assert_one_of() { + local product_prefix="$1" + local key_a="$2" + local key_b="$3" + local label_a="${product_prefix}_${key_a}" + local label_b="${product_prefix}_${key_b}" + local value_a value_b + value_a="$(resolve_value "${product_prefix}" "${key_a}")" + value_b="$(resolve_value "${product_prefix}" "${key_b}")" + + if [[ -z "${value_a}" && -z "${value_b}" ]]; then + add_missing "one of ${label_a} or ${label_b} (or unprefixed variants)" + return + fi + + if [[ -n "${value_a}" ]]; then + validate_not_placeholder "${label_a}" "${value_a}" + fi + if [[ -n "${value_b}" ]]; then + validate_not_placeholder "${label_b}" "${value_b}" + fi +} + +assert_example_has_keys() { + local file="$1" + shift + local keys=("$@") + local missing_local=() + + for key in "${keys[@]}"; do + if ! rg -q "^[[:space:]]*${key}=" "${file}"; then + missing_local+=("${key}") + fi + done + + if [[ "${#missing_local[@]}" -gt 0 ]]; then + add_invalid "Missing keys in ${file}: ${missing_local[*]}" + fi +} + +assert_pair_if_either_present() { + local product_prefix="$1" + local key_a="$2" + local key_b="$3" + local label_a="${product_prefix}_${key_a}" + local label_b="${product_prefix}_${key_b}" + local value_a value_b + value_a="$(resolve_value "${product_prefix}" "${key_a}")" + value_b="$(resolve_value "${product_prefix}" "${key_b}")" + + if [[ -z "${value_a}" && -z "${value_b}" ]]; then + return 0 + fi + + if [[ -z "${value_a}" ]]; then + add_missing "${label_a} (required when ${label_b} is set)" + else + validate_not_placeholder "${label_a}" "${value_a}" + fi + + if [[ -z "${value_b}" ]]; then + add_missing "${label_b} (required when ${label_a} is set)" + else + validate_not_placeholder "${label_b}" "${value_b}" + fi +} + +if [[ "${TARGET_ENV}" == "local" ]]; then + # Minimal keys to run AITM API in local development without paid integrations. + assert_required "AITM" "DATABASE_URL" +else + # Gitgram backend + web core deploy requirements + assert_required "GITGRAM" "DATABASE_URL" + assert_required "GITGRAM" "REDIS_URL" + assert_required "GITGRAM" "JWT_SECRET" + assert_required "GITGRAM" "JWT_REFRESH_SECRET" + assert_required "GITGRAM" "FRONTEND_URL" + assert_required "GITGRAM" "CORS_ORIGIN" + assert_required "GITGRAM" "STORAGE_TYPE" + assert_allowed_value "GITGRAM" "STORAGE_TYPE" "local" "s3" + assert_required "GITGRAM" "NEXT_PUBLIC_API_URL" + assert_one_of "GITGRAM" "NEXT_PUBLIC_WS_URL" "NEXT_PUBLIC_SOCKET_URL" + + storage_type="$(resolve_value "GITGRAM" "STORAGE_TYPE")" + if [[ "${storage_type}" == "s3" ]]; then + assert_required "GITGRAM" "AWS_ACCESS_KEY_ID" + assert_required "GITGRAM" "AWS_SECRET_ACCESS_KEY" + assert_required "GITGRAM" "AWS_S3_BUCKET" + assert_required "GITGRAM" "AWS_REGION" + fi + + # AITM API core deploy requirements + assert_required "AITM" "DATABASE_URL" + assert_required "AITM" "INTERNAL_SECRET" + assert_required "AITM" "CORS_ORIGIN" + assert_required "AITM" "WEBSITE_URL" + + # Optional AITM website/self-hosted dashboard vars: validate pairs if present. + assert_pair_if_either_present "AITM" "CLERK_SECRET_KEY" "NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY" + assert_pair_if_either_present "AITM" "API_URL" "NEXT_PUBLIC_API_URL" + assert_pair_if_either_present "AITM" "NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID" "NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID" + + if [[ "${VALIDATION_PROFILE}" == "full" ]]; then + # Third-party integrations needed for a full launch contract. + assert_required "GITGRAM" "GITHUB_CLIENT_ID" + assert_required "GITGRAM" "GITHUB_CLIENT_SECRET" + assert_required "GITGRAM" "NEXT_PUBLIC_GITHUB_CLIENT_ID" + assert_required "GITGRAM" "POLAR_ACCESS_TOKEN" + assert_required "GITGRAM" "POLAR_WEBHOOK_SECRET" + assert_required "GITGRAM" "POLAR_PREMIUM_PRODUCT_ID" + assert_required "GITGRAM" "POLAR_CREATOR_PRODUCT_ID" + assert_required "GITGRAM" "POLAR_MODE" + assert_allowed_value "GITGRAM" "POLAR_MODE" "sandbox" "production" + assert_required "GITGRAM" "RESEND_API_KEY" + assert_required "GITGRAM" "SENTRY_DSN" + + assert_required "AITM" "POLAR_ACCESS_TOKEN" + assert_required "AITM" "POLAR_MODE" + assert_allowed_value "AITM" "POLAR_MODE" "sandbox" "production" + assert_required "AITM" "POLAR_WEBHOOK_SECRET" + assert_required "AITM" "POLAR_PRO_PRODUCT_ID" + assert_required "AITM" "POLAR_TEAM_PRODUCT_ID" + assert_required "AITM" "RESEND_API_KEY" + assert_required "AITM" "SENTRY_DSN" + else + # Core mode still validates provider mode if it is already set. + assert_allowed_value "GITGRAM" "POLAR_MODE" "sandbox" "production" + assert_allowed_value "AITM" "POLAR_MODE" "sandbox" "production" + fi +fi + +if [[ "${TARGET_ENV}" == "production" ]]; then + polar_mode="$(resolve_value "GITGRAM" "POLAR_MODE")" + if [[ -n "${polar_mode}" && "${polar_mode}" != "production" ]]; then + add_invalid "GITGRAM_POLAR_MODE must be production for a production launch" + fi + + aitm_polar_mode="$(resolve_value "AITM" "POLAR_MODE")" + if [[ -n "${aitm_polar_mode}" && "${aitm_polar_mode}" != "production" ]]; then + add_invalid "AITM_POLAR_MODE must be production for a production launch" + fi + + clerk_pub="$(resolve_value "AITM" "NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY")" + if [[ -n "${clerk_pub}" && "${clerk_pub}" == pk_test_* ]]; then + add_invalid "AITM_NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY must be live key in production (found test key pattern)" + fi +fi + +# Keep env templates aligned with required contract keys. +assert_example_has_keys "${ROOT_DIR}/gitgram/backend/.env.example" \ + DATABASE_URL JWT_SECRET JWT_REFRESH_SECRET FRONTEND_URL REDIS_URL \ + GITHUB_CLIENT_ID GITHUB_CLIENT_SECRET GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET \ + POLAR_ACCESS_TOKEN POLAR_WEBHOOK_SECRET POLAR_PREMIUM_PRODUCT_ID POLAR_CREATOR_PRODUCT_ID \ + RESEND_API_KEY SENTRY_DSN STORAGE_TYPE + +assert_example_has_keys "${ROOT_DIR}/aitimemachine/api-server/.env.example" \ + DATABASE_URL INTERNAL_SECRET CORS_ORIGIN WEBSITE_URL \ + POLAR_ACCESS_TOKEN POLAR_MODE POLAR_WEBHOOK_SECRET POLAR_PRO_PRODUCT_ID POLAR_TEAM_PRODUCT_ID + +assert_example_has_keys "${ROOT_DIR}/gitgram/web/.env.example" \ + NEXT_PUBLIC_API_URL NEXT_PUBLIC_WS_URL NEXT_PUBLIC_GITHUB_CLIENT_ID + +assert_example_has_keys "${ROOT_DIR}/aitimemachine/website/.env.example" \ + CLERK_SECRET_KEY NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY NEXT_PUBLIC_API_URL API_URL INTERNAL_SECRET \ + POLAR_ACCESS_TOKEN POLAR_MODE NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID + +assert_example_has_keys "${ROOT_DIR}/.env.gitgram.example" \ + DATABASE_URL REDIS_URL JWT_SECRET JWT_REFRESH_SECRET FRONTEND_URL CORS_ORIGIN \ + GITHUB_CLIENT_ID GITHUB_CLIENT_SECRET POLAR_ACCESS_TOKEN POLAR_WEBHOOK_SECRET \ + POLAR_PREMIUM_PRODUCT_ID POLAR_CREATOR_PRODUCT_ID POLAR_MODE STORAGE_TYPE + +assert_example_has_keys "${ROOT_DIR}/.env.gitgram-web.example" \ + NEXT_PUBLIC_API_URL NEXT_PUBLIC_WS_URL NEXT_PUBLIC_GITHUB_CLIENT_ID + +assert_example_has_keys "${ROOT_DIR}/.env.aitm-api.example" \ + DATABASE_URL INTERNAL_SECRET CORS_ORIGIN WEBSITE_URL POLAR_ACCESS_TOKEN POLAR_MODE POLAR_WEBHOOK_SECRET \ + POLAR_PRO_PRODUCT_ID POLAR_TEAM_PRODUCT_ID + +assert_example_has_keys "${ROOT_DIR}/.env.aitm-website.example" \ + NEXT_PUBLIC_API_URL API_URL INTERNAL_SECRET NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY CLERK_SECRET_KEY \ + POLAR_ACCESS_TOKEN POLAR_MODE NEXT_PUBLIC_POLAR_PRO_PRODUCT_ID NEXT_PUBLIC_POLAR_TEAM_PRODUCT_ID + +if [[ "${#MISSING[@]}" -eq 0 && "${#INVALID[@]}" -eq 0 ]]; then + echo "Secrets validation passed for ${TARGET_ENV} (${VALIDATION_PROFILE})." + exit 0 +fi + +echo "Secrets validation failed for ${TARGET_ENV} (${VALIDATION_PROFILE})." + +if [[ "${#MISSING[@]}" -gt 0 ]]; then + echo "" + echo "Missing variables:" + for item in "${MISSING[@]}"; do + echo " - ${item}" + done +fi + +if [[ "${#INVALID[@]}" -gt 0 ]]; then + echo "" + echo "Invalid values/config:" + for item in "${INVALID[@]}"; do + echo " - ${item}" + done +fi + +exit 1 diff --git a/scripts/verify-gitgram-db.ts b/scripts/verify-gitgram-db.ts new file mode 100644 index 00000000..298beb71 --- /dev/null +++ b/scripts/verify-gitgram-db.ts @@ -0,0 +1,463 @@ +#!/usr/bin/env npx tsx +/** + * Gitgram PostgreSQL Production Verification Script + * + * Connects to the gitgram backend PostgreSQL database using DATABASE_URL, + * then runs: + * 1. Connection probe + * 2. Latency probe + * 3. Prisma migration status check (via prisma CLI) + * 4. Core table existence checks (Account, User, Post, etc.) + * 5. Schema drift detection (expected columns vs actual) + * + * Uses @prisma/client for database access (must run prisma generate first). + * + * Exit codes: + * 0 - All checks passed + * 1 - One or more checks failed + * 2 - Configuration error (missing DATABASE_URL) + * + * Usage: + * DATABASE_URL=postgres://... npx tsx scripts/verify-gitgram-db.ts + * DATABASE_URL=postgres://... npx tsx scripts/verify-gitgram-db.ts --json + */ + +import { execSync } from "child_process"; + +// ============================================================================ +// Configuration +// ============================================================================ + +const DATABASE_URL = process.env.DATABASE_URL; +const OUTPUT_JSON = process.argv.includes("--json"); + +if (!DATABASE_URL) { + console.error("ERROR: DATABASE_URL environment variable is required"); + process.exit(2); +} + +// ============================================================================ +// Expected Prisma models -> table names and key columns +// Derived from gitgram/backend/prisma/schema.prisma +// ============================================================================ + +const EXPECTED_TABLES: Record<string, { columns: string[] }> = { + Account: { + columns: [ + "id", "type", "username", "displayName", "avatar", "bio", "verified", + "createdAt", "updatedAt", "userId", "repositoryId", "organizationId", + ], + }, + User: { + columns: [ + "id", "email", "password", "username", "displayName", "avatar", + "bio", "role", "githubId", "githubUsername", "emailVerified", + "verified", "isPrivate", "isBanned", "createdAt", + ], + }, + Repository: { + columns: ["id", "name", "fullName", "description", "language", "createdAt"], + }, + Organization: { + columns: ["id", "name", "description", "createdAt"], + }, + Post: { + columns: ["id", "type", "caption", "createdAt", "updatedAt", "accountId"], + }, + Comment: { + columns: ["id", "content", "createdAt", "updatedAt", "accountId", "postId"], + }, + Like: { + columns: ["id", "createdAt", "accountId", "postId"], + }, + Follow: { + columns: ["id", "createdAt", "followerId", "followingId"], + }, + Story: { + columns: ["id", "createdAt", "expiresAt", "accountId"], + }, + Message: { + columns: ["id", "content", "createdAt", "senderId", "receiverId"], + }, + Notification: { + columns: ["id", "type", "message", "read", "createdAt", "userId"], + }, +}; + +// ============================================================================ +// Probe functions using PrismaClient.$queryRawUnsafe +// ============================================================================ + +interface ProbeResult { + name: string; + status: "pass" | "fail" | "warn"; + message: string; + duration_ms: number; + details?: Record<string, unknown>; +} + +// Dynamic import of PrismaClient to handle cases where it hasn't been generated +let prisma: any = null; + +async function initPrisma(): Promise<boolean> { + try { + const { PrismaClient } = await import("@prisma/client"); + prisma = new PrismaClient({ datasources: { db: { url: DATABASE_URL } } }); + return true; + } catch (error) { + return false; + } +} + +async function disconnectPrisma(): Promise<void> { + if (prisma) { + await prisma.$disconnect(); + } +} + +async function probeConnection(): Promise<ProbeResult> { + const start = Date.now(); + try { + await prisma.$queryRawUnsafe("SELECT 1 AS ping"); + return { + name: "connection", + status: "pass", + message: "Database connection successful", + duration_ms: Date.now() - start, + }; + } catch (error) { + return { + name: "connection", + status: "fail", + message: `Connection failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probeLatency(): Promise<ProbeResult> { + const start = Date.now(); + try { + await prisma.$queryRawUnsafe("SELECT now() AS server_time"); + const ms = Date.now() - start; + const status = ms > 2000 ? "fail" : ms > 500 ? "warn" : "pass"; + return { + name: "latency", + status, + message: `Query latency: ${ms}ms`, + duration_ms: ms, + }; + } catch (error) { + return { + name: "latency", + status: "fail", + message: `Latency probe failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probePrismaMigrations(): Promise<ProbeResult> { + const start = Date.now(); + try { + // Check if _prisma_migrations table exists + const tableCheck: any[] = await prisma.$queryRawUnsafe( + `SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = '_prisma_migrations' + ) AS "exists"` + ); + + if (!tableCheck[0]?.exists) { + return { + name: "prisma_migrations", + status: "warn", + message: "Prisma migrations table not found (fresh database or using db push)", + duration_ms: Date.now() - start, + }; + } + + // Get migration status + const migrations: any[] = await prisma.$queryRawUnsafe( + `SELECT migration_name, finished_at, applied_steps_count, + CASE WHEN rolled_back_at IS NOT NULL THEN true ELSE false END AS rolled_back + FROM "_prisma_migrations" + ORDER BY started_at` + ); + + const totalMigrations = migrations.length; + const failedMigrations = migrations.filter( + (m: any) => m.finished_at === null && !m.rolled_back + ); + const rolledBack = migrations.filter((m: any) => m.rolled_back); + + if (failedMigrations.length > 0) { + return { + name: "prisma_migrations", + status: "fail", + message: `${failedMigrations.length} migration(s) incomplete: [${failedMigrations.map((m: any) => m.migration_name).join(", ")}]`, + duration_ms: Date.now() - start, + details: { + total: totalMigrations, + failed: failedMigrations.map((m: any) => m.migration_name), + rolledBack: rolledBack.map((m: any) => m.migration_name), + }, + }; + } + + if (rolledBack.length > 0) { + return { + name: "prisma_migrations", + status: "warn", + message: `${totalMigrations} migrations applied, ${rolledBack.length} rolled back`, + duration_ms: Date.now() - start, + details: { + total: totalMigrations, + rolledBack: rolledBack.map((m: any) => m.migration_name), + }, + }; + } + + return { + name: "prisma_migrations", + status: "pass", + message: `${totalMigrations} migrations applied successfully`, + duration_ms: Date.now() - start, + details: { + total: totalMigrations, + latest: migrations[migrations.length - 1]?.migration_name || "none", + }, + }; + } catch (error) { + return { + name: "prisma_migrations", + status: "fail", + message: `Migration check failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probeTable( + tableName: string, + expectedColumns: string[] +): Promise<ProbeResult> { + const start = Date.now(); + try { + // Prisma uses the model name as-is for the table name by default + let actualColumns: string[] = []; + + // Try the exact table name first + const colResult: any[] = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 + ORDER BY ordinal_position`, + tableName + ); + actualColumns = colResult.map((r: any) => r.column_name); + + // If not found, try with double-quote escaped version + if (actualColumns.length === 0) { + const quotedResult: any[] = await prisma.$queryRawUnsafe( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 + ORDER BY ordinal_position`, + tableName + ); + actualColumns = quotedResult.map((r: any) => r.column_name); + } + + if (actualColumns.length === 0) { + return { + name: `schema:${tableName}`, + status: "fail", + message: `Table '${tableName}' does not exist`, + duration_ms: Date.now() - start, + }; + } + + // Check for missing columns (Prisma column names match model field names) + const missingCols = expectedColumns.filter((col) => { + return !actualColumns.includes(col); + }); + + let status: "pass" | "fail" | "warn" = "pass"; + if (missingCols.length > 0) { + status = missingCols.length > expectedColumns.length / 2 ? "fail" : "warn"; + } + + return { + name: `schema:${tableName}`, + status, + message: + missingCols.length > 0 + ? `${tableName}: missing columns [${missingCols.join(", ")}]` + : `${tableName}: OK (${actualColumns.length} columns)`, + duration_ms: Date.now() - start, + details: { + actualColumnCount: actualColumns.length, + missingColumns: missingCols, + }, + }; + } catch (error) { + return { + name: `schema:${tableName}`, + status: "fail", + message: `Schema probe for '${tableName}' failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +// ============================================================================ +// Prisma CLI migration status check +// ============================================================================ + +async function probePrismaCLIMigrationStatus(): Promise<ProbeResult> { + const start = Date.now(); + try { + const output = execSync("npx prisma migrate status 2>&1", { + encoding: "utf-8", + timeout: 30000, + env: { ...process.env, DATABASE_URL }, + }); + + const hasPending = output.includes("Following migration(s) have not yet been applied"); + const hasFailed = output.includes("failed to apply"); + const isUpToDate = + output.includes("Database schema is up to date") || + output.includes("migrations found in prisma/migrations"); + + if (hasFailed) { + return { + name: "prisma_cli_status", + status: "fail", + message: "Prisma reports failed migrations", + duration_ms: Date.now() - start, + details: { output: output.trim() }, + }; + } + + if (hasPending) { + return { + name: "prisma_cli_status", + status: "warn", + message: "Prisma reports pending (unapplied) migrations", + duration_ms: Date.now() - start, + details: { output: output.trim() }, + }; + } + + return { + name: "prisma_cli_status", + status: "pass", + message: "Prisma migration status: up to date", + duration_ms: Date.now() - start, + }; + } catch (error) { + // prisma migrate status exits non-zero when there are pending migrations + const stderr = error instanceof Error ? (error as any).stderr || error.message : "unknown"; + const stdout = error instanceof Error ? (error as any).stdout || "" : ""; + + return { + name: "prisma_cli_status", + status: "warn", + message: `Prisma CLI status check: ${stdout || stderr}`.slice(0, 200), + duration_ms: Date.now() - start, + }; + } +} + +// ============================================================================ +// Main +// ============================================================================ + +async function main() { + const results: ProbeResult[] = []; + + console.log("=== Gitgram DB Production Verification ===\n"); + + // Initialize Prisma Client + console.log("Initializing Prisma Client..."); + const prismaReady = await initPrisma(); + + if (!prismaReady) { + console.error("Failed to initialize Prisma Client. Ensure 'prisma generate' has been run."); + results.push({ + name: "prisma_init", + status: "fail", + message: "Failed to initialize PrismaClient (run prisma generate first)", + duration_ms: 0, + }); + + // Fall back to CLI-only checks + console.log("[1/1] Prisma CLI migration status..."); + results.push(await probePrismaCLIMigrationStatus()); + } else { + // Connection probe + console.log("[1/5] Connection probe..."); + const connResult = await probeConnection(); + results.push(connResult); + + if (connResult.status === "fail") { + console.error("Connection failed, cannot continue with DB probes."); + } else { + // Latency probe + console.log("[2/5] Latency probe..."); + results.push(await probeLatency()); + + // Prisma migration status (via SQL) + console.log("[3/5] Prisma migration status (SQL)..."); + results.push(await probePrismaMigrations()); + + // Prisma CLI migration status + console.log("[4/5] Prisma CLI migration status..."); + results.push(await probePrismaCLIMigrationStatus()); + + // Schema probes + console.log("[5/5] Schema probes..."); + for (const [tableName, def] of Object.entries(EXPECTED_TABLES)) { + results.push(await probeTable(tableName, def.columns)); + } + } + + await disconnectPrisma(); + } + + // Output + if (OUTPUT_JSON) { + const passed = results.filter((r) => r.status === "pass").length; + const warned = results.filter((r) => r.status === "warn").length; + const failed = results.filter((r) => r.status === "fail").length; + console.log( + JSON.stringify( + { + timestamp: new Date().toISOString(), + overall: failed > 0 ? "unhealthy" : warned > 0 ? "degraded" : "healthy", + summary: { total: results.length, passed, warned, failed }, + results, + }, + null, + 2 + ) + ); + } else { + console.log("\n=== Results ===\n"); + for (const r of results) { + const icon = r.status === "pass" ? "PASS" : r.status === "warn" ? "WARN" : "FAIL"; + console.log(` [${icon}] ${r.name}: ${r.message} (${r.duration_ms}ms)`); + } + const passed = results.filter((r) => r.status === "pass").length; + const warned = results.filter((r) => r.status === "warn").length; + const failed = results.filter((r) => r.status === "fail").length; + console.log(`\nTotal: ${results.length} | Passed: ${passed} | Warnings: ${warned} | Failed: ${failed}`); + } + + const hasFails = results.some((r) => r.status === "fail"); + process.exit(hasFails ? 1 : 0); +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/scripts/verify-launch-readiness.sh b/scripts/verify-launch-readiness.sh new file mode 100755 index 00000000..f113d76f --- /dev/null +++ b/scripts/verify-launch-readiness.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_ENV="${1:-all}" +if [[ "${TARGET_ENV}" != "all" && "${TARGET_ENV}" != "staging" && "${TARGET_ENV}" != "production" ]]; then + echo "Usage: $0 [all|staging|production] [core|full]" >&2 + exit 1 +fi + +VALIDATION_PROFILE="${2:-core}" +if [[ "${VALIDATION_PROFILE}" != "core" && "${VALIDATION_PROFILE}" != "full" ]]; then + echo "Usage: $0 [all|staging|production] [core|full]" >&2 + exit 1 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUN_TYPECHECK="${RUN_TYPECHECK:-0}" +RUN_LOCAL_SMOKE="${RUN_LOCAL_SMOKE:-0}" + +MISSING_FILES=() +FAILED_STEPS=() + +log() { + printf '%s\n' "$1" +} + +check_cmd() { + local cmd="$1" + if ! command -v "${cmd}" >/dev/null 2>&1; then + log "Missing required command: ${cmd}" + exit 1 + fi +} + +require_file() { + local path="$1" + if [[ ! -f "${path}" ]]; then + MISSING_FILES+=("${path}") + fi +} + +resolve_env_file() { + local base="$1" + local env_name="$2" + local candidate="${base}.${env_name}" + + if [[ -f "${candidate}" ]]; then + printf '%s' "${candidate}" + return 0 + fi + + printf '%s' "${base}" +} + +require_env_set() { + local env_name="$1" + require_file "$(resolve_env_file "${ROOT_DIR}/.env.gitgram" "${env_name}")" + require_file "$(resolve_env_file "${ROOT_DIR}/.env.gitgram-web" "${env_name}")" + require_file "$(resolve_env_file "${ROOT_DIR}/.env.aitm-api" "${env_name}")" +} + +log "=== Launch Readiness Gate ===" +log "Repo: ${ROOT_DIR}" +log "Profile: ${VALIDATION_PROFILE}" + +log "" +log "1) Checking required commands" +check_cmd bash +check_cmd node +check_cmd npm +check_cmd pm2 + +log "PASS node: $(node -v)" +log "PASS npm: $(npm -v)" +log "PASS pm2 command: $(command -v pm2)" + +log "" +log "2) Checking launch artifacts" +require_file "${ROOT_DIR}/gitgram/ecosystem.config.js" +require_file "${ROOT_DIR}/aitimemachine/deploy/ecosystem.config.cjs" +require_file "${ROOT_DIR}/.env.gitgram.example" +require_file "${ROOT_DIR}/.env.gitgram-web.example" +require_file "${ROOT_DIR}/.env.aitm-api.example" + +if [[ "${#MISSING_FILES[@]}" -gt 0 ]]; then + log "Missing launch artifacts:" + for item in "${MISSING_FILES[@]}"; do + log " - ${item}" + done + exit 1 +fi +log "PASS launch artifacts present" + +log "" +log "3) Checking operator env files" +MISSING_FILES=() +if [[ "${TARGET_ENV}" == "all" ]]; then + require_env_set "staging" + require_env_set "production" +else + require_env_set "${TARGET_ENV}" +fi + +if [[ "${#MISSING_FILES[@]}" -gt 0 ]]; then + log "Missing operator env files:" + for item in "${MISSING_FILES[@]}"; do + log " - ${item}" + done + log "" + log "Create them from the templates. Preferred layout:" + log " cp .env.gitgram.example .env.gitgram.production" + log " cp .env.gitgram-web.example .env.gitgram-web.production" + log " cp .env.aitm-api.example .env.aitm-api.production" + log " cp .env.gitgram.example .env.gitgram.staging" + log " cp .env.gitgram-web.example .env.gitgram-web.staging" + log " cp .env.aitm-api.example .env.aitm-api.staging" + log "Fallback layout still supported:" + log " cp .env.gitgram.example .env.gitgram" + log " cp .env.gitgram-web.example .env.gitgram-web" + log " cp .env.aitm-api.example .env.aitm-api" + log " # optional" + log " cp .env.aitm-website.example .env.aitm-website" + exit 1 +fi +log "PASS operator env files present" + +validate_env() { + local env_name="$1" + log "" + log "4) Validating ${VALIDATION_PROFILE} secrets contract for ${env_name}" + if ! bash "${ROOT_DIR}/scripts/validate-prod-secrets.sh" "${env_name}" "${VALIDATION_PROFILE}"; then + FAILED_STEPS+=("secrets:${env_name}:${VALIDATION_PROFILE}") + fi +} + +if [[ "${TARGET_ENV}" == "all" ]]; then + validate_env "staging" + validate_env "production" +else + validate_env "${TARGET_ENV}" +fi + +log "" +log "5) Verifying AITM npm package publishability" +if ! npm --prefix "${ROOT_DIR}" run verify:aitm-packages; then + FAILED_STEPS+=("aitm-packages") +fi + +if [[ "${RUN_TYPECHECK}" == "1" ]]; then + log "" + log "6) Running typechecks" + if ! npm --prefix "${ROOT_DIR}" run typecheck:gitgram; then + FAILED_STEPS+=("typecheck:gitgram") + fi + if ! npm --prefix "${ROOT_DIR}" run typecheck:aitm; then + FAILED_STEPS+=("typecheck:aitm") + fi +else + log "" + log "6) Skipping typechecks (set RUN_TYPECHECK=1 to enable)" +fi + +if [[ "${RUN_LOCAL_SMOKE}" == "1" ]]; then + log "" + log "7) Running local smoke script" + if ! bash "${ROOT_DIR}/scripts/smoke-local-launch.sh"; then + FAILED_STEPS+=("local-smoke") + fi +else + log "" + log "7) Skipping local smoke (set RUN_LOCAL_SMOKE=1 to enable)" +fi + +if [[ "${#FAILED_STEPS[@]}" -gt 0 ]]; then + log "" + log "Launch readiness failed." + log "Failed checks:" + for item in "${FAILED_STEPS[@]}"; do + log " - ${item}" + done + exit 1 +fi + +log "" +log "Launch readiness checks passed." diff --git a/scripts/verify-neon.ts b/scripts/verify-neon.ts new file mode 100644 index 00000000..709e3c1f --- /dev/null +++ b/scripts/verify-neon.ts @@ -0,0 +1,415 @@ +#!/usr/bin/env npx tsx +/** + * Neon PostgreSQL Production Verification Script + * + * Connects to the Neon database using DATABASE_URL, then runs: + * 1. Connection probe (SELECT 1) + * 2. Latency probe (SELECT now()) + * 3. Replication lag check + * 4. Schema probes (table existence, columns, indexes) + * 5. Migration drift check + * + * Exit codes: + * 0 - All checks passed + * 1 - One or more checks failed + * 2 - Configuration error (missing DATABASE_URL) + * + * Usage: + * DATABASE_URL=postgres://... npx tsx scripts/verify-neon.ts + * DATABASE_URL=postgres://... npx tsx scripts/verify-neon.ts --json + */ + +import { neon } from "@neondatabase/serverless"; +import { drizzle } from "drizzle-orm/neon-http"; +import { sql } from "drizzle-orm"; + +// ============================================================================ +// Configuration +// ============================================================================ + +const DATABASE_URL = process.env.DATABASE_URL; +const OUTPUT_JSON = process.argv.includes("--json"); + +if (!DATABASE_URL) { + console.error("ERROR: DATABASE_URL environment variable is required"); + process.exit(2); +} + +// ============================================================================ +// Expected schema definition (mirrored from db-health.ts for standalone use) +// ============================================================================ + +const EXPECTED_TABLES: Record<string, { columns: string[]; indexes: string[] }> = { + users: { + columns: ["id", "email", "name", "plan", "polar_customer_id", "storage_used_bytes", "created_at", "updated_at"], + indexes: ["users_pkey"], + }, + api_keys: { + columns: ["id", "user_id", "key_hash", "key_prefix", "name", "scopes", "expires_at", "allowed_ips", "last_used_at", "created_at", "revoked_at"], + indexes: ["api_keys_pkey"], + }, + sessions: { + columns: ["id", "user_id", "token_hash", "ip_address", "user_agent", "device_info", "last_active_at", "expires_at", "revoked", "revoked_at", "created_at"], + indexes: ["sessions_pkey", "idx_auth_sessions_user", "idx_auth_sessions_token_hash", "idx_auth_sessions_expires", "idx_auth_sessions_revoked"], + }, + subscriptions: { + columns: ["id", "user_id", "polar_customer_id", "polar_product_id", "status", "plan", "current_period_start", "current_period_end", "cancel_at_period_end", "canceled_at", "created_at", "updated_at"], + indexes: ["subscriptions_pkey", "idx_subscriptions_user", "idx_subscriptions_status", "idx_subscriptions_polar_customer"], + }, + organizations: { + columns: ["id", "name", "slug", "owner_id", "plan", "metadata", "created_at", "updated_at"], + indexes: ["organizations_pkey", "idx_organizations_slug", "idx_organizations_owner"], + }, + projects: { + columns: ["id", "user_id", "org_id", "name", "root_path", "created_at", "updated_at", "last_synced_at"], + indexes: ["projects_pkey", "idx_projects_user", "idx_projects_org"], + }, + checkpoints: { + columns: ["id", "project_id", "user_id", "org_id", "parent_id", "name", "description", "scope", "manifest_hash", "root_tree_hash", "trigger_type", "linked_checkpoint_id", "created_at", "metadata"], + indexes: ["checkpoints_pkey", "idx_checkpoints_project", "idx_checkpoints_user", "idx_checkpoints_org", "idx_checkpoints_created"], + }, + checkpoint_files: { + columns: ["checkpoint_id", "file_path", "blob_hash", "size", "mode", "mtime_ms", "change_type"], + indexes: ["checkpoint_files_pkey", "idx_checkpoint_files_path"], + }, + blobs: { + columns: ["hash", "user_id", "size", "mime_type", "content", "storage_type", "created_at"], + indexes: ["blobs_pkey"], + }, + trees: { + columns: ["id", "project_id", "hash", "parent_path", "entries", "created_at"], + indexes: ["trees_pkey", "idx_trees_project_hash", "idx_trees_hash", "idx_trees_parent_path"], + }, + chat_sessions: { + columns: ["id", "project_id", "user_id", "tool", "started_at", "ended_at", "metadata"], + indexes: ["chat_sessions_pkey", "idx_sessions_project", "idx_sessions_user"], + }, + chat_messages: { + columns: ["id", "session_id", "project_id", "user_id", "checkpoint_id", "role", "content", "content_hash", "is_context_injection", "timestamp", "metadata"], + indexes: ["chat_messages_pkey", "idx_messages_session", "idx_messages_project", "idx_messages_timestamp"], + }, + webhooks: { + columns: ["id", "user_id", "url", "secret", "events", "is_active", "created_at", "updated_at", "last_triggered_at", "failure_count"], + indexes: ["webhooks_pkey", "idx_webhooks_user", "idx_webhooks_active"], + }, + repositories: { + columns: ["id", "project_id", "user_id", "name", "root_path", "last_indexed_at", "index_status", "file_count", "symbol_count", "created_at", "updated_at", "metadata"], + indexes: ["repositories_pkey", "idx_repositories_project", "idx_repositories_user"], + }, + audit_logs: { + columns: ["id", "user_id", "action", "resource_type", "resource_id", "ip_address", "user_agent", "metadata", "created_at"], + indexes: ["audit_logs_pkey", "idx_audit_logs_user", "idx_audit_logs_action", "idx_audit_logs_resource_type", "idx_audit_logs_created_at"], + }, + posts: { + columns: ["id", "author_user_id", "repository_id", "project_id", "type", "caption", "code_snippet", "language", "linked_checkpoint_id", "visibility", "published_at", "created_at", "updated_at"], + indexes: ["posts_pkey", "idx_posts_repo_created", "idx_posts_author_created"], + }, + follows: { + columns: ["id", "follower_user_id", "target_type", "target_user_id", "target_repository_id", "created_at"], + indexes: ["follows_pkey", "idx_follows_target_user", "idx_follows_target_repo"], + }, + trend_scores: { + columns: ["id", "repository_id", "window_start", "window_end", "score", "signals", "created_at"], + indexes: ["trend_scores_pkey", "idx_trend_scores_repo_window"], + }, +}; + +const EXPECTED_MIGRATIONS = [ + "0001_add_storage_used_bytes", + "0002_add_knowledge_base_tables", + "0003_add_canonical_time_context_governance", + "0004_add_gitgram_social_layer", +]; + +// ============================================================================ +// Database connection +// ============================================================================ + +const neonSql = neon(DATABASE_URL); +const db = drizzle(neonSql); + +// ============================================================================ +// Probe functions +// ============================================================================ + +interface ProbeResult { + name: string; + status: "pass" | "fail" | "warn"; + message: string; + duration_ms: number; + details?: Record<string, unknown>; +} + +async function probeConnection(): Promise<ProbeResult> { + const start = Date.now(); + try { + await db.execute(sql`SELECT 1 AS ping`); + return { + name: "connection", + status: "pass", + message: "Database connection successful", + duration_ms: Date.now() - start, + }; + } catch (error) { + return { + name: "connection", + status: "fail", + message: `Connection failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probeLatency(): Promise<ProbeResult> { + const start = Date.now(); + try { + await db.execute(sql`SELECT now() AS server_time`); + const ms = Date.now() - start; + const status = ms > 2000 ? "fail" : ms > 500 ? "warn" : "pass"; + return { + name: "latency", + status, + message: `Query latency: ${ms}ms`, + duration_ms: ms, + }; + } catch (error) { + return { + name: "latency", + status: "fail", + message: `Latency probe failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probeReplication(): Promise<ProbeResult> { + const start = Date.now(); + try { + const result = await db.execute( + sql`SELECT CASE + WHEN pg_is_in_recovery() THEN + EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp())) + ELSE 0 + END AS lag_seconds` + ); + const ms = Date.now() - start; + const lag = parseFloat((result as any[])[0]?.lag_seconds ?? "0"); + const status = lag > 30 ? "fail" : lag > 5 ? "warn" : "pass"; + return { + name: "replication", + status, + message: `Replication lag: ${lag.toFixed(1)}s`, + duration_ms: ms, + details: { lag_seconds: lag }, + }; + } catch { + return { + name: "replication", + status: "pass", + message: "Replication check: not available (Neon serverless mode)", + duration_ms: Date.now() - start, + }; + } +} + +async function probeTable( + tableName: string, + expectedColumns: string[], + expectedIndexes: string[] +): Promise<ProbeResult> { + const start = Date.now(); + try { + const colResult = await db.execute( + sql`SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = ${tableName} + ORDER BY ordinal_position` + ); + const actualColumns = (colResult as any[]).map((r: any) => r.column_name); + + if (actualColumns.length === 0) { + return { + name: `schema:${tableName}`, + status: "fail", + message: `Table '${tableName}' does not exist`, + duration_ms: Date.now() - start, + }; + } + + const missingCols = expectedColumns.filter((c) => !actualColumns.includes(c)); + + const idxResult = await db.execute( + sql`SELECT indexname FROM pg_indexes + WHERE schemaname = 'public' AND tablename = ${tableName} + ORDER BY indexname` + ); + const actualIndexes = (idxResult as any[]).map((r: any) => r.indexname); + const missingIdxs = expectedIndexes.filter((i) => !actualIndexes.includes(i)); + + const messages: string[] = []; + let status: "pass" | "fail" | "warn" = "pass"; + + if (missingCols.length > 0) { + status = "fail"; + messages.push(`missing columns: [${missingCols.join(", ")}]`); + } + if (missingIdxs.length > 0) { + if (status !== "fail") status = "warn"; + messages.push(`missing indexes: [${missingIdxs.join(", ")}]`); + } + + return { + name: `schema:${tableName}`, + status, + message: messages.length > 0 + ? `${tableName}: ${messages.join("; ")}` + : `${tableName}: OK (${actualColumns.length} columns, ${actualIndexes.length} indexes)`, + duration_ms: Date.now() - start, + details: { + columns: actualColumns.length, + indexes: actualIndexes.length, + missingColumns: missingCols, + missingIndexes: missingIdxs, + }, + }; + } catch (error) { + return { + name: `schema:${tableName}`, + status: "fail", + message: `Schema probe for '${tableName}' failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +async function probeMigrationDrift(): Promise<ProbeResult> { + const start = Date.now(); + try { + const tableCheck = await db.execute( + sql`SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = '__drizzle_migrations' + ) AS exists` + ); + + if (!(tableCheck as any[])[0]?.exists) { + return { + name: "migration_drift", + status: "warn", + message: "Migration tracking table not found (may use db:push)", + duration_ms: Date.now() - start, + }; + } + + const migResult = await db.execute( + sql`SELECT tag FROM __drizzle_migrations ORDER BY created_at` + ); + const applied = (migResult as any[]).map((r: any) => r.tag); + const missing = EXPECTED_MIGRATIONS.filter((m) => !applied.includes(m)); + const extra = applied.filter((m: string) => !EXPECTED_MIGRATIONS.includes(m)); + + if (missing.length > 0) { + return { + name: "migration_drift", + status: "fail", + message: `Missing migrations: [${missing.join(", ")}]`, + duration_ms: Date.now() - start, + details: { applied, missing, extra }, + }; + } + + if (extra.length > 0) { + return { + name: "migration_drift", + status: "warn", + message: `Unexpected migrations: [${extra.join(", ")}]`, + duration_ms: Date.now() - start, + details: { applied, missing, extra }, + }; + } + + return { + name: "migration_drift", + status: "pass", + message: `All ${EXPECTED_MIGRATIONS.length} expected migrations applied`, + duration_ms: Date.now() - start, + details: { applied }, + }; + } catch (error) { + return { + name: "migration_drift", + status: "fail", + message: `Migration drift check failed: ${error instanceof Error ? error.message : "unknown"}`, + duration_ms: Date.now() - start, + }; + } +} + +// ============================================================================ +// Main +// ============================================================================ + +async function main() { + const results: ProbeResult[] = []; + + // Health probes + console.log("=== Neon Production Verification ===\n"); + + console.log("[1/4] Connection probe..."); + results.push(await probeConnection()); + + console.log("[2/4] Latency probe..."); + results.push(await probeLatency()); + + console.log("[3/4] Replication probe..."); + results.push(await probeReplication()); + + // Schema probes + console.log("[4/4] Schema probes..."); + const tableNames = Object.keys(EXPECTED_TABLES); + for (const tableName of tableNames) { + const def = EXPECTED_TABLES[tableName]; + results.push(await probeTable(tableName, def.columns, def.indexes)); + } + + // Migration drift + console.log(" Migration drift check..."); + results.push(await probeMigrationDrift()); + + // Output + if (OUTPUT_JSON) { + const passed = results.filter((r) => r.status === "pass").length; + const warned = results.filter((r) => r.status === "warn").length; + const failed = results.filter((r) => r.status === "fail").length; + console.log( + JSON.stringify( + { + timestamp: new Date().toISOString(), + overall: failed > 0 ? "unhealthy" : warned > 0 ? "degraded" : "healthy", + summary: { total: results.length, passed, warned, failed }, + results, + }, + null, + 2 + ) + ); + } else { + console.log("\n=== Results ===\n"); + for (const r of results) { + const icon = r.status === "pass" ? "PASS" : r.status === "warn" ? "WARN" : "FAIL"; + console.log(` [${icon}] ${r.name}: ${r.message} (${r.duration_ms}ms)`); + } + const passed = results.filter((r) => r.status === "pass").length; + const warned = results.filter((r) => r.status === "warn").length; + const failed = results.filter((r) => r.status === "fail").length; + console.log(`\nTotal: ${results.length} | Passed: ${passed} | Warnings: ${warned} | Failed: ${failed}`); + } + + // Exit code + const hasFails = results.some((r) => r.status === "fail"); + process.exit(hasFails ? 1 : 0); +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/scripts/verify-public-launch.sh b/scripts/verify-public-launch.sh new file mode 100755 index 00000000..92deced1 --- /dev/null +++ b/scripts/verify-public-launch.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +GITGRAM_PUBLIC_URL="${GITGRAM_PUBLIC_URL:-https://gitgram.app}" +AITM_API_PUBLIC_URL="${AITM_API_PUBLIC_URL:-https://timemachine-api.gitgram.app}" +AITM_WEBSITE_PUBLIC_URL="${AITM_WEBSITE_PUBLIC_URL:-https://timemachine.gitgram.app}" +SKIP_AITM_WEBSITE="${SKIP_AITM_WEBSITE:-0}" + +log() { + printf '%s\n' "$1" +} + +check_http_ok() { + local label="$1" + local url="$2" + if curl -fsSL --max-time 15 "${url}" >/dev/null; then + log "PASS ${label}: ${url}" + else + log "FAIL ${label}: ${url}" + return 1 + fi +} + +log "=== Public Launch Verification ===" +log "Gitgram: ${GITGRAM_PUBLIC_URL}" +log "AITM API: ${AITM_API_PUBLIC_URL}" + +check_http_ok "Gitgram web" "${GITGRAM_PUBLIC_URL}" +check_http_ok "Gitgram backend health" "${GITGRAM_PUBLIC_URL}/api/health" +check_http_ok "Gitgram backend readiness" "${GITGRAM_PUBLIC_URL}/api/health/ready" +check_http_ok "AITM API health" "${AITM_API_PUBLIC_URL}/health" +check_http_ok "AITM API readiness" "${AITM_API_PUBLIC_URL}/health/ready" + +if [[ "${SKIP_AITM_WEBSITE}" != "1" ]]; then + check_http_ok "AITM website" "${AITM_WEBSITE_PUBLIC_URL}" +fi + +log "" +log "Public launch verification passed."