From 91ba7b5c5c75cb00302e34d7df2972341ff32dd9 Mon Sep 17 00:00:00 2001 From: openclaw Date: Tue, 9 Jun 2026 15:45:58 -0600 Subject: [PATCH] fix(deploy): add health/db-schema guard + curl gate; ship migration assets; document prod bootstrap (executed per systematic-debugging plan) --- .gitea/workflows/deploy.yml | 30 +++- CLAUDE.md | 27 ++++ ...06-prod-db-schema-migration-reliability.md | 147 ++++++++++++++++++ src/app/api/health/db-schema/route.ts | 32 ++++ 4 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md create mode 100644 src/app/api/health/db-schema/route.ts diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index cd23358..781d6d0 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -24,7 +24,25 @@ jobs: env: DATABASE_URL: ${{ secrets.DATABASE_URL }} run: | - npm run migrate:one || echo "No migration script or migrations already applied" + set -e + echo "=== Running migrations against DATABASE_URL (masked value for logs) ===" + npm run migrate:one + echo "=== Post-migration verification: critical table admin_users must exist ===" + node -e ' + const { Client } = require("pg"); + const url = process.env.DATABASE_URL; + if (!url) { console.error("No DATABASE_URL"); process.exit(1); } + const c = new Client({ connectionString: url }); + c.connect() + .then(() => c.query("SELECT 1 FROM admin_users LIMIT 1")) + .then(() => { console.log("✓ admin_users table exists and is queryable"); return c.end(); }) + .then(() => process.exit(0)) + .catch(e => { + console.error("✗ FATAL: admin_users relation missing after migrate:", e.message); + console.error("The deploy cannot continue. The secret DATABASE_URL must have had db/migrations/0001_init.sql applied successfully."); + process.exit(1); + }); + ' - name: Build env: @@ -183,8 +201,16 @@ jobs: scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no package.json tyler@route.crispygoat.com:$APP_DIR/ scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no next.config.ts tyler@route.crispygoat.com:$APP_DIR/ 2>/dev/null || true + # Ship the migration runner + SQL so the server has a recovery path (scripts/ and db/migrations/ were previously omitted from the artifact). + # This allows `node scripts/migrate.js` (after sourcing .env.production) to work directly on the target if needed for bootstrap or emergencies. + # See docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md + echo "Ensuring migration directories on server and copying runner + SQL..." + ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "mkdir -p $APP_DIR/scripts $APP_DIR/db" + scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no scripts/migrate.js tyler@route.crispygoat.com:$APP_DIR/scripts/migrate.js 2>/dev/null || true + scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r db/migrations tyler@route.crispygoat.com:$APP_DIR/db/ 2>/dev/null || true + # Install deps and restart on server echo "Installing deps and restarting PM2..." - ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save" + ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save && sleep 4 && curl -f -s http://localhost:3100/api/health/db-schema || { echo 'Health check failed after start - schema not applied (see plan)'; exit 1; }" echo "Deployed successfully" \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index f37d3e7..f59d3ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,6 +101,33 @@ The app connects to **Postgres directly** — no Supabase platform, JS client, o - A single shared `pg` `Pool` is exported from `src/lib/db.ts` (TBD — to be created/confirmed during the migration). Server actions and API routes import it and call `pool.query(...)` against RPC names. - No `NEXT_PUBLIC_SUPABASE_URL` / `SUPABASE_SERVICE_ROLE_KEY` / `@supabase/*` imports — these are being purged from the codebase. +#### First production deploy / new prod DB bootstrap (critical for admin access) + +The `admin_users` + `admin_user_brands` tables (and the rest of the schema) come **only** from `db/migrations/0001_init.sql`. + +If the prod `DATABASE_URL` has never had the migrations applied, `getAdminUser()` will fail with "relation \"admin_users\" does not exist", the layout will show "Access Denied", and even a signed-in Neon Auth user will be blocked. + +**Correct bootstrap sequence (do this from a machine with the full source tree before the first push that exercises /admin):** + +1. Ensure the Gitea secret `DATABASE_URL` points at the real prod Neon Postgres (the one with `neon_auth.user` already present). +2. Sign in once at the live prod URL (`/login`) with the email you want as the first `platform_admin`. This creates the row in `neon_auth.user`. +3. From your laptop (or any box with the checkout): + + ```bash + # Paste the real prod connection string (get it from Gitea secrets or the target's .env.production) + DATABASE_URL="postgresql://...prod-full-string..." node scripts/migrate.js + + # Then provision (the script will link to the first brand it finds) + DATABASE_URL="postgresql://...prod-full-string..." \ + npx tsx scripts/provision-admin.ts you@real.com platform_admin + ``` + +4. Push to main. The deploy workflow now has a hard gate (see `.gitea/workflows/deploy.yml` "Run migrations" + verification query for `admin_users`) and ships the migrate runner + SQL files, so future deploys and server-side recovery are protected. + +See the full root-cause + plan: `docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md` + +The old `|| echo` masking around `npm run migrate:one` has been removed; a missing critical table will now fail the CI job with a clear message. + #### SECURITY DEFINER RPCs + Brand Scoping The app uses **PostgreSQL SECURITY DEFINER functions** for all data access. These run with the function owner's privileges and bypass any future RLS. This means: diff --git a/docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md b/docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md new file mode 100644 index 0000000..09ad1fb --- /dev/null +++ b/docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md @@ -0,0 +1,147 @@ +# Production DB Schema Migration Reliability Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:systematic-debugging (completed root cause), superpowers:writing-plans (this), superpowers:test-driven-development where code changes have tests, superpowers:verification-before-completion, and superpowers:executing-plans or subagent-driven-development to implement task-by-task. Steps use checkbox syntax. + +**Goal:** Ensure that every production deploy (push to main) results in the full schema from `db/migrations/0001_init.sql` (including the `admin_users` and `admin_user_brands` tables required by `getAdminUser()`) being present in the `DATABASE_URL` that the running app connects to via its pool / drizzle client. Eliminate "relation does not exist" errors and the resulting Access Denied screen for properly provisioned Neon Auth users. + +**Architecture:** +- Make the CI "Run migrations" step a hard gate (fatal on failure, plus explicit post-migrate verification that critical tables exist). +- Ship the minimal migration runner + SQL files as part of the deploy artifact so the target server has a recovery path. +- Add a lightweight post-deploy / startup verification in the app or deploy script (fail fast with clear message instead of silent 500s on first admin request). +- Keep the "migrate from full source locally" path working for initial prod DB bootstrap and emergencies. +- Do not change the core migration logic or 0001_init.sql in this plan (that would be a separate architectural change if the double-BEGIN wrapping proves fragile). + +**Tech Stack:** Gitea Actions (YAML), Node 22 + pg + drizzle on target, Next.js standalone output, pm2 on Ubuntu server, Neon Postgres (with neon_auth schema). + +**Root Cause (from systematic-debugging Phase 1):** The prod runtime DB lacked the `admin_users` table because (1) the migration step in `.gitea/workflows/deploy.yml` used `|| echo` making any failure (connection, SQL error in the huge 0001 file, FK to neon_auth.user, tx nesting from the file's BEGIN + script's BEGIN) non-fatal, (2) only `.next/`, `public/`, `package.json` (and optional next.config) are scp'd — `scripts/migrate.js` and `db/migrations/` are never on the server, (3) no verification after "migrate" or at app startup that the tables the admin permission layer depends on actually exist, (4) the `.env.production` written from the same secret as the CI migrate step was used, but the apply didn't happen or was skipped due to _migrations state or partial rollback. + +**Evidence Gathered:** +- deploy.yml: Run migrations step, limited scp, .env.production printf, pm2 "npm start". +- scripts/migrate.js: dotenv .env.local + env override, _migrations tracking, per-file client.query(sql) inside script tx, re-throw on error. +- db/migrations/0001_init.sql: explicit CREATE TABLE admin_users (with FK to neon_auth.user), admin_user_brands, brands; file starts with BEGIN;. +- db/client.ts + src/lib/admin-permissions.ts: withPlatformAdmin → drizzle select on adminUsers from schema (the exact query that 42P01s). +- next.config.ts: output: 'standalone' (explains pm2 warning). +- Runtime logs: the repeated "Database query failed" + "relation does not exist", app starts fine. +- Git history: recent deploy "fixes" focused on SSH/env writing, not migration reliability. + +**Files to touch (decomposition by responsibility):** +- `.gitea/workflows/deploy.yml` (CI pipeline gates + artifact contents) +- `scripts/migrate.js` (minor hardening if needed for verification hook) +- `src/app/api/health/route.ts` or similar (new, for startup/schema check — or add to existing) +- `CLAUDE.md` + `PRODUCTION_DEPLOYMENT_CHECKLIST.md` (docs) +- Possibly a small `scripts/verify-prod-schema.js` helper + +--- +### Task 1: Make CI migration step a hard failure + add explicit verification for admin_users + +**Files:** +- Modify: `.gitea/workflows/deploy.yml:23-27` (the Run migrations step and surrounding) + +- [ ] **Step 1.1:** Replace the non-fatal migration line with a strict block that fails the job if migrate fails or the critical table is missing after. + +```yaml +- name: Run migrations + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + set -e + echo "=== Running migrations against prod DB ===" + npm run migrate:one + echo "=== Verifying critical schema (admin_users) ===" + node -e ' + const {Client} = require("pg"); + const c = new Client({connectionString: process.env.DATABASE_URL}); + c.connect().then(() => c.query("SELECT 1 FROM admin_users LIMIT 1")).then(() => { + console.log("✓ admin_users table exists"); + return c.end(); + }).then(() => process.exit(0)).catch(e => { + console.error("✗ admin_users missing or inaccessible:", e.message); + process.exit(1); + }); + ' +``` + +- [ ] **Step 1.2:** Run a local simulation or note that the Gitea runner will now fail the whole deploy if the secret DB is missing the table (good — forces the bootstrap to happen before code that depends on it ships). + +- [ ] **Step 1.3:** Commit the yml change with message referencing the root cause (missing table in prod due to masked migration). + +### Task 2: Ship migration capability in the deploy artifact so server has a recovery path + +**Files:** +- Modify: `.gitea/workflows/deploy.yml` in the "Deploy" step (the scp and ssh sections) + +- [ ] **Step 2.1:** Add scp for the migration assets (after the existing public/.next scp): + +```bash +echo "Copying migration runner and SQL..." +scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r scripts/migrate.js tyler@...:$APP_DIR/scripts/ || true +scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r db/migrations tyler@...:$APP_DIR/db/ || true +``` + +- [ ] **Step 2.2:** Update the server ssh install/restart line to also ensure the scripts dir has the right perms if needed, but mainly document that now `node scripts/migrate.js` will work on the server with the .env.production. + +- [ ] **Step 2.3:** In the same Deploy step, after writing .env.production and before or after the pm2 restart, optionally run the migrate on the server as a belt-and-suspenders (using the just-written .env): + +```bash +ssh ... "cd $APP_DIR && source .env.production 2>/dev/null || export \$(grep DATABASE_URL .env.production); node scripts/migrate.js || echo 'migrate on server completed or not needed'" +``` + + (Keep it non-fatal on server for now; the CI gate is the hard one.) + +- [ ] **Step 2.4:** Test the scp paths in a dry-run or note the change. + +### Task 3: Add a minimal runtime / startup guard (fail fast with clear message) + +**Files:** +- Create: `src/app/api/health/db-schema/route.ts` (or add to an existing health if present) +- Or simpler: in the admin layout or a top level, but a dedicated health is better for PM2/docker. + +- [ ] **Step 3.1:** Create a tiny health endpoint that does the same check the CI verification does (SELECT 1 FROM admin_users) using the existing pool or withDb, returns 200 or 503 with message "Schema not applied - run migrations". + +- [ ] **Step 3.2:** Wire it so the deploy can curl it after restart as a final gate (in the workflow ssh step). + +- [ ] **Step 3.3:** (Optional but recommended per defense-in-depth) Call a similar check early in getAdminUser or the admin layout and log a very loud message + return a better error than generic "does not have admin access" when the table is literally missing. + +### Task 4: Update documentation and bootstrap instructions (so humans know the right sequence) + +**Files:** +- Modify: `CLAUDE.md` (the Commands and Important File Locations + Gotchas sections) +- Modify or create: `PRODUCTION_DEPLOYMENT_CHECKLIST.md` or a new `docs/PROD_BOOTSTRAP.md` + +- [ ] **Step 4.1:** In CLAUDE.md under "Commands" and "Adding a New Brand" / auth section, add a "First production deploy / new prod DB bootstrap" subsection: + + 1. Ensure the Neon project has neon_auth enabled and the DATABASE_URL secret in Gitea points to it. + 2. (Before first code push that depends on admin) Locally or in a throwaway runner: `DATABASE_URL=prod... node scripts/migrate.js` + 3. Then `DATABASE_URL=prod... npx tsx scripts/provision-admin.ts you@real.com platform_admin` (after signing in on the prod URL). + 4. Push; the CI gate + shipped runner will keep it healthy on future deploys. + 5. If you ever see "relation admin_users does not exist" in prod logs, the DB the app is talking to is not the one that had migrate run. + +- [ ] **Step 4.2:** Add a note about the `|| echo` anti-pattern that was removed and why the new verification step exists. + +- [ ] **Step 4.3:** Mention the standalone vs npm start issue (already in logs) and that the start command on server should eventually be updated to `node .next/standalone/server.js -p 3100` (can be a follow-up task). + +### Task 5: Verification before claiming success (use the dedicated skill) + +**Files:** (none new, just process) + +- [ ] **Step 5.1:** Before merging the plan changes, use `superpowers:verification-before-completion` checklist: the change makes a fresh DB get the table, an "already applied" DB is a no-op, a deploy with missing table now fails the job early with clear output, a manual server migrate works because the files are there, the runtime health returns 200 when table present. + +- [ ] **Step 5.2:** After the PR is on a branch, trigger a deploy to a staging or the real prod (with a test DB first if possible), capture the CI log showing the new verification passing, and the app logs showing no more "Database query failed" on /admin. + +- [ ] **Step 5.3:** Run the provision script as the final user-visible test; confirm the Access Denied with email message is gone and the platform_admin can see the UI. + +- [ ] **Step 5.4:** Document the before/after in the plan or a memory file. + +### Task 6: (Stretch / follow-up) Improve the migrate script's resilience for huge init files (if the double tx ever bites again) + +**Files:** +- Modify: `scripts/migrate.js` + +- [ ] Only if during verification the 0001 apply is flaky: change the per-file execution to not wrap the file's own BEGIN/COMMIT, or use a separate connection, or exec `psql -f` (but keep node/pg for consistency). Add a comment explaining the previous fragility. + +**Rollback / emergency:** If a deploy breaks because of this, the server now has the scripts + db/migrations copied, so SSH + `source .env.production; node scripts/migrate.js` is the recovery (exactly what the user was trying to do manually). + +**Success criteria:** +- A brand new prod DB + push to main results in a green deploy + working /admin after provision. +- The error "relation \"admin_users\" does not exist" no longer appears in prod pm2 logs for normal admin flows. +- The pipeline fails loudly (with the table name in the error) instead of shipping a broken app that only shows "Access Denied". diff --git a/src/app/api/health/db-schema/route.ts b/src/app/api/health/db-schema/route.ts new file mode 100644 index 0000000..22de93b --- /dev/null +++ b/src/app/api/health/db-schema/route.ts @@ -0,0 +1,32 @@ +import { NextResponse } from "next/server"; +import { pool } from "@/lib/db"; + +/** + * Lightweight health check for the critical schema tables required by the + * admin permission system (admin_users). + * + * Used by the deploy workflow as a final post-start gate. + * Returns 200/JSON on success, 503 on missing table or connection error. + * + * This provides the "fail fast with clear message" instead of the previous + * silent "Access Denied" that only appeared on first admin request. + */ +export async function GET() { + try { + // Same check used in the CI verification step and the original error path. + await pool.query("SELECT 1 FROM admin_users LIMIT 1"); + return NextResponse.json( + { status: "ok", message: "admin_users table present" }, + { status: 200 } + ); + } catch (err: any) { + return NextResponse.json( + { + status: "error", + message: err?.message ?? "Database schema check failed", + hint: "Run migrations against the DATABASE_URL (see docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md)", + }, + { status: 503 } + ); + } +}