fix(deploy): add health/db-schema guard + curl gate; ship migration assets; document prod bootstrap (executed per systematic-debugging plan)

.gitea/workflows/deploy.yml

@@ -24,7 +24,25 @@ jobs:
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
         run: |
-          npm run migrate:one || echo "No migration script or migrations already applied"
+          set -e
+          echo "=== Running migrations against DATABASE_URL (masked value for logs) ==="
+          npm run migrate:one
+          echo "=== Post-migration verification: critical table admin_users must exist ==="
+          node -e '
+            const { Client } = require("pg");
+            const url = process.env.DATABASE_URL;
+            if (!url) { console.error("No DATABASE_URL"); process.exit(1); }
+            const c = new Client({ connectionString: url });
+            c.connect()
+              .then(() => c.query("SELECT 1 FROM admin_users LIMIT 1"))
+              .then(() => { console.log("✓ admin_users table exists and is queryable"); return c.end(); })
+              .then(() => process.exit(0))
+              .catch(e => {
+                console.error("✗ FATAL: admin_users relation missing after migrate:", e.message);
+                console.error("The deploy cannot continue. The secret DATABASE_URL must have had db/migrations/0001_init.sql applied successfully.");
+                process.exit(1);
+              });
+          '
 
       - name: Build
         env:
@@ -183,8 +201,16 @@ jobs:
           scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no package.json tyler@route.crispygoat.com:$APP_DIR/
           scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no next.config.ts tyler@route.crispygoat.com:$APP_DIR/ 2>/dev/null || true
 
+          # Ship the migration runner + SQL so the server has a recovery path (scripts/ and db/migrations/ were previously omitted from the artifact).
+          # This allows `node scripts/migrate.js` (after sourcing .env.production) to work directly on the target if needed for bootstrap or emergencies.
+          # See docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md
+          echo "Ensuring migration directories on server and copying runner + SQL..."
+          ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "mkdir -p $APP_DIR/scripts $APP_DIR/db"
+          scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no scripts/migrate.js tyler@route.crispygoat.com:$APP_DIR/scripts/migrate.js 2>/dev/null || true
+          scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r db/migrations tyler@route.crispygoat.com:$APP_DIR/db/ 2>/dev/null || true
+
           # Install deps and restart on server
           echo "Installing deps and restarting PM2..."
-          ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save"
+          ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save && sleep 4 && curl -f -s http://localhost:3100/api/health/db-schema || { echo 'Health check failed after start - schema not applied (see plan)'; exit 1; }"
 
           echo "Deployed successfully"

CLAUDE.md

@@ -101,6 +101,33 @@ The app connects to **Postgres directly** — no Supabase platform, JS client, o
 - A single shared `pg` `Pool` is exported from `src/lib/db.ts` (TBD — to be created/confirmed during the migration). Server actions and API routes import it and call `pool.query(...)` against RPC names.
 - No `NEXT_PUBLIC_SUPABASE_URL` / `SUPABASE_SERVICE_ROLE_KEY` / `@supabase/*` imports — these are being purged from the codebase.
 
+#### First production deploy / new prod DB bootstrap (critical for admin access)
+
+The `admin_users` + `admin_user_brands` tables (and the rest of the schema) come **only** from `db/migrations/0001_init.sql`.
+
+If the prod `DATABASE_URL` has never had the migrations applied, `getAdminUser()` will fail with "relation \"admin_users\" does not exist", the layout will show "Access Denied", and even a signed-in Neon Auth user will be blocked.
+
+**Correct bootstrap sequence (do this from a machine with the full source tree before the first push that exercises /admin):**
+
+1. Ensure the Gitea secret `DATABASE_URL` points at the real prod Neon Postgres (the one with `neon_auth.user` already present).
+2. Sign in once at the live prod URL (`/login`) with the email you want as the first `platform_admin`. This creates the row in `neon_auth.user`.
+3. From your laptop (or any box with the checkout):
+
+   ```bash
+   # Paste the real prod connection string (get it from Gitea secrets or the target's .env.production)
+   DATABASE_URL="postgresql://...prod-full-string..." node scripts/migrate.js
+
+   # Then provision (the script will link to the first brand it finds)
+   DATABASE_URL="postgresql://...prod-full-string..." \
+     npx tsx scripts/provision-admin.ts you@real.com platform_admin
+   ```
+
+4. Push to main. The deploy workflow now has a hard gate (see `.gitea/workflows/deploy.yml` "Run migrations" + verification query for `admin_users`) and ships the migrate runner + SQL files, so future deploys and server-side recovery are protected.
+
+See the full root-cause + plan: `docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md`
+
+The old `|| echo` masking around `npm run migrate:one` has been removed; a missing critical table will now fail the CI job with a clear message.
+
 #### SECURITY DEFINER RPCs + Brand Scoping
 
 The app uses **PostgreSQL SECURITY DEFINER functions** for all data access. These run with the function owner's privileges and bypass any future RLS. This means:

docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md

@@ -0,0 +1,147 @@
+# Production DB Schema Migration Reliability Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:systematic-debugging (completed root cause), superpowers:writing-plans (this), superpowers:test-driven-development where code changes have tests, superpowers:verification-before-completion, and superpowers:executing-plans or subagent-driven-development to implement task-by-task. Steps use checkbox syntax.
+
+**Goal:** Ensure that every production deploy (push to main) results in the full schema from `db/migrations/0001_init.sql` (including the `admin_users` and `admin_user_brands` tables required by `getAdminUser()`) being present in the `DATABASE_URL` that the running app connects to via its pool / drizzle client. Eliminate "relation does not exist" errors and the resulting Access Denied screen for properly provisioned Neon Auth users.
+
+**Architecture:** 
+- Make the CI "Run migrations" step a hard gate (fatal on failure, plus explicit post-migrate verification that critical tables exist).
+- Ship the minimal migration runner + SQL files as part of the deploy artifact so the target server has a recovery path.
+- Add a lightweight post-deploy / startup verification in the app or deploy script (fail fast with clear message instead of silent 500s on first admin request).
+- Keep the "migrate from full source locally" path working for initial prod DB bootstrap and emergencies.
+- Do not change the core migration logic or 0001_init.sql in this plan (that would be a separate architectural change if the double-BEGIN wrapping proves fragile).
+
+**Tech Stack:** Gitea Actions (YAML), Node 22 + pg + drizzle on target, Next.js standalone output, pm2 on Ubuntu server, Neon Postgres (with neon_auth schema).
+
+**Root Cause (from systematic-debugging Phase 1):** The prod runtime DB lacked the `admin_users` table because (1) the migration step in `.gitea/workflows/deploy.yml` used `|| echo` making any failure (connection, SQL error in the huge 0001 file, FK to neon_auth.user, tx nesting from the file's BEGIN + script's BEGIN) non-fatal, (2) only `.next/`, `public/`, `package.json` (and optional next.config) are scp'd — `scripts/migrate.js` and `db/migrations/` are never on the server, (3) no verification after "migrate" or at app startup that the tables the admin permission layer depends on actually exist, (4) the `.env.production` written from the same secret as the CI migrate step was used, but the apply didn't happen or was skipped due to _migrations state or partial rollback.
+
+**Evidence Gathered:**
+- deploy.yml: Run migrations step, limited scp, .env.production printf, pm2 "npm start".
+- scripts/migrate.js: dotenv .env.local + env override, _migrations tracking, per-file client.query(sql) inside script tx, re-throw on error.
+- db/migrations/0001_init.sql: explicit CREATE TABLE admin_users (with FK to neon_auth.user), admin_user_brands, brands; file starts with BEGIN;.
+- db/client.ts + src/lib/admin-permissions.ts: withPlatformAdmin → drizzle select on adminUsers from schema (the exact query that 42P01s).
+- next.config.ts: output: 'standalone' (explains pm2 warning).
+- Runtime logs: the repeated "Database query failed" + "relation does not exist", app starts fine.
+- Git history: recent deploy "fixes" focused on SSH/env writing, not migration reliability.
+
+**Files to touch (decomposition by responsibility):**
+- `.gitea/workflows/deploy.yml` (CI pipeline gates + artifact contents)
+- `scripts/migrate.js` (minor hardening if needed for verification hook)
+- `src/app/api/health/route.ts` or similar (new, for startup/schema check — or add to existing)
+- `CLAUDE.md` + `PRODUCTION_DEPLOYMENT_CHECKLIST.md` (docs)
+- Possibly a small `scripts/verify-prod-schema.js` helper
+
+---
+### Task 1: Make CI migration step a hard failure + add explicit verification for admin_users
+
+**Files:**
+- Modify: `.gitea/workflows/deploy.yml:23-27` (the Run migrations step and surrounding)
+
+- [ ] **Step 1.1:** Replace the non-fatal migration line with a strict block that fails the job if migrate fails or the critical table is missing after.
+
+```yaml
+- name: Run migrations
+  env:
+    DATABASE_URL: ${{ secrets.DATABASE_URL }}
+  run: |
+    set -e
+    echo "=== Running migrations against prod DB ==="
+    npm run migrate:one
+    echo "=== Verifying critical schema (admin_users) ==="
+    node -e '
+      const {Client} = require("pg");
+      const c = new Client({connectionString: process.env.DATABASE_URL});
+      c.connect().then(() => c.query("SELECT 1 FROM admin_users LIMIT 1")).then(() => {
+        console.log("✓ admin_users table exists");
+        return c.end();
+      }).then(() => process.exit(0)).catch(e => {
+        console.error("✗ admin_users missing or inaccessible:", e.message);
+        process.exit(1);
+      });
+    '
+```
+
+- [ ] **Step 1.2:** Run a local simulation or note that the Gitea runner will now fail the whole deploy if the secret DB is missing the table (good — forces the bootstrap to happen before code that depends on it ships).
+
+- [ ] **Step 1.3:** Commit the yml change with message referencing the root cause (missing table in prod due to masked migration).
+
+### Task 2: Ship migration capability in the deploy artifact so server has a recovery path
+
+**Files:**
+- Modify: `.gitea/workflows/deploy.yml` in the "Deploy" step (the scp and ssh sections)
+
+- [ ] **Step 2.1:** Add scp for the migration assets (after the existing public/.next scp):
+
+```bash
+echo "Copying migration runner and SQL..."
+scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r scripts/migrate.js tyler@...:$APP_DIR/scripts/ || true
+scp -o ConnectTimeout=15 -o StrictHostKeyChecking=no -r db/migrations tyler@...:$APP_DIR/db/ || true
+```
+
+- [ ] **Step 2.2:** Update the server ssh install/restart line to also ensure the scripts dir has the right perms if needed, but mainly document that now `node scripts/migrate.js` will work on the server with the .env.production.
+
+- [ ] **Step 2.3:** In the same Deploy step, after writing .env.production and before or after the pm2 restart, optionally run the migrate on the server as a belt-and-suspenders (using the just-written .env):
+
+```bash
+ssh ... "cd $APP_DIR && source .env.production 2>/dev/null || export \$(grep DATABASE_URL .env.production); node scripts/migrate.js || echo 'migrate on server completed or not needed'"
+```
+
+  (Keep it non-fatal on server for now; the CI gate is the hard one.)
+
+- [ ] **Step 2.4:** Test the scp paths in a dry-run or note the change.
+
+### Task 3: Add a minimal runtime / startup guard (fail fast with clear message)
+
+**Files:**
+- Create: `src/app/api/health/db-schema/route.ts` (or add to an existing health if present)
+- Or simpler: in the admin layout or a top level, but a dedicated health is better for PM2/docker.
+
+- [ ] **Step 3.1:** Create a tiny health endpoint that does the same check the CI verification does (SELECT 1 FROM admin_users) using the existing pool or withDb, returns 200 or 503 with message "Schema not applied - run migrations".
+
+- [ ] **Step 3.2:** Wire it so the deploy can curl it after restart as a final gate (in the workflow ssh step).
+
+- [ ] **Step 3.3:** (Optional but recommended per defense-in-depth) Call a similar check early in getAdminUser or the admin layout and log a very loud message + return a better error than generic "does not have admin access" when the table is literally missing.
+
+### Task 4: Update documentation and bootstrap instructions (so humans know the right sequence)
+
+**Files:**
+- Modify: `CLAUDE.md` (the Commands and Important File Locations + Gotchas sections)
+- Modify or create: `PRODUCTION_DEPLOYMENT_CHECKLIST.md` or a new `docs/PROD_BOOTSTRAP.md`
+
+- [ ] **Step 4.1:** In CLAUDE.md under "Commands" and "Adding a New Brand" / auth section, add a "First production deploy / new prod DB bootstrap" subsection:
+
+  1. Ensure the Neon project has neon_auth enabled and the DATABASE_URL secret in Gitea points to it.
+  2. (Before first code push that depends on admin) Locally or in a throwaway runner: `DATABASE_URL=prod... node scripts/migrate.js`
+  3. Then `DATABASE_URL=prod... npx tsx scripts/provision-admin.ts you@real.com platform_admin` (after signing in on the prod URL).
+  4. Push; the CI gate + shipped runner will keep it healthy on future deploys.
+  5. If you ever see "relation admin_users does not exist" in prod logs, the DB the app is talking to is not the one that had migrate run.
+
+- [ ] **Step 4.2:** Add a note about the `|| echo` anti-pattern that was removed and why the new verification step exists.
+
+- [ ] **Step 4.3:** Mention the standalone vs npm start issue (already in logs) and that the start command on server should eventually be updated to `node .next/standalone/server.js -p 3100` (can be a follow-up task).
+
+### Task 5: Verification before claiming success (use the dedicated skill)
+
+**Files:** (none new, just process)
+
+- [ ] **Step 5.1:** Before merging the plan changes, use `superpowers:verification-before-completion` checklist: the change makes a fresh DB get the table, an "already applied" DB is a no-op, a deploy with missing table now fails the job early with clear output, a manual server migrate works because the files are there, the runtime health returns 200 when table present.
+
+- [ ] **Step 5.2:** After the PR is on a branch, trigger a deploy to a staging or the real prod (with a test DB first if possible), capture the CI log showing the new verification passing, and the app logs showing no more "Database query failed" on /admin.
+
+- [ ] **Step 5.3:** Run the provision script as the final user-visible test; confirm the Access Denied with email message is gone and the platform_admin can see the UI.
+
+- [ ] **Step 5.4:** Document the before/after in the plan or a memory file.
+
+### Task 6: (Stretch / follow-up) Improve the migrate script's resilience for huge init files (if the double tx ever bites again)
+
+**Files:**
+- Modify: `scripts/migrate.js`
+
+- [ ] Only if during verification the 0001 apply is flaky: change the per-file execution to not wrap the file's own BEGIN/COMMIT, or use a separate connection, or exec `psql -f` (but keep node/pg for consistency). Add a comment explaining the previous fragility.
+
+**Rollback / emergency:** If a deploy breaks because of this, the server now has the scripts + db/migrations copied, so SSH + `source .env.production; node scripts/migrate.js` is the recovery (exactly what the user was trying to do manually).
+
+**Success criteria:**
+- A brand new prod DB + push to main results in a green deploy + working /admin after provision.
+- The error "relation \"admin_users\" does not exist" no longer appears in prod pm2 logs for normal admin flows.
+- The pipeline fails loudly (with the table name in the error) instead of shipping a broken app that only shows "Access Denied".

src/app/api/health/db-schema/route.ts

@@ -0,0 +1,32 @@
+import { NextResponse } from "next/server";
+import { pool } from "@/lib/db";
+
+/**
+ * Lightweight health check for the critical schema tables required by the
+ * admin permission system (admin_users).
+ *
+ * Used by the deploy workflow as a final post-start gate.
+ * Returns 200/JSON on success, 503 on missing table or connection error.
+ *
+ * This provides the "fail fast with clear message" instead of the previous
+ * silent "Access Denied" that only appeared on first admin request.
+ */
+export async function GET() {
+  try {
+    // Same check used in the CI verification step and the original error path.
+    await pool.query("SELECT 1 FROM admin_users LIMIT 1");
+    return NextResponse.json(
+      { status: "ok", message: "admin_users table present" },
+      { status: 200 }
+    );
+  } catch (err: any) {
+    return NextResponse.json(
+      {
+        status: "error",
+        message: err?.message ?? "Database schema check failed",
+        hint: "Run migrations against the DATABASE_URL (see docs/superpowers/plans/2026-06-prod-db-schema-migration-reliability.md)",
+      },
+      { status: 503 }
+    );
+  }
+}