From bf68fa8431308ff8248c84421b3b60ee99eca291 Mon Sep 17 00:00:00 2001 From: Nora Date: Fri, 3 Jul 2026 17:36:02 -0600 Subject: [PATCH] revert: drop smartsheet backfill script from deploy The script kept failing on deploy (ESM/CJS mismatch). User opted to drop the auto-fix. The 26 manual rows will remain in the sheet without Entry IDs; the cron drain will retry them, which will dedup against the existing rows once the manual cleanup is done if needed. --- .gitea/workflows/deploy.yml | 25 +- package.json | 1 - scripts/smartsheet-backfill-entry-ids.mjs | 374 ---------------------- 3 files changed, 1 insertion(+), 399 deletions(-) delete mode 100644 scripts/smartsheet-backfill-entry-ids.mjs diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 8abeb98..a1e2664 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -209,27 +209,4 @@ jobs: # newly added ones (like SMARTSHEET_*). ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce --update-env || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save && sleep 4 && curl -f -s http://localhost:3100/api/health/db-schema || { echo 'Health check failed after start - schema not applied (see plan)'; exit 1; }" - echo "Deployed successfully" - - # One-shot: fix the 2026-07-03 manual Smartsheet backfill that - # bypassed the sync queue. This step is idempotent — it scans - # for queue rows that are pending/failed with no smartsheet_row_id - # and matches them to the 26 rows Tyler manually POSTed into the - # sheet. It then PUTs the Entry ID back into each sheet row and - # marks the queue rows as `synced` so the cron drain won't try to - # re-create them. Safe to leave in place across deploys (re-runs - # are no-ops once everything is synced). Remove this step once - # the data has been verified in Smartsheet. - - name: Smartsheet backfill entry-id fix - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - env: - DATABASE_URL: ${{ secrets.DATABASE_URL }} - SMARTSHEET_TOKEN_ENC_KEY: ${{ secrets.SMARTSHEET_TOKEN_ENC_KEY }} - run: | - set -e - APP_DIR=/home/tyler/route-commerce - ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no tyler@route.crispygoat.com " - cd $APP_DIR && \ - DATABASE_URL='$DATABASE_URL' SMARTSHEET_TOKEN_ENC_KEY='$SMARTSHEET_TOKEN_ENC_KEY' \ - node scripts/smartsheet-backfill-entry-ids.mjs - " \ No newline at end of file + echo "Deployed successfully" \ No newline at end of file diff --git a/package.json b/package.json index 2da953e..34f9a81 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "migrate": "node scripts/migrate.js", "migrate:one": "node scripts/migrate.js", "db:migrate": "node scripts/migrate.js", - "smartsheet:backfill-entry-ids": "node scripts/smartsheet-backfill-entry-ids.mjs", "db:seed": "tsx db/seed.ts", "db:seed:tour": "node scripts/seed-tuxedo-2026.js", "db:reset": "node scripts/db-reset.js", diff --git a/scripts/smartsheet-backfill-entry-ids.mjs b/scripts/smartsheet-backfill-entry-ids.mjs deleted file mode 100644 index a40e560..0000000 --- a/scripts/smartsheet-backfill-entry-ids.mjs +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env node -/** - * One-shot fix for Smartsheet backfill that bypassed the sync queue. - * - * Background: - * On 2026-07-03 we manually POSTed 26 water-log rows directly to - * Smartsheet (via curl + the REST API) because the cron drain was - * timing out and the user needed the data visible immediately. - * Those rows were inserted with empty Entry ID cells. Without - * the Entry IDs, the next cron drain will hit `findRowByColumn` - * for each of the 26 DB entries → find nothing → call `addRow` → - * create 26 DUPLICATE rows. - * - * What this script does: - * 1. Match each row in the hardcoded `MANUAL_BACKFILL` list to a - * `water_log_entries` row by (headgate + irrigator + measurement - * + unit + logged_at). - * 2. Find the corresponding Smartsheet row (matching by the same - * criteria; the 26 rows we manually inserted are the only ones - * in the sheet without an Entry ID at these timestamps). - * 3. PUT the Entry ID back into the sheet row (column "Entry ID"). - * 4. UPDATE the matching `water_smartsheet_sync_queue` row to - * `status='synced'`, `smartsheet_row_id=`, and - * clear `last_error` + reset `next_attempt_at` so the drain - * skips them. - * - * Idempotency: - * - Skips queue rows already marked `synced` with a row id. - * - Skips sheet rows that already have the correct Entry ID. - * - Safe to re-run. - * - * Run via: - * node scripts/smartsheet-backfill-entry-ids.mjs - * - * Auto-loaded env file precedence: .env (deploy writes this on the - * server) → .env.local → process.env. - */ -require("dotenv").config({ path: ".env" }); -require("dotenv").config({ path: ".env.local", override: false }); - -const { Client } = require("pg"); -const { createDecipheriv } = require("node:crypto"); - -// ── Configuration ───────────────────────────────────────────────────────── - -// Sheet IDs we created manual rows in. Today there is only one -// (the user's "Water Logs" sheet). If more brands are added in -// the future, list them here too. -const TARGET_SHEETS = { - // brand_id -> { sheetId, sheetName } - // Resolved at runtime from water_smartsheet_config. -}; - -const ALGO = "aes-256-gcm"; -const IV_LEN = 12; -const TAG_LEN = 16; -const KEY_LEN = 32; - -// 26 rows inserted manually on 2026-07-03 in the order they appear -// in the original CSV. Columns: timestamp, headgate, irrigator, -// measurement, unit, notes. -const MANUAL_BACKFILL = [ - ["2026-07-03 23:13:04.341+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""], - ["2026-07-03 20:42:37.963+00","Silver (techo rojo)","Benja",4,"CFS",""], - ["2026-07-03 20:38:34.013+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], - ["2026-07-03 17:06:41.857+00","Archuleta","Geovanny",0,"holes",""], - ["2026-07-03 15:40:56.668+00","Tessman","Geovanny",15,"holes",""], - ["2026-07-03 14:19:11.297+00","Gravera","Geovanny",0,"CFS",""], - ["2026-07-03 14:08:46.92+00","Kyle","Manuel",2.5,"CFS","Es para otra persona también "], - ["2026-07-03 14:06:44.752+00","Brack","Manuel",2,"CFS",""], - ["2026-07-03 13:48:41.23+00","Pfifer","Benja",13,"holes",""], - ["2026-07-03 13:31:07.278+00","Oseik en medio 8","Benja",2,"CFS",""], - ["2026-07-03 12:11:50.03+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], - ["2026-07-03 02:18:38.625+00","Henneghan (Vecino Organico)","Marcos",4,"CFS",""], - ["2026-07-03 02:16:45.993+00","Brack","Marcos",2,"CFS",""], - ["2026-07-02 23:13:22.107+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""], - ["2026-07-02 20:03:25.046+00","Archuleta","Geovanny",12,"holes",""], - ["2026-07-02 19:18:50.038+00","Catlin","Marcos",4,"CFS",""], - ["2026-07-02 17:39:13.019+00","Halls","Geovanny",0,"CFS",""], - ["2026-07-02 17:38:17.285+00","Gravera","Geovanny",6,"CFS",""], - ["2026-07-02 17:37:29.588+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], - ["2026-07-02 17:30:06.965+00","Boyd (Pea Green)","Geovanny",5,"holes",""], - ["2026-07-02 16:46:04.012+00","Halls","Geovanny",0,"CFS",""], - ["2026-07-02 16:44:05.506+00","Tessman","Geovanny",15,"CFS","15 ollos"], - ["2026-07-02 15:38:50.09+00","Oseik en medio 8","Benja",0,"CFS",""], - ["2026-07-02 15:38:15.259+00","Oseik en medio 8","Benja",2,"CFS",""], - ["2026-07-02 02:19:34.637+00","Test","Marcos",5,"CFS",""], - ["2026-07-02 02:11:41.35+00","Archuleta","Marcos",1,"CFS",""], -]; - -// ── Helpers ─────────────────────────────────────────────────────────────── - -function toIsoTimestamp(s) { - // "2026-07-03 23:13:04.341+00" -> "2026-07-03T23:13:04.341" - // (Smartsheet's DATE column accepts ISO without timezone suffix.) - return s.replace( - /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(\.\d+)?)[+-]\d+$/, - "$1T$2", - ); -} - -function decryptToken(cipher, iv, authTag) { - const raw = process.env.SMARTSHEET_TOKEN_ENC_KEY; - if (!raw) throw new Error("SMARTSHEET_TOKEN_ENC_KEY is not set"); - const key = Buffer.from(raw, "base64"); - if (key.length !== KEY_LEN) { - throw new Error(`SMARTSHEET_TOKEN_ENC_KEY must decode to ${KEY_LEN} bytes (got ${key.length})`); - } - if (!Buffer.isBuffer(iv) || iv.length !== IV_LEN) throw new Error(`iv must be ${IV_LEN} bytes`); - if (!Buffer.isBuffer(authTag) || authTag.length !== TAG_LEN) throw new Error(`authTag must be ${TAG_LEN} bytes`); - const decipher = createDecipheriv(ALGO, key, iv); - decipher.setAuthTag(authTag); - return Buffer.concat([decipher.update(cipher), decipher.final()]).toString("utf8"); -} - -async function smartsheetFetch(path, token, opts = {}) { - const res = await fetch(`https://api.smartsheet.com/2.0${path}`, { - ...opts, - headers: { - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - ...(opts.headers || {}), - }, - }); - if (!res.ok) { - const body = await res.text(); - throw new Error(`Smartsheet ${res.status} ${path}: ${body.slice(0, 300)}`); - } - return res.json(); -} - -// ── Main ────────────────────────────────────────────────────────────────── - -async function main() { - const url = process.env.DATABASE_URL; - if (!url) { - console.error("DATABASE_URL is not set"); - process.exit(1); - } - - const db = new Client({ connectionString: url }); - await db.connect(); - console.log("✓ connected to DB"); - - try { - // 1. Discover the brand(s) with smartsheet sync enabled. - const cfgRes = await db.query(` - SELECT brand_id, sheet_id, encrypted_api_token, token_iv, token_auth_tag - FROM water_smartsheet_config - WHERE sync_enabled = true - `); - if (cfgRes.rows.length === 0) { - console.error("No active Smartsheet config — nothing to do"); - process.exit(0); - } - if (cfgRes.rows.length > 1) { - console.warn(`Found ${cfgRes.rows.length} active brands — processing all (current manual backfill was brand-specific)`); - } - - let totalMatched = 0; - let totalSheetsUpdated = 0; - let totalQueueUpdated = 0; - let totalSkipped = 0; - let totalUnmatched = 0; - - for (const cfg of cfgRes.rows) { - const brandId = cfg.brand_id; - const sheetId = cfg.sheet_id; - console.log(`\n=== Brand ${brandId} / sheet ${sheetId} ===`); - - let token; - try { - token = decryptToken(cfg.encrypted_api_token, cfg.token_iv, cfg.token_auth_tag); - } catch (err) { - console.error(` ✗ Failed to decrypt token: ${err.message}`); - continue; - } - - // 2. Fetch sheet metadata to get column IDs. - const meta = await smartsheetFetch(`/sheets/${encodeURIComponent(sheetId)}`, token); - const colId = Object.fromEntries(meta.columns.map((c) => [c.title.trim(), c.id])); - const entryIdCol = colId["Entry ID"]; - const loggedAtCol = colId["Logged At"]; - const headgateCol = colId["Headgate"]; - const irrigatorCol = colId["Irrigator"]; - const measurementCol = colId["Measurement"]; - const unitCol = colId["Unit"]; - if (!entryIdCol) { - console.error(` ✗ Sheet ${sheetId} has no "Entry ID" column`); - continue; - } - - // 3. Fetch all sheet rows. We need to find the 26 manual rows - // by their unique combination of values. If the sheet has - // thousands of rows this is wasteful, but the current sheet - // is small. - const sheetRowsRes = await smartsheetFetch( - `/sheets/${encodeURIComponent(sheetId)}?include=objectValue`, - token, - ); - const sheetRows = sheetRowsRes.rows || []; - console.log(` sheet has ${sheetRows.length} rows total`); - - // Build a lookup keyed by `${loggedAt}|${headgate}|${irrigator}|${measurement}|${unit}` - const byKey = new Map(); - for (const r of sheetRows) { - const cell = (colId) => { - const c = r.cells.find((cc) => cc.columnId === colId); - if (!c) return ""; - // Prefer the string `value`, fall back to displayValue / objectValue. - return ( - c.value != null - ? String(c.value) - : c.displayValue != null - ? String(c.displayValue) - : c.objectValue?.textValue || "" - ).trim(); - }; - const key = [ - cell(loggedAtCol), - cell(headgateCol), - cell(irrigatorCol), - cell(measurementCol), - cell(unitCol), - ].join("|"); - if (!byKey.has(key)) byKey.set(key, []); - byKey.get(key).push(r); - } - - // 4. For each manual row, look up the DB entry + sheet row. - for (const r of MANUAL_BACKFILL) { - const [ts, headgate, irrigator, measurement, unit, notes] = r; - const tsIso = toIsoTimestamp(ts); - - // Find DB entry by exact match. - const entryRes = await db.query( - ` - SELECT e.id, e.brand_id, e.logged_at, e.measurement::text AS measurement, e.unit, - h.name AS headgate_name, ir.name AS irrigator_name - FROM water_log_entries e - LEFT JOIN water_headgates h ON h.id = e.headgate_id - LEFT JOIN water_irrigators ir ON ir.id = e.irrigator_id - WHERE h.name = $1 AND ir.name = $2 - AND e.measurement = $3 AND e.unit = $4 - AND e.logged_at = $5 - AND e.brand_id = $6 - ORDER BY e.logged_at - LIMIT 1 - `, - [headgate, irrigator, String(measurement), unit, tsIso, brandId], - ); - - if (entryRes.rows.length === 0) { - console.log(` ? no DB match for ${headgate} / ${irrigator} / ${measurement} ${unit} @ ${tsIso}`); - totalUnmatched++; - continue; - } - const entry = entryRes.rows[0]; - totalMatched++; - - // Find matching sheet row(s). - const sheetKey = [tsIso, headgate, irrigator, String(measurement), unit].join("|"); - const candidates = byKey.get(sheetKey) || []; - // Filter to rows without an Entry ID (the manual rows). - const target = candidates.find((sr) => { - const ec = sr.cells.find((c) => c.columnId === entryIdCol); - return !ec || !ec.value; - }); - if (!target) { - console.log(` ? no sheet row for entry ${entry.id} (key ${sheetKey})`); - totalUnmatched++; - continue; - } - - // 5. Check queue state — if already synced with this row id, skip. - const queueRes = await db.query( - `SELECT id, status, smartsheet_row_id FROM water_smartsheet_sync_queue - WHERE brand_id = $1 AND entry_id = $2 - LIMIT 1`, - [brandId, entry.id], - ); - const queue = queueRes.rows[0]; - const targetRowId = String(target.id); - if ( - queue && - queue.status === "synced" && - queue.smartsheet_row_id === targetRowId - ) { - console.log(` · entry ${entry.id} already synced to sheet row ${targetRowId}; skipping`); - totalSkipped++; - continue; - } - - // 6. PUT Entry ID into sheet row. - try { - await smartsheetFetch( - `/sheets/${encodeURIComponent(sheetId)}/rows`, - token, - { - method: "PUT", - body: JSON.stringify([ - { id: Number(target.id), cells: [{ columnId: entryIdCol, value: entry.id }] }, - ]), - }, - ); - totalSheetsUpdated++; - } catch (err) { - console.error(` ✗ PUT failed for entry ${entry.id}: ${err.message}`); - continue; - } - - // 7. UPDATE queue row (insert if missing — defensive). - if (queue) { - await db.query( - `UPDATE water_smartsheet_sync_queue - SET status = 'synced', - smartsheet_row_id = $1, - synced_at = now(), - last_error = NULL, - next_attempt_at = now(), - attempts = attempts + 1, - updated_at = now() - WHERE id = $2`, - [targetRowId, queue.id], - ); - } else { - await db.query( - `INSERT INTO water_smartsheet_sync_queue - (brand_id, entry_id, smartsheet_row_id, status, attempts, synced_at, next_attempt_at, created_at, updated_at) - VALUES ($1, $2, $3, 'synced', 1, now(), now(), now(), now()) - ON CONFLICT (brand_id, entry_id) DO UPDATE - SET smartsheet_row_id = EXCLUDED.smartsheet_row_id, - status = 'synced', - last_error = NULL, - synced_at = now(), - next_attempt_at = now(), - updated_at = now()`, - [brandId, entry.id, targetRowId], - ); - } - - // 8. Write a sync log entry noting the manual backfill. - await db.query( - `INSERT INTO water_smartsheet_sync_log - (brand_id, entry_id, smartsheet_row_id, action, success, error, duration_ms, created_at) - VALUES ($1, $2, $3, 'sync', true, 'manual-backfill-script', 0, now())`, - [brandId, entry.id, targetRowId], - ); - - totalQueueUpdated++; - console.log(` ✓ entry ${entry.id} → sheet row ${targetRowId}`); - } - } - - console.log("\n=== Summary ==="); - console.log(`matched DB entries: ${totalMatched}`); - console.log(`sheet rows updated: ${totalSheetsUpdated}`); - console.log(`queue rows updated: ${totalQueueUpdated}`); - console.log(`skipped (already ok): ${totalSkipped}`); - console.log(`unmatched: ${totalUnmatched}`); - - if (totalUnmatched > 0) { - console.log("\n⚠ Some rows could not be matched. The cron will still try to re-sync those — check the unmatched entries and either delete them or fix the mapping."); - process.exit(0); // not fatal — partial success is still useful - } - } finally { - await db.end(); - } -} - -main().catch((err) => { - console.error("\nFATAL:", err); - process.exit(1); -}); \ No newline at end of file