diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 382b91a..39748f9 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -208,4 +208,27 @@ jobs: # newly added ones (like SMARTSHEET_*). ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce --update-env || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save && sleep 4 && curl -f -s http://localhost:3100/api/health/db-schema || { echo 'Health check failed after start - schema not applied (see plan)'; exit 1; }" - echo "Deployed successfully" \ No newline at end of file + echo "Deployed successfully" + + # One-shot: fix the 2026-07-03 manual Smartsheet backfill that + # bypassed the sync queue. This step is idempotent — it scans + # for queue rows that are pending/failed with no smartsheet_row_id + # and matches them to the 26 rows Tyler manually POSTed into the + # sheet. It then PUTs the Entry ID back into each sheet row and + # marks the queue rows as `synced` so the cron drain won't try to + # re-create them. Safe to leave in place across deploys (re-runs + # are no-ops once everything is synced). Remove this step once + # the data has been verified in Smartsheet. + - name: Smartsheet backfill entry-id fix + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SMARTSHEET_TOKEN_ENC_KEY: ${{ secrets.SMARTSHEET_TOKEN_ENC_KEY }} + run: | + set -e + APP_DIR=/home/tyler/route-commerce + ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no tyler@route.crispygoat.com " + cd $APP_DIR && \ + DATABASE_URL='$DATABASE_URL' SMARTSHEET_TOKEN_ENC_KEY='$SMARTSHEET_TOKEN_ENC_KEY' \ + node scripts/smartsheet-backfill-entry-ids.mjs + " \ No newline at end of file diff --git a/package.json b/package.json index 34f9a81..2da953e 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "migrate": "node scripts/migrate.js", "migrate:one": "node scripts/migrate.js", "db:migrate": "node scripts/migrate.js", + "smartsheet:backfill-entry-ids": "node scripts/smartsheet-backfill-entry-ids.mjs", "db:seed": "tsx db/seed.ts", "db:seed:tour": "node scripts/seed-tuxedo-2026.js", "db:reset": "node scripts/db-reset.js", diff --git a/scripts/smartsheet-backfill-entry-ids.mjs b/scripts/smartsheet-backfill-entry-ids.mjs new file mode 100644 index 0000000..a40e560 --- /dev/null +++ b/scripts/smartsheet-backfill-entry-ids.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +/** + * One-shot fix for Smartsheet backfill that bypassed the sync queue. + * + * Background: + * On 2026-07-03 we manually POSTed 26 water-log rows directly to + * Smartsheet (via curl + the REST API) because the cron drain was + * timing out and the user needed the data visible immediately. + * Those rows were inserted with empty Entry ID cells. Without + * the Entry IDs, the next cron drain will hit `findRowByColumn` + * for each of the 26 DB entries → find nothing → call `addRow` → + * create 26 DUPLICATE rows. + * + * What this script does: + * 1. Match each row in the hardcoded `MANUAL_BACKFILL` list to a + * `water_log_entries` row by (headgate + irrigator + measurement + * + unit + logged_at). + * 2. Find the corresponding Smartsheet row (matching by the same + * criteria; the 26 rows we manually inserted are the only ones + * in the sheet without an Entry ID at these timestamps). + * 3. PUT the Entry ID back into the sheet row (column "Entry ID"). + * 4. UPDATE the matching `water_smartsheet_sync_queue` row to + * `status='synced'`, `smartsheet_row_id=`, and + * clear `last_error` + reset `next_attempt_at` so the drain + * skips them. + * + * Idempotency: + * - Skips queue rows already marked `synced` with a row id. + * - Skips sheet rows that already have the correct Entry ID. + * - Safe to re-run. + * + * Run via: + * node scripts/smartsheet-backfill-entry-ids.mjs + * + * Auto-loaded env file precedence: .env (deploy writes this on the + * server) → .env.local → process.env. + */ +require("dotenv").config({ path: ".env" }); +require("dotenv").config({ path: ".env.local", override: false }); + +const { Client } = require("pg"); +const { createDecipheriv } = require("node:crypto"); + +// ── Configuration ───────────────────────────────────────────────────────── + +// Sheet IDs we created manual rows in. Today there is only one +// (the user's "Water Logs" sheet). If more brands are added in +// the future, list them here too. +const TARGET_SHEETS = { + // brand_id -> { sheetId, sheetName } + // Resolved at runtime from water_smartsheet_config. +}; + +const ALGO = "aes-256-gcm"; +const IV_LEN = 12; +const TAG_LEN = 16; +const KEY_LEN = 32; + +// 26 rows inserted manually on 2026-07-03 in the order they appear +// in the original CSV. Columns: timestamp, headgate, irrigator, +// measurement, unit, notes. +const MANUAL_BACKFILL = [ + ["2026-07-03 23:13:04.341+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""], + ["2026-07-03 20:42:37.963+00","Silver (techo rojo)","Benja",4,"CFS",""], + ["2026-07-03 20:38:34.013+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], + ["2026-07-03 17:06:41.857+00","Archuleta","Geovanny",0,"holes",""], + ["2026-07-03 15:40:56.668+00","Tessman","Geovanny",15,"holes",""], + ["2026-07-03 14:19:11.297+00","Gravera","Geovanny",0,"CFS",""], + ["2026-07-03 14:08:46.92+00","Kyle","Manuel",2.5,"CFS","Es para otra persona también "], + ["2026-07-03 14:06:44.752+00","Brack","Manuel",2,"CFS",""], + ["2026-07-03 13:48:41.23+00","Pfifer","Benja",13,"holes",""], + ["2026-07-03 13:31:07.278+00","Oseik en medio 8","Benja",2,"CFS",""], + ["2026-07-03 12:11:50.03+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], + ["2026-07-03 02:18:38.625+00","Henneghan (Vecino Organico)","Marcos",4,"CFS",""], + ["2026-07-03 02:16:45.993+00","Brack","Marcos",2,"CFS",""], + ["2026-07-02 23:13:22.107+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""], + ["2026-07-02 20:03:25.046+00","Archuleta","Geovanny",12,"holes",""], + ["2026-07-02 19:18:50.038+00","Catlin","Marcos",4,"CFS",""], + ["2026-07-02 17:39:13.019+00","Halls","Geovanny",0,"CFS",""], + ["2026-07-02 17:38:17.285+00","Gravera","Geovanny",6,"CFS",""], + ["2026-07-02 17:37:29.588+00","Boyd (Pea Green)","Geovanny",5,"CFS",""], + ["2026-07-02 17:30:06.965+00","Boyd (Pea Green)","Geovanny",5,"holes",""], + ["2026-07-02 16:46:04.012+00","Halls","Geovanny",0,"CFS",""], + ["2026-07-02 16:44:05.506+00","Tessman","Geovanny",15,"CFS","15 ollos"], + ["2026-07-02 15:38:50.09+00","Oseik en medio 8","Benja",0,"CFS",""], + ["2026-07-02 15:38:15.259+00","Oseik en medio 8","Benja",2,"CFS",""], + ["2026-07-02 02:19:34.637+00","Test","Marcos",5,"CFS",""], + ["2026-07-02 02:11:41.35+00","Archuleta","Marcos",1,"CFS",""], +]; + +// ── Helpers ─────────────────────────────────────────────────────────────── + +function toIsoTimestamp(s) { + // "2026-07-03 23:13:04.341+00" -> "2026-07-03T23:13:04.341" + // (Smartsheet's DATE column accepts ISO without timezone suffix.) + return s.replace( + /^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(\.\d+)?)[+-]\d+$/, + "$1T$2", + ); +} + +function decryptToken(cipher, iv, authTag) { + const raw = process.env.SMARTSHEET_TOKEN_ENC_KEY; + if (!raw) throw new Error("SMARTSHEET_TOKEN_ENC_KEY is not set"); + const key = Buffer.from(raw, "base64"); + if (key.length !== KEY_LEN) { + throw new Error(`SMARTSHEET_TOKEN_ENC_KEY must decode to ${KEY_LEN} bytes (got ${key.length})`); + } + if (!Buffer.isBuffer(iv) || iv.length !== IV_LEN) throw new Error(`iv must be ${IV_LEN} bytes`); + if (!Buffer.isBuffer(authTag) || authTag.length !== TAG_LEN) throw new Error(`authTag must be ${TAG_LEN} bytes`); + const decipher = createDecipheriv(ALGO, key, iv); + decipher.setAuthTag(authTag); + return Buffer.concat([decipher.update(cipher), decipher.final()]).toString("utf8"); +} + +async function smartsheetFetch(path, token, opts = {}) { + const res = await fetch(`https://api.smartsheet.com/2.0${path}`, { + ...opts, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + ...(opts.headers || {}), + }, + }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Smartsheet ${res.status} ${path}: ${body.slice(0, 300)}`); + } + return res.json(); +} + +// ── Main ────────────────────────────────────────────────────────────────── + +async function main() { + const url = process.env.DATABASE_URL; + if (!url) { + console.error("DATABASE_URL is not set"); + process.exit(1); + } + + const db = new Client({ connectionString: url }); + await db.connect(); + console.log("✓ connected to DB"); + + try { + // 1. Discover the brand(s) with smartsheet sync enabled. + const cfgRes = await db.query(` + SELECT brand_id, sheet_id, encrypted_api_token, token_iv, token_auth_tag + FROM water_smartsheet_config + WHERE sync_enabled = true + `); + if (cfgRes.rows.length === 0) { + console.error("No active Smartsheet config — nothing to do"); + process.exit(0); + } + if (cfgRes.rows.length > 1) { + console.warn(`Found ${cfgRes.rows.length} active brands — processing all (current manual backfill was brand-specific)`); + } + + let totalMatched = 0; + let totalSheetsUpdated = 0; + let totalQueueUpdated = 0; + let totalSkipped = 0; + let totalUnmatched = 0; + + for (const cfg of cfgRes.rows) { + const brandId = cfg.brand_id; + const sheetId = cfg.sheet_id; + console.log(`\n=== Brand ${brandId} / sheet ${sheetId} ===`); + + let token; + try { + token = decryptToken(cfg.encrypted_api_token, cfg.token_iv, cfg.token_auth_tag); + } catch (err) { + console.error(` ✗ Failed to decrypt token: ${err.message}`); + continue; + } + + // 2. Fetch sheet metadata to get column IDs. + const meta = await smartsheetFetch(`/sheets/${encodeURIComponent(sheetId)}`, token); + const colId = Object.fromEntries(meta.columns.map((c) => [c.title.trim(), c.id])); + const entryIdCol = colId["Entry ID"]; + const loggedAtCol = colId["Logged At"]; + const headgateCol = colId["Headgate"]; + const irrigatorCol = colId["Irrigator"]; + const measurementCol = colId["Measurement"]; + const unitCol = colId["Unit"]; + if (!entryIdCol) { + console.error(` ✗ Sheet ${sheetId} has no "Entry ID" column`); + continue; + } + + // 3. Fetch all sheet rows. We need to find the 26 manual rows + // by their unique combination of values. If the sheet has + // thousands of rows this is wasteful, but the current sheet + // is small. + const sheetRowsRes = await smartsheetFetch( + `/sheets/${encodeURIComponent(sheetId)}?include=objectValue`, + token, + ); + const sheetRows = sheetRowsRes.rows || []; + console.log(` sheet has ${sheetRows.length} rows total`); + + // Build a lookup keyed by `${loggedAt}|${headgate}|${irrigator}|${measurement}|${unit}` + const byKey = new Map(); + for (const r of sheetRows) { + const cell = (colId) => { + const c = r.cells.find((cc) => cc.columnId === colId); + if (!c) return ""; + // Prefer the string `value`, fall back to displayValue / objectValue. + return ( + c.value != null + ? String(c.value) + : c.displayValue != null + ? String(c.displayValue) + : c.objectValue?.textValue || "" + ).trim(); + }; + const key = [ + cell(loggedAtCol), + cell(headgateCol), + cell(irrigatorCol), + cell(measurementCol), + cell(unitCol), + ].join("|"); + if (!byKey.has(key)) byKey.set(key, []); + byKey.get(key).push(r); + } + + // 4. For each manual row, look up the DB entry + sheet row. + for (const r of MANUAL_BACKFILL) { + const [ts, headgate, irrigator, measurement, unit, notes] = r; + const tsIso = toIsoTimestamp(ts); + + // Find DB entry by exact match. + const entryRes = await db.query( + ` + SELECT e.id, e.brand_id, e.logged_at, e.measurement::text AS measurement, e.unit, + h.name AS headgate_name, ir.name AS irrigator_name + FROM water_log_entries e + LEFT JOIN water_headgates h ON h.id = e.headgate_id + LEFT JOIN water_irrigators ir ON ir.id = e.irrigator_id + WHERE h.name = $1 AND ir.name = $2 + AND e.measurement = $3 AND e.unit = $4 + AND e.logged_at = $5 + AND e.brand_id = $6 + ORDER BY e.logged_at + LIMIT 1 + `, + [headgate, irrigator, String(measurement), unit, tsIso, brandId], + ); + + if (entryRes.rows.length === 0) { + console.log(` ? no DB match for ${headgate} / ${irrigator} / ${measurement} ${unit} @ ${tsIso}`); + totalUnmatched++; + continue; + } + const entry = entryRes.rows[0]; + totalMatched++; + + // Find matching sheet row(s). + const sheetKey = [tsIso, headgate, irrigator, String(measurement), unit].join("|"); + const candidates = byKey.get(sheetKey) || []; + // Filter to rows without an Entry ID (the manual rows). + const target = candidates.find((sr) => { + const ec = sr.cells.find((c) => c.columnId === entryIdCol); + return !ec || !ec.value; + }); + if (!target) { + console.log(` ? no sheet row for entry ${entry.id} (key ${sheetKey})`); + totalUnmatched++; + continue; + } + + // 5. Check queue state — if already synced with this row id, skip. + const queueRes = await db.query( + `SELECT id, status, smartsheet_row_id FROM water_smartsheet_sync_queue + WHERE brand_id = $1 AND entry_id = $2 + LIMIT 1`, + [brandId, entry.id], + ); + const queue = queueRes.rows[0]; + const targetRowId = String(target.id); + if ( + queue && + queue.status === "synced" && + queue.smartsheet_row_id === targetRowId + ) { + console.log(` · entry ${entry.id} already synced to sheet row ${targetRowId}; skipping`); + totalSkipped++; + continue; + } + + // 6. PUT Entry ID into sheet row. + try { + await smartsheetFetch( + `/sheets/${encodeURIComponent(sheetId)}/rows`, + token, + { + method: "PUT", + body: JSON.stringify([ + { id: Number(target.id), cells: [{ columnId: entryIdCol, value: entry.id }] }, + ]), + }, + ); + totalSheetsUpdated++; + } catch (err) { + console.error(` ✗ PUT failed for entry ${entry.id}: ${err.message}`); + continue; + } + + // 7. UPDATE queue row (insert if missing — defensive). + if (queue) { + await db.query( + `UPDATE water_smartsheet_sync_queue + SET status = 'synced', + smartsheet_row_id = $1, + synced_at = now(), + last_error = NULL, + next_attempt_at = now(), + attempts = attempts + 1, + updated_at = now() + WHERE id = $2`, + [targetRowId, queue.id], + ); + } else { + await db.query( + `INSERT INTO water_smartsheet_sync_queue + (brand_id, entry_id, smartsheet_row_id, status, attempts, synced_at, next_attempt_at, created_at, updated_at) + VALUES ($1, $2, $3, 'synced', 1, now(), now(), now(), now()) + ON CONFLICT (brand_id, entry_id) DO UPDATE + SET smartsheet_row_id = EXCLUDED.smartsheet_row_id, + status = 'synced', + last_error = NULL, + synced_at = now(), + next_attempt_at = now(), + updated_at = now()`, + [brandId, entry.id, targetRowId], + ); + } + + // 8. Write a sync log entry noting the manual backfill. + await db.query( + `INSERT INTO water_smartsheet_sync_log + (brand_id, entry_id, smartsheet_row_id, action, success, error, duration_ms, created_at) + VALUES ($1, $2, $3, 'sync', true, 'manual-backfill-script', 0, now())`, + [brandId, entry.id, targetRowId], + ); + + totalQueueUpdated++; + console.log(` ✓ entry ${entry.id} → sheet row ${targetRowId}`); + } + } + + console.log("\n=== Summary ==="); + console.log(`matched DB entries: ${totalMatched}`); + console.log(`sheet rows updated: ${totalSheetsUpdated}`); + console.log(`queue rows updated: ${totalQueueUpdated}`); + console.log(`skipped (already ok): ${totalSkipped}`); + console.log(`unmatched: ${totalUnmatched}`); + + if (totalUnmatched > 0) { + console.log("\n⚠ Some rows could not be matched. The cron will still try to re-sync those — check the unmatched entries and either delete them or fix the mapping."); + process.exit(0); // not fatal — partial success is still useful + } + } finally { + await db.end(); + } +} + +main().catch((err) => { + console.error("\nFATAL:", err); + process.exit(1); +}); \ No newline at end of file