revert: drop smartsheet backfill script from deploy
Deploy to route.crispygoat.com / deploy (push) Successful in 4m12s
Deploy to route.crispygoat.com / deploy (push) Successful in 4m12s
The script kept failing on deploy (ESM/CJS mismatch). User opted to drop the auto-fix. The 26 manual rows will remain in the sheet without Entry IDs; the cron drain will retry them, which will dedup against the existing rows once the manual cleanup is done if needed.
This commit is contained in:
Nora
@@ -210,26 +210,3 @@ jobs:
|
||||
ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "cd $APP_DIR && npm install --omit=dev 2>&1 | tail -5 && pm2 restart route-commerce --update-env || pm2 start npm --name route-commerce -- start -- -p 3100 && pm2 save && sleep 4 && curl -f -s http://localhost:3100/api/health/db-schema || { echo 'Health check failed after start - schema not applied (see plan)'; exit 1; }"
|
||||
|
||||
echo "Deployed successfully"
|
||||
|
||||
# One-shot: fix the 2026-07-03 manual Smartsheet backfill that
|
||||
# bypassed the sync queue. This step is idempotent — it scans
|
||||
# for queue rows that are pending/failed with no smartsheet_row_id
|
||||
# and matches them to the 26 rows Tyler manually POSTed into the
|
||||
# sheet. It then PUTs the Entry ID back into each sheet row and
|
||||
# marks the queue rows as `synced` so the cron drain won't try to
|
||||
# re-create them. Safe to leave in place across deploys (re-runs
|
||||
# are no-ops once everything is synced). Remove this step once
|
||||
# the data has been verified in Smartsheet.
|
||||
- name: Smartsheet backfill entry-id fix
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
||||
SMARTSHEET_TOKEN_ENC_KEY: ${{ secrets.SMARTSHEET_TOKEN_ENC_KEY }}
|
||||
run: |
|
||||
set -e
|
||||
APP_DIR=/home/tyler/route-commerce
|
||||
ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no tyler@route.crispygoat.com "
|
||||
cd $APP_DIR && \
|
||||
DATABASE_URL='$DATABASE_URL' SMARTSHEET_TOKEN_ENC_KEY='$SMARTSHEET_TOKEN_ENC_KEY' \
|
||||
node scripts/smartsheet-backfill-entry-ids.mjs
|
||||
"
|
||||
@@ -11,7 +11,6 @@
|
||||
"migrate": "node scripts/migrate.js",
|
||||
"migrate:one": "node scripts/migrate.js",
|
||||
"db:migrate": "node scripts/migrate.js",
|
||||
"smartsheet:backfill-entry-ids": "node scripts/smartsheet-backfill-entry-ids.mjs",
|
||||
"db:seed": "tsx db/seed.ts",
|
||||
"db:seed:tour": "node scripts/seed-tuxedo-2026.js",
|
||||
"db:reset": "node scripts/db-reset.js",
|
||||
|
||||
@@ -1,374 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* One-shot fix for Smartsheet backfill that bypassed the sync queue.
|
||||
*
|
||||
* Background:
|
||||
* On 2026-07-03 we manually POSTed 26 water-log rows directly to
|
||||
* Smartsheet (via curl + the REST API) because the cron drain was
|
||||
* timing out and the user needed the data visible immediately.
|
||||
* Those rows were inserted with empty Entry ID cells. Without
|
||||
* the Entry IDs, the next cron drain will hit `findRowByColumn`
|
||||
* for each of the 26 DB entries → find nothing → call `addRow` →
|
||||
* create 26 DUPLICATE rows.
|
||||
*
|
||||
* What this script does:
|
||||
* 1. Match each row in the hardcoded `MANUAL_BACKFILL` list to a
|
||||
* `water_log_entries` row by (headgate + irrigator + measurement
|
||||
* + unit + logged_at).
|
||||
* 2. Find the corresponding Smartsheet row (matching by the same
|
||||
* criteria; the 26 rows we manually inserted are the only ones
|
||||
* in the sheet without an Entry ID at these timestamps).
|
||||
* 3. PUT the Entry ID back into the sheet row (column "Entry ID").
|
||||
* 4. UPDATE the matching `water_smartsheet_sync_queue` row to
|
||||
* `status='synced'`, `smartsheet_row_id=<sheet row id>`, and
|
||||
* clear `last_error` + reset `next_attempt_at` so the drain
|
||||
* skips them.
|
||||
*
|
||||
* Idempotency:
|
||||
* - Skips queue rows already marked `synced` with a row id.
|
||||
* - Skips sheet rows that already have the correct Entry ID.
|
||||
* - Safe to re-run.
|
||||
*
|
||||
* Run via:
|
||||
* node scripts/smartsheet-backfill-entry-ids.mjs
|
||||
*
|
||||
* Auto-loaded env file precedence: .env (deploy writes this on the
|
||||
* server) → .env.local → process.env.
|
||||
*/
|
||||
require("dotenv").config({ path: ".env" });
|
||||
require("dotenv").config({ path: ".env.local", override: false });
|
||||
|
||||
const { Client } = require("pg");
|
||||
const { createDecipheriv } = require("node:crypto");
|
||||
|
||||
// ── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
// Sheet IDs we created manual rows in. Today there is only one
|
||||
// (the user's "Water Logs" sheet). If more brands are added in
|
||||
// the future, list them here too.
|
||||
const TARGET_SHEETS = {
|
||||
// brand_id -> { sheetId, sheetName }
|
||||
// Resolved at runtime from water_smartsheet_config.
|
||||
};
|
||||
|
||||
const ALGO = "aes-256-gcm";
|
||||
const IV_LEN = 12;
|
||||
const TAG_LEN = 16;
|
||||
const KEY_LEN = 32;
|
||||
|
||||
// 26 rows inserted manually on 2026-07-03 in the order they appear
|
||||
// in the original CSV. Columns: timestamp, headgate, irrigator,
|
||||
// measurement, unit, notes.
|
||||
const MANUAL_BACKFILL = [
|
||||
["2026-07-03 23:13:04.341+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""],
|
||||
["2026-07-03 20:42:37.963+00","Silver (techo rojo)","Benja",4,"CFS",""],
|
||||
["2026-07-03 20:38:34.013+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-03 17:06:41.857+00","Archuleta","Geovanny",0,"holes",""],
|
||||
["2026-07-03 15:40:56.668+00","Tessman","Geovanny",15,"holes",""],
|
||||
["2026-07-03 14:19:11.297+00","Gravera","Geovanny",0,"CFS",""],
|
||||
["2026-07-03 14:08:46.92+00","Kyle","Manuel",2.5,"CFS","Es para otra persona también "],
|
||||
["2026-07-03 14:06:44.752+00","Brack","Manuel",2,"CFS",""],
|
||||
["2026-07-03 13:48:41.23+00","Pfifer","Benja",13,"holes",""],
|
||||
["2026-07-03 13:31:07.278+00","Oseik en medio 8","Benja",2,"CFS",""],
|
||||
["2026-07-03 12:11:50.03+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-03 02:18:38.625+00","Henneghan (Vecino Organico)","Marcos",4,"CFS",""],
|
||||
["2026-07-03 02:16:45.993+00","Brack","Marcos",2,"CFS",""],
|
||||
["2026-07-02 23:13:22.107+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""],
|
||||
["2026-07-02 20:03:25.046+00","Archuleta","Geovanny",12,"holes",""],
|
||||
["2026-07-02 19:18:50.038+00","Catlin","Marcos",4,"CFS",""],
|
||||
["2026-07-02 17:39:13.019+00","Halls","Geovanny",0,"CFS",""],
|
||||
["2026-07-02 17:38:17.285+00","Gravera","Geovanny",6,"CFS",""],
|
||||
["2026-07-02 17:37:29.588+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-02 17:30:06.965+00","Boyd (Pea Green)","Geovanny",5,"holes",""],
|
||||
["2026-07-02 16:46:04.012+00","Halls","Geovanny",0,"CFS",""],
|
||||
["2026-07-02 16:44:05.506+00","Tessman","Geovanny",15,"CFS","15 ollos"],
|
||||
["2026-07-02 15:38:50.09+00","Oseik en medio 8","Benja",0,"CFS",""],
|
||||
["2026-07-02 15:38:15.259+00","Oseik en medio 8","Benja",2,"CFS",""],
|
||||
["2026-07-02 02:19:34.637+00","Test","Marcos",5,"CFS",""],
|
||||
["2026-07-02 02:11:41.35+00","Archuleta","Marcos",1,"CFS",""],
|
||||
];
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
function toIsoTimestamp(s) {
|
||||
// "2026-07-03 23:13:04.341+00" -> "2026-07-03T23:13:04.341"
|
||||
// (Smartsheet's DATE column accepts ISO without timezone suffix.)
|
||||
return s.replace(
|
||||
/^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(\.\d+)?)[+-]\d+$/,
|
||||
"$1T$2",
|
||||
);
|
||||
}
|
||||
|
||||
function decryptToken(cipher, iv, authTag) {
|
||||
const raw = process.env.SMARTSHEET_TOKEN_ENC_KEY;
|
||||
if (!raw) throw new Error("SMARTSHEET_TOKEN_ENC_KEY is not set");
|
||||
const key = Buffer.from(raw, "base64");
|
||||
if (key.length !== KEY_LEN) {
|
||||
throw new Error(`SMARTSHEET_TOKEN_ENC_KEY must decode to ${KEY_LEN} bytes (got ${key.length})`);
|
||||
}
|
||||
if (!Buffer.isBuffer(iv) || iv.length !== IV_LEN) throw new Error(`iv must be ${IV_LEN} bytes`);
|
||||
if (!Buffer.isBuffer(authTag) || authTag.length !== TAG_LEN) throw new Error(`authTag must be ${TAG_LEN} bytes`);
|
||||
const decipher = createDecipheriv(ALGO, key, iv);
|
||||
decipher.setAuthTag(authTag);
|
||||
return Buffer.concat([decipher.update(cipher), decipher.final()]).toString("utf8");
|
||||
}
|
||||
|
||||
async function smartsheetFetch(path, token, opts = {}) {
|
||||
const res = await fetch(`https://api.smartsheet.com/2.0${path}`, {
|
||||
...opts,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
...(opts.headers || {}),
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Smartsheet ${res.status} ${path}: ${body.slice(0, 300)}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const url = process.env.DATABASE_URL;
|
||||
if (!url) {
|
||||
console.error("DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = new Client({ connectionString: url });
|
||||
await db.connect();
|
||||
console.log("✓ connected to DB");
|
||||
|
||||
try {
|
||||
// 1. Discover the brand(s) with smartsheet sync enabled.
|
||||
const cfgRes = await db.query(`
|
||||
SELECT brand_id, sheet_id, encrypted_api_token, token_iv, token_auth_tag
|
||||
FROM water_smartsheet_config
|
||||
WHERE sync_enabled = true
|
||||
`);
|
||||
if (cfgRes.rows.length === 0) {
|
||||
console.error("No active Smartsheet config — nothing to do");
|
||||
process.exit(0);
|
||||
}
|
||||
if (cfgRes.rows.length > 1) {
|
||||
console.warn(`Found ${cfgRes.rows.length} active brands — processing all (current manual backfill was brand-specific)`);
|
||||
}
|
||||
|
||||
let totalMatched = 0;
|
||||
let totalSheetsUpdated = 0;
|
||||
let totalQueueUpdated = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalUnmatched = 0;
|
||||
|
||||
for (const cfg of cfgRes.rows) {
|
||||
const brandId = cfg.brand_id;
|
||||
const sheetId = cfg.sheet_id;
|
||||
console.log(`\n=== Brand ${brandId} / sheet ${sheetId} ===`);
|
||||
|
||||
let token;
|
||||
try {
|
||||
token = decryptToken(cfg.encrypted_api_token, cfg.token_iv, cfg.token_auth_tag);
|
||||
} catch (err) {
|
||||
console.error(` ✗ Failed to decrypt token: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Fetch sheet metadata to get column IDs.
|
||||
const meta = await smartsheetFetch(`/sheets/${encodeURIComponent(sheetId)}`, token);
|
||||
const colId = Object.fromEntries(meta.columns.map((c) => [c.title.trim(), c.id]));
|
||||
const entryIdCol = colId["Entry ID"];
|
||||
const loggedAtCol = colId["Logged At"];
|
||||
const headgateCol = colId["Headgate"];
|
||||
const irrigatorCol = colId["Irrigator"];
|
||||
const measurementCol = colId["Measurement"];
|
||||
const unitCol = colId["Unit"];
|
||||
if (!entryIdCol) {
|
||||
console.error(` ✗ Sheet ${sheetId} has no "Entry ID" column`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Fetch all sheet rows. We need to find the 26 manual rows
|
||||
// by their unique combination of values. If the sheet has
|
||||
// thousands of rows this is wasteful, but the current sheet
|
||||
// is small.
|
||||
const sheetRowsRes = await smartsheetFetch(
|
||||
`/sheets/${encodeURIComponent(sheetId)}?include=objectValue`,
|
||||
token,
|
||||
);
|
||||
const sheetRows = sheetRowsRes.rows || [];
|
||||
console.log(` sheet has ${sheetRows.length} rows total`);
|
||||
|
||||
// Build a lookup keyed by `${loggedAt}|${headgate}|${irrigator}|${measurement}|${unit}`
|
||||
const byKey = new Map();
|
||||
for (const r of sheetRows) {
|
||||
const cell = (colId) => {
|
||||
const c = r.cells.find((cc) => cc.columnId === colId);
|
||||
if (!c) return "";
|
||||
// Prefer the string `value`, fall back to displayValue / objectValue.
|
||||
return (
|
||||
c.value != null
|
||||
? String(c.value)
|
||||
: c.displayValue != null
|
||||
? String(c.displayValue)
|
||||
: c.objectValue?.textValue || ""
|
||||
).trim();
|
||||
};
|
||||
const key = [
|
||||
cell(loggedAtCol),
|
||||
cell(headgateCol),
|
||||
cell(irrigatorCol),
|
||||
cell(measurementCol),
|
||||
cell(unitCol),
|
||||
].join("|");
|
||||
if (!byKey.has(key)) byKey.set(key, []);
|
||||
byKey.get(key).push(r);
|
||||
}
|
||||
|
||||
// 4. For each manual row, look up the DB entry + sheet row.
|
||||
for (const r of MANUAL_BACKFILL) {
|
||||
const [ts, headgate, irrigator, measurement, unit, notes] = r;
|
||||
const tsIso = toIsoTimestamp(ts);
|
||||
|
||||
// Find DB entry by exact match.
|
||||
const entryRes = await db.query(
|
||||
`
|
||||
SELECT e.id, e.brand_id, e.logged_at, e.measurement::text AS measurement, e.unit,
|
||||
h.name AS headgate_name, ir.name AS irrigator_name
|
||||
FROM water_log_entries e
|
||||
LEFT JOIN water_headgates h ON h.id = e.headgate_id
|
||||
LEFT JOIN water_irrigators ir ON ir.id = e.irrigator_id
|
||||
WHERE h.name = $1 AND ir.name = $2
|
||||
AND e.measurement = $3 AND e.unit = $4
|
||||
AND e.logged_at = $5
|
||||
AND e.brand_id = $6
|
||||
ORDER BY e.logged_at
|
||||
LIMIT 1
|
||||
`,
|
||||
[headgate, irrigator, String(measurement), unit, tsIso, brandId],
|
||||
);
|
||||
|
||||
if (entryRes.rows.length === 0) {
|
||||
console.log(` ? no DB match for ${headgate} / ${irrigator} / ${measurement} ${unit} @ ${tsIso}`);
|
||||
totalUnmatched++;
|
||||
continue;
|
||||
}
|
||||
const entry = entryRes.rows[0];
|
||||
totalMatched++;
|
||||
|
||||
// Find matching sheet row(s).
|
||||
const sheetKey = [tsIso, headgate, irrigator, String(measurement), unit].join("|");
|
||||
const candidates = byKey.get(sheetKey) || [];
|
||||
// Filter to rows without an Entry ID (the manual rows).
|
||||
const target = candidates.find((sr) => {
|
||||
const ec = sr.cells.find((c) => c.columnId === entryIdCol);
|
||||
return !ec || !ec.value;
|
||||
});
|
||||
if (!target) {
|
||||
console.log(` ? no sheet row for entry ${entry.id} (key ${sheetKey})`);
|
||||
totalUnmatched++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 5. Check queue state — if already synced with this row id, skip.
|
||||
const queueRes = await db.query(
|
||||
`SELECT id, status, smartsheet_row_id FROM water_smartsheet_sync_queue
|
||||
WHERE brand_id = $1 AND entry_id = $2
|
||||
LIMIT 1`,
|
||||
[brandId, entry.id],
|
||||
);
|
||||
const queue = queueRes.rows[0];
|
||||
const targetRowId = String(target.id);
|
||||
if (
|
||||
queue &&
|
||||
queue.status === "synced" &&
|
||||
queue.smartsheet_row_id === targetRowId
|
||||
) {
|
||||
console.log(` · entry ${entry.id} already synced to sheet row ${targetRowId}; skipping`);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 6. PUT Entry ID into sheet row.
|
||||
try {
|
||||
await smartsheetFetch(
|
||||
`/sheets/${encodeURIComponent(sheetId)}/rows`,
|
||||
token,
|
||||
{
|
||||
method: "PUT",
|
||||
body: JSON.stringify([
|
||||
{ id: Number(target.id), cells: [{ columnId: entryIdCol, value: entry.id }] },
|
||||
]),
|
||||
},
|
||||
);
|
||||
totalSheetsUpdated++;
|
||||
} catch (err) {
|
||||
console.error(` ✗ PUT failed for entry ${entry.id}: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 7. UPDATE queue row (insert if missing — defensive).
|
||||
if (queue) {
|
||||
await db.query(
|
||||
`UPDATE water_smartsheet_sync_queue
|
||||
SET status = 'synced',
|
||||
smartsheet_row_id = $1,
|
||||
synced_at = now(),
|
||||
last_error = NULL,
|
||||
next_attempt_at = now(),
|
||||
attempts = attempts + 1,
|
||||
updated_at = now()
|
||||
WHERE id = $2`,
|
||||
[targetRowId, queue.id],
|
||||
);
|
||||
} else {
|
||||
await db.query(
|
||||
`INSERT INTO water_smartsheet_sync_queue
|
||||
(brand_id, entry_id, smartsheet_row_id, status, attempts, synced_at, next_attempt_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, 'synced', 1, now(), now(), now(), now())
|
||||
ON CONFLICT (brand_id, entry_id) DO UPDATE
|
||||
SET smartsheet_row_id = EXCLUDED.smartsheet_row_id,
|
||||
status = 'synced',
|
||||
last_error = NULL,
|
||||
synced_at = now(),
|
||||
next_attempt_at = now(),
|
||||
updated_at = now()`,
|
||||
[brandId, entry.id, targetRowId],
|
||||
);
|
||||
}
|
||||
|
||||
// 8. Write a sync log entry noting the manual backfill.
|
||||
await db.query(
|
||||
`INSERT INTO water_smartsheet_sync_log
|
||||
(brand_id, entry_id, smartsheet_row_id, action, success, error, duration_ms, created_at)
|
||||
VALUES ($1, $2, $3, 'sync', true, 'manual-backfill-script', 0, now())`,
|
||||
[brandId, entry.id, targetRowId],
|
||||
);
|
||||
|
||||
totalQueueUpdated++;
|
||||
console.log(` ✓ entry ${entry.id} → sheet row ${targetRowId}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n=== Summary ===");
|
||||
console.log(`matched DB entries: ${totalMatched}`);
|
||||
console.log(`sheet rows updated: ${totalSheetsUpdated}`);
|
||||
console.log(`queue rows updated: ${totalQueueUpdated}`);
|
||||
console.log(`skipped (already ok): ${totalSkipped}`);
|
||||
console.log(`unmatched: ${totalUnmatched}`);
|
||||
|
||||
if (totalUnmatched > 0) {
|
||||
console.log("\n⚠ Some rows could not be matched. The cron will still try to re-sync those — check the unmatched entries and either delete them or fix the mapping.");
|
||||
process.exit(0); // not fatal — partial success is still useful
|
||||
}
|
||||
} finally {
|
||||
await db.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\nFATAL:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user