fix(smartsheet): one-shot backfill script to re-add Entry IDs to manually inserted rows
Deploy to route.crispygoat.com / deploy (push) Failing after 3m59s
Deploy to route.crispygoat.com / deploy (push) Failing after 3m59s
26 water-log rows were POSTed directly to Smartsheet on 2026-07-03 because
the cron drain was failing. Those rows went in without Entry IDs, so the
next cron drain would have created duplicates (findRowByColumn returns
nothing when Entry ID is empty).
This script:
1. Matches each of the 26 hardcoded rows to a DB entry by
(headgate + irrigator + measurement + unit + logged_at).
2. Finds the corresponding Smartsheet row by the same criteria
(only the manually-inserted rows are missing Entry IDs).
3. PUTs the Entry ID back into the sheet row.
4. Marks the sync queue row as 'synced' with smartsheet_row_id set,
so the drain skips them.
5. Writes a sync log entry noting the manual backfill.
Idempotent: re-runs are no-ops. Wired into deploy.yml as a final step
so it runs on the next push; safe to leave in place across deploys.
Local invocation:
npm run smartsheet:backfill-entry-ids
This commit is contained in:
Nora
@@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* One-shot fix for Smartsheet backfill that bypassed the sync queue.
|
||||
*
|
||||
* Background:
|
||||
* On 2026-07-03 we manually POSTed 26 water-log rows directly to
|
||||
* Smartsheet (via curl + the REST API) because the cron drain was
|
||||
* timing out and the user needed the data visible immediately.
|
||||
* Those rows were inserted with empty Entry ID cells. Without
|
||||
* the Entry IDs, the next cron drain will hit `findRowByColumn`
|
||||
* for each of the 26 DB entries → find nothing → call `addRow` →
|
||||
* create 26 DUPLICATE rows.
|
||||
*
|
||||
* What this script does:
|
||||
* 1. Match each row in the hardcoded `MANUAL_BACKFILL` list to a
|
||||
* `water_log_entries` row by (headgate + irrigator + measurement
|
||||
* + unit + logged_at).
|
||||
* 2. Find the corresponding Smartsheet row (matching by the same
|
||||
* criteria; the 26 rows we manually inserted are the only ones
|
||||
* in the sheet without an Entry ID at these timestamps).
|
||||
* 3. PUT the Entry ID back into the sheet row (column "Entry ID").
|
||||
* 4. UPDATE the matching `water_smartsheet_sync_queue` row to
|
||||
* `status='synced'`, `smartsheet_row_id=<sheet row id>`, and
|
||||
* clear `last_error` + reset `next_attempt_at` so the drain
|
||||
* skips them.
|
||||
*
|
||||
* Idempotency:
|
||||
* - Skips queue rows already marked `synced` with a row id.
|
||||
* - Skips sheet rows that already have the correct Entry ID.
|
||||
* - Safe to re-run.
|
||||
*
|
||||
* Run via:
|
||||
* node scripts/smartsheet-backfill-entry-ids.mjs
|
||||
*
|
||||
* Auto-loaded env file precedence: .env (deploy writes this on the
|
||||
* server) → .env.local → process.env.
|
||||
*/
|
||||
require("dotenv").config({ path: ".env" });
|
||||
require("dotenv").config({ path: ".env.local", override: false });
|
||||
|
||||
const { Client } = require("pg");
|
||||
const { createDecipheriv } = require("node:crypto");
|
||||
|
||||
// ── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
// Sheet IDs we created manual rows in. Today there is only one
|
||||
// (the user's "Water Logs" sheet). If more brands are added in
|
||||
// the future, list them here too.
|
||||
const TARGET_SHEETS = {
|
||||
// brand_id -> { sheetId, sheetName }
|
||||
// Resolved at runtime from water_smartsheet_config.
|
||||
};
|
||||
|
||||
const ALGO = "aes-256-gcm";
|
||||
const IV_LEN = 12;
|
||||
const TAG_LEN = 16;
|
||||
const KEY_LEN = 32;
|
||||
|
||||
// 26 rows inserted manually on 2026-07-03 in the order they appear
|
||||
// in the original CSV. Columns: timestamp, headgate, irrigator,
|
||||
// measurement, unit, notes.
|
||||
const MANUAL_BACKFILL = [
|
||||
["2026-07-03 23:13:04.341+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""],
|
||||
["2026-07-03 20:42:37.963+00","Silver (techo rojo)","Benja",4,"CFS",""],
|
||||
["2026-07-03 20:38:34.013+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-03 17:06:41.857+00","Archuleta","Geovanny",0,"holes",""],
|
||||
["2026-07-03 15:40:56.668+00","Tessman","Geovanny",15,"holes",""],
|
||||
["2026-07-03 14:19:11.297+00","Gravera","Geovanny",0,"CFS",""],
|
||||
["2026-07-03 14:08:46.92+00","Kyle","Manuel",2.5,"CFS","Es para otra persona también "],
|
||||
["2026-07-03 14:06:44.752+00","Brack","Manuel",2,"CFS",""],
|
||||
["2026-07-03 13:48:41.23+00","Pfifer","Benja",13,"holes",""],
|
||||
["2026-07-03 13:31:07.278+00","Oseik en medio 8","Benja",2,"CFS",""],
|
||||
["2026-07-03 12:11:50.03+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-03 02:18:38.625+00","Henneghan (Vecino Organico)","Marcos",4,"CFS",""],
|
||||
["2026-07-03 02:16:45.993+00","Brack","Marcos",2,"CFS",""],
|
||||
["2026-07-02 23:13:22.107+00","Pierson (Casa de Celso)","Marcos",3.5,"CFS",""],
|
||||
["2026-07-02 20:03:25.046+00","Archuleta","Geovanny",12,"holes",""],
|
||||
["2026-07-02 19:18:50.038+00","Catlin","Marcos",4,"CFS",""],
|
||||
["2026-07-02 17:39:13.019+00","Halls","Geovanny",0,"CFS",""],
|
||||
["2026-07-02 17:38:17.285+00","Gravera","Geovanny",6,"CFS",""],
|
||||
["2026-07-02 17:37:29.588+00","Boyd (Pea Green)","Geovanny",5,"CFS",""],
|
||||
["2026-07-02 17:30:06.965+00","Boyd (Pea Green)","Geovanny",5,"holes",""],
|
||||
["2026-07-02 16:46:04.012+00","Halls","Geovanny",0,"CFS",""],
|
||||
["2026-07-02 16:44:05.506+00","Tessman","Geovanny",15,"CFS","15 ollos"],
|
||||
["2026-07-02 15:38:50.09+00","Oseik en medio 8","Benja",0,"CFS",""],
|
||||
["2026-07-02 15:38:15.259+00","Oseik en medio 8","Benja",2,"CFS",""],
|
||||
["2026-07-02 02:19:34.637+00","Test","Marcos",5,"CFS",""],
|
||||
["2026-07-02 02:11:41.35+00","Archuleta","Marcos",1,"CFS",""],
|
||||
];
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
function toIsoTimestamp(s) {
|
||||
// "2026-07-03 23:13:04.341+00" -> "2026-07-03T23:13:04.341"
|
||||
// (Smartsheet's DATE column accepts ISO without timezone suffix.)
|
||||
return s.replace(
|
||||
/^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}(\.\d+)?)[+-]\d+$/,
|
||||
"$1T$2",
|
||||
);
|
||||
}
|
||||
|
||||
function decryptToken(cipher, iv, authTag) {
|
||||
const raw = process.env.SMARTSHEET_TOKEN_ENC_KEY;
|
||||
if (!raw) throw new Error("SMARTSHEET_TOKEN_ENC_KEY is not set");
|
||||
const key = Buffer.from(raw, "base64");
|
||||
if (key.length !== KEY_LEN) {
|
||||
throw new Error(`SMARTSHEET_TOKEN_ENC_KEY must decode to ${KEY_LEN} bytes (got ${key.length})`);
|
||||
}
|
||||
if (!Buffer.isBuffer(iv) || iv.length !== IV_LEN) throw new Error(`iv must be ${IV_LEN} bytes`);
|
||||
if (!Buffer.isBuffer(authTag) || authTag.length !== TAG_LEN) throw new Error(`authTag must be ${TAG_LEN} bytes`);
|
||||
const decipher = createDecipheriv(ALGO, key, iv);
|
||||
decipher.setAuthTag(authTag);
|
||||
return Buffer.concat([decipher.update(cipher), decipher.final()]).toString("utf8");
|
||||
}
|
||||
|
||||
async function smartsheetFetch(path, token, opts = {}) {
|
||||
const res = await fetch(`https://api.smartsheet.com/2.0${path}`, {
|
||||
...opts,
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
...(opts.headers || {}),
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Smartsheet ${res.status} ${path}: ${body.slice(0, 300)}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const url = process.env.DATABASE_URL;
|
||||
if (!url) {
|
||||
console.error("DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const db = new Client({ connectionString: url });
|
||||
await db.connect();
|
||||
console.log("✓ connected to DB");
|
||||
|
||||
try {
|
||||
// 1. Discover the brand(s) with smartsheet sync enabled.
|
||||
const cfgRes = await db.query(`
|
||||
SELECT brand_id, sheet_id, encrypted_api_token, token_iv, token_auth_tag
|
||||
FROM water_smartsheet_config
|
||||
WHERE sync_enabled = true
|
||||
`);
|
||||
if (cfgRes.rows.length === 0) {
|
||||
console.error("No active Smartsheet config — nothing to do");
|
||||
process.exit(0);
|
||||
}
|
||||
if (cfgRes.rows.length > 1) {
|
||||
console.warn(`Found ${cfgRes.rows.length} active brands — processing all (current manual backfill was brand-specific)`);
|
||||
}
|
||||
|
||||
let totalMatched = 0;
|
||||
let totalSheetsUpdated = 0;
|
||||
let totalQueueUpdated = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalUnmatched = 0;
|
||||
|
||||
for (const cfg of cfgRes.rows) {
|
||||
const brandId = cfg.brand_id;
|
||||
const sheetId = cfg.sheet_id;
|
||||
console.log(`\n=== Brand ${brandId} / sheet ${sheetId} ===`);
|
||||
|
||||
let token;
|
||||
try {
|
||||
token = decryptToken(cfg.encrypted_api_token, cfg.token_iv, cfg.token_auth_tag);
|
||||
} catch (err) {
|
||||
console.error(` ✗ Failed to decrypt token: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Fetch sheet metadata to get column IDs.
|
||||
const meta = await smartsheetFetch(`/sheets/${encodeURIComponent(sheetId)}`, token);
|
||||
const colId = Object.fromEntries(meta.columns.map((c) => [c.title.trim(), c.id]));
|
||||
const entryIdCol = colId["Entry ID"];
|
||||
const loggedAtCol = colId["Logged At"];
|
||||
const headgateCol = colId["Headgate"];
|
||||
const irrigatorCol = colId["Irrigator"];
|
||||
const measurementCol = colId["Measurement"];
|
||||
const unitCol = colId["Unit"];
|
||||
if (!entryIdCol) {
|
||||
console.error(` ✗ Sheet ${sheetId} has no "Entry ID" column`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Fetch all sheet rows. We need to find the 26 manual rows
|
||||
// by their unique combination of values. If the sheet has
|
||||
// thousands of rows this is wasteful, but the current sheet
|
||||
// is small.
|
||||
const sheetRowsRes = await smartsheetFetch(
|
||||
`/sheets/${encodeURIComponent(sheetId)}?include=objectValue`,
|
||||
token,
|
||||
);
|
||||
const sheetRows = sheetRowsRes.rows || [];
|
||||
console.log(` sheet has ${sheetRows.length} rows total`);
|
||||
|
||||
// Build a lookup keyed by `${loggedAt}|${headgate}|${irrigator}|${measurement}|${unit}`
|
||||
const byKey = new Map();
|
||||
for (const r of sheetRows) {
|
||||
const cell = (colId) => {
|
||||
const c = r.cells.find((cc) => cc.columnId === colId);
|
||||
if (!c) return "";
|
||||
// Prefer the string `value`, fall back to displayValue / objectValue.
|
||||
return (
|
||||
c.value != null
|
||||
? String(c.value)
|
||||
: c.displayValue != null
|
||||
? String(c.displayValue)
|
||||
: c.objectValue?.textValue || ""
|
||||
).trim();
|
||||
};
|
||||
const key = [
|
||||
cell(loggedAtCol),
|
||||
cell(headgateCol),
|
||||
cell(irrigatorCol),
|
||||
cell(measurementCol),
|
||||
cell(unitCol),
|
||||
].join("|");
|
||||
if (!byKey.has(key)) byKey.set(key, []);
|
||||
byKey.get(key).push(r);
|
||||
}
|
||||
|
||||
// 4. For each manual row, look up the DB entry + sheet row.
|
||||
for (const r of MANUAL_BACKFILL) {
|
||||
const [ts, headgate, irrigator, measurement, unit, notes] = r;
|
||||
const tsIso = toIsoTimestamp(ts);
|
||||
|
||||
// Find DB entry by exact match.
|
||||
const entryRes = await db.query(
|
||||
`
|
||||
SELECT e.id, e.brand_id, e.logged_at, e.measurement::text AS measurement, e.unit,
|
||||
h.name AS headgate_name, ir.name AS irrigator_name
|
||||
FROM water_log_entries e
|
||||
LEFT JOIN water_headgates h ON h.id = e.headgate_id
|
||||
LEFT JOIN water_irrigators ir ON ir.id = e.irrigator_id
|
||||
WHERE h.name = $1 AND ir.name = $2
|
||||
AND e.measurement = $3 AND e.unit = $4
|
||||
AND e.logged_at = $5
|
||||
AND e.brand_id = $6
|
||||
ORDER BY e.logged_at
|
||||
LIMIT 1
|
||||
`,
|
||||
[headgate, irrigator, String(measurement), unit, tsIso, brandId],
|
||||
);
|
||||
|
||||
if (entryRes.rows.length === 0) {
|
||||
console.log(` ? no DB match for ${headgate} / ${irrigator} / ${measurement} ${unit} @ ${tsIso}`);
|
||||
totalUnmatched++;
|
||||
continue;
|
||||
}
|
||||
const entry = entryRes.rows[0];
|
||||
totalMatched++;
|
||||
|
||||
// Find matching sheet row(s).
|
||||
const sheetKey = [tsIso, headgate, irrigator, String(measurement), unit].join("|");
|
||||
const candidates = byKey.get(sheetKey) || [];
|
||||
// Filter to rows without an Entry ID (the manual rows).
|
||||
const target = candidates.find((sr) => {
|
||||
const ec = sr.cells.find((c) => c.columnId === entryIdCol);
|
||||
return !ec || !ec.value;
|
||||
});
|
||||
if (!target) {
|
||||
console.log(` ? no sheet row for entry ${entry.id} (key ${sheetKey})`);
|
||||
totalUnmatched++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 5. Check queue state — if already synced with this row id, skip.
|
||||
const queueRes = await db.query(
|
||||
`SELECT id, status, smartsheet_row_id FROM water_smartsheet_sync_queue
|
||||
WHERE brand_id = $1 AND entry_id = $2
|
||||
LIMIT 1`,
|
||||
[brandId, entry.id],
|
||||
);
|
||||
const queue = queueRes.rows[0];
|
||||
const targetRowId = String(target.id);
|
||||
if (
|
||||
queue &&
|
||||
queue.status === "synced" &&
|
||||
queue.smartsheet_row_id === targetRowId
|
||||
) {
|
||||
console.log(` · entry ${entry.id} already synced to sheet row ${targetRowId}; skipping`);
|
||||
totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// 6. PUT Entry ID into sheet row.
|
||||
try {
|
||||
await smartsheetFetch(
|
||||
`/sheets/${encodeURIComponent(sheetId)}/rows`,
|
||||
token,
|
||||
{
|
||||
method: "PUT",
|
||||
body: JSON.stringify([
|
||||
{ id: Number(target.id), cells: [{ columnId: entryIdCol, value: entry.id }] },
|
||||
]),
|
||||
},
|
||||
);
|
||||
totalSheetsUpdated++;
|
||||
} catch (err) {
|
||||
console.error(` ✗ PUT failed for entry ${entry.id}: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 7. UPDATE queue row (insert if missing — defensive).
|
||||
if (queue) {
|
||||
await db.query(
|
||||
`UPDATE water_smartsheet_sync_queue
|
||||
SET status = 'synced',
|
||||
smartsheet_row_id = $1,
|
||||
synced_at = now(),
|
||||
last_error = NULL,
|
||||
next_attempt_at = now(),
|
||||
attempts = attempts + 1,
|
||||
updated_at = now()
|
||||
WHERE id = $2`,
|
||||
[targetRowId, queue.id],
|
||||
);
|
||||
} else {
|
||||
await db.query(
|
||||
`INSERT INTO water_smartsheet_sync_queue
|
||||
(brand_id, entry_id, smartsheet_row_id, status, attempts, synced_at, next_attempt_at, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, 'synced', 1, now(), now(), now(), now())
|
||||
ON CONFLICT (brand_id, entry_id) DO UPDATE
|
||||
SET smartsheet_row_id = EXCLUDED.smartsheet_row_id,
|
||||
status = 'synced',
|
||||
last_error = NULL,
|
||||
synced_at = now(),
|
||||
next_attempt_at = now(),
|
||||
updated_at = now()`,
|
||||
[brandId, entry.id, targetRowId],
|
||||
);
|
||||
}
|
||||
|
||||
// 8. Write a sync log entry noting the manual backfill.
|
||||
await db.query(
|
||||
`INSERT INTO water_smartsheet_sync_log
|
||||
(brand_id, entry_id, smartsheet_row_id, action, success, error, duration_ms, created_at)
|
||||
VALUES ($1, $2, $3, 'sync', true, 'manual-backfill-script', 0, now())`,
|
||||
[brandId, entry.id, targetRowId],
|
||||
);
|
||||
|
||||
totalQueueUpdated++;
|
||||
console.log(` ✓ entry ${entry.id} → sheet row ${targetRowId}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n=== Summary ===");
|
||||
console.log(`matched DB entries: ${totalMatched}`);
|
||||
console.log(`sheet rows updated: ${totalSheetsUpdated}`);
|
||||
console.log(`queue rows updated: ${totalQueueUpdated}`);
|
||||
console.log(`skipped (already ok): ${totalSkipped}`);
|
||||
console.log(`unmatched: ${totalUnmatched}`);
|
||||
|
||||
if (totalUnmatched > 0) {
|
||||
console.log("\n⚠ Some rows could not be matched. The cron will still try to re-sync those — check the unmatched entries and either delete them or fix the mapping.");
|
||||
process.exit(0); // not fatal — partial success is still useful
|
||||
}
|
||||
} finally {
|
||||
await db.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\nFATAL:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user