feat(water-log): add manual backfill to push historical entries to Smartsheet
Deploy to route.crispygoat.com / deploy (push) Successful in 4m0s

Adds a backfill button to the Smartsheet Integration card so brand admins
can push existing water log entries (all time / last 30d / last 7d) to
their Smartsheet sheet — important for customers who configure
Smartsheet after months/years of recording entries.

- New enqueueSmartsheetBackfill(brandId, {since?}) server action:
  - Counts candidates, enqueues in 500-row chunks with
    onConflictDoNothing (UNIQUE on brand_id+entry_id absorbs dups).
  - Runs one inline drain pass (batchSize=200) for immediate feedback.
  - Cron picks up remaining queue rows across the next ticks.
  - Requires active Smartsheet config and admin permission; logs an
    audit event for the backfill.
- New "Backfill historical entries" card in SmartsheetIntegrationCard
  with range selector, confirmation prompt, and inline result feedback
  (counts of newly queued / already queued / drained synced / skipped /
  failed / remaining).
- Sync engine dedup by Entry ID + UNIQUE constraint means re-runs are
  safe: re-running never creates duplicate Smartsheet rows.
This commit is contained in:
Nora
2026-07-03 15:23:25 -06:00
parent 9910e588b1
commit 7d3de121c6
2 changed files with 423 additions and 3 deletions
+228 -2
View File
@@ -15,9 +15,10 @@
* - Read actions return `maskedToken` ("••••abcd") and never the
* plaintext.
*/
import { desc, eq } from "drizzle-orm";
import { desc, eq, and, gte, inArray } from "drizzle-orm";
import { withBrand } from "@/db/client";
import {
waterLogEntries,
waterSmartsheetConfig,
waterSmartsheetSyncLog,
waterSmartsheetSyncQueue,
@@ -29,7 +30,7 @@ import {
} from "@/db/schema/water-log";
import { decryptToken, encryptToken, maskToken } from "@/lib/crypto";
import { extractSheetId, getSheetMeta, SmartsheetApiError } from "@/lib/smartsheet";
import { syncEntryToSmartsheet } from "@/services/smartsheet-sync";
import { syncEntryToSmartsheet, drainSyncQueue } from "@/services/smartsheet-sync";
import { getAdminUser } from "@/lib/admin-permissions";
import { logAuditEvent } from "@/lib/water-log-audit";
import { getSession } from "@/lib/auth";
@@ -452,6 +453,231 @@ export async function triggerSyncForEntry(
// ── Helpers ────────────────────────────────────────────────────────────────
/**
* Backfill: enqueue every existing water log entry for the brand so it
* gets pushed to Smartsheet.
*
* Use cases:
* - Customer configured Smartsheet after months/years of recording
* entries → they need historical data in the sheet.
* - Customer rotated their Smartsheet sheet (different ID) → they
* want a clean re-push without changing the per-entry hook.
*
* Safety:
* - Insert is `ON CONFLICT DO NOTHING` so re-runs are no-ops on
* already-queued entries. The UNIQUE(brand_id, entry_id) on the
* queue absorbs duplicates.
* - The sync engine dedups on the Smartsheet side by Entry ID
* (`findRowByColumn`), so even if we re-enqueue an old entry whose
* queue row was deleted, the sheet still won't gain duplicates.
*
* Performance / scope:
* - Inserts are chunked at 500 rows per round-trip.
* - After enqueueing, we kick off one inline drain pass
* (batchSize 200) so the admin gets immediate feedback. Any
* remaining rows are drained by the cron over time.
*/
export type EnqueueBackfillOptions = {
/** Only enqueue entries logged at-or-after this date. */
since?: Date;
};
export type EnqueueBackfillResult = {
success: boolean;
error?: string;
/** Total entries in `water_log_entries` for the brand matching the filter. */
considered: number;
/** Newly inserted rows in `water_smartsheet_sync_queue`. */
newlyQueued: number;
/** Entries that were already in the queue (UNIQUE absorbed). */
alreadyQueued: number;
/**
* Result of the inline drain pass that was kicked off after enqueue.
* `remaining` is the count of pending/failed queue rows that the
* next cron tick will pick up.
*/
drained: {
synced: number;
skipped: number;
failed: number;
remaining: number;
};
};
const BACKFILL_INSERT_CHUNK = 500;
const BACKFILL_DRAIN_BATCH = 200;
export async function enqueueSmartsheetBackfill(
brandId: string,
opts: EnqueueBackfillOptions = {},
): Promise<EnqueueBackfillResult> {
await getSession();
const auth = await requireWaterAdminPermission();
if (!auth.ok) {
const empty = {
considered: 0,
newlyQueued: 0,
alreadyQueued: 0,
drained: { synced: 0, skipped: 0, failed: 0, remaining: 0 },
};
return { success: false, error: auth.error, ...empty };
}
return withBrand(brandId, async (db) => {
// 1. Require active config.
const configRows = await db
.select({
syncEnabled: waterSmartsheetConfig.syncEnabled,
})
.from(waterSmartsheetConfig)
.where(eq(waterSmartsheetConfig.brandId, brandId))
.limit(1);
const config = configRows[0];
if (!config) {
return {
success: false,
error: "Configure Smartsheet first (save your API token and column mapping).",
considered: 0,
newlyQueued: 0,
alreadyQueued: 0,
drained: { synced: 0, skipped: 0, failed: 0, remaining: 0 },
};
}
if (!config.syncEnabled) {
return {
success: false,
error: "Smartsheet sync is disabled — enable it in the toggle above before backfilling.",
considered: 0,
newlyQueued: 0,
alreadyQueued: 0,
drained: { synced: 0, skipped: 0, failed: 0, remaining: 0 },
};
}
// 2. Find candidate entry IDs.
const entryWhere = opts.since
? and(
eq(waterLogEntries.brandId, brandId),
gte(waterLogEntries.loggedAt, opts.since),
)
: eq(waterLogEntries.brandId, brandId);
const entryRows = await db
.select({ id: waterLogEntries.id })
.from(waterLogEntries)
.where(entryWhere);
const entryIds = entryRows.map((r) => r.id);
if (entryIds.length === 0) {
return {
success: true,
considered: 0,
newlyQueued: 0,
alreadyQueued: 0,
drained: { synced: 0, skipped: 0, failed: 0, remaining: 0 },
};
}
// 3. Find which ones are already queued.
const existingRows = await db
.select({ entryId: waterSmartsheetSyncQueue.entryId })
.from(waterSmartsheetSyncQueue)
.where(eq(waterSmartsheetSyncQueue.brandId, brandId));
const existingSet = new Set(existingRows.map((r) => r.entryId));
const toInsert = entryIds.filter((id) => !existingSet.has(id));
const alreadyQueued = entryIds.length - toInsert.length;
// 4. Insert in chunks (idempotent on the UNIQUE constraint).
let newlyQueued = 0;
for (let i = 0; i < toInsert.length; i += BACKFILL_INSERT_CHUNK) {
const chunk = toInsert.slice(i, i + BACKFILL_INSERT_CHUNK);
const values = chunk.map((entryId) => ({
brandId,
entryId,
status: "pending" as const,
}));
const inserted = await db
.insert(waterSmartsheetSyncQueue)
.values(values)
.onConflictDoNothing({
target: [
waterSmartsheetSyncQueue.brandId,
waterSmartsheetSyncQueue.entryId,
],
})
.returning({ entryId: waterSmartsheetSyncQueue.entryId });
newlyQueued += inserted.length;
}
// 5. Kick off a single drain pass for immediate feedback. Cron
// picks up the rest over the next ticks.
let drained: EnqueueBackfillResult["drained"] = {
synced: 0,
skipped: 0,
failed: 0,
remaining: 0,
};
try {
const result = await drainSyncQueue(brandId, {
batchSize: BACKFILL_DRAIN_BATCH,
});
const remainingRows = await db
.select({
id: waterSmartsheetSyncQueue.id,
})
.from(waterSmartsheetSyncQueue)
.where(
and(
eq(waterSmartsheetSyncQueue.brandId, brandId),
inArray(waterSmartsheetSyncQueue.status, ["pending", "failed"]),
),
);
drained = {
synced: result.synced,
skipped: result.skipped,
failed: result.failed,
remaining: remainingRows.length,
};
} catch (err) {
// Don't fail the whole backfill if the inline drain errors —
// the cron will retry. Just surface to the result.
drained = {
synced: 0,
skipped: 0,
failed: 0,
remaining: -1,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...({ error: (err as Error).message } as any),
};
}
await logAuditEvent({
brandId,
actorId: auth.adminUser.user_id ?? null,
actorLabel: auth.adminUser.email ?? auth.adminUser.display_name ?? "admin",
action: "create",
entityType: "settings",
details: {
feature: "smartsheet",
event: "backfill",
considered: entryIds.length,
newlyQueued,
alreadyQueued,
drainedSynced: drained.synced,
drainedSkipped: drained.skipped,
drainedFailed: drained.failed,
since: opts.since?.toISOString() ?? null,
},
});
return {
success: true,
considered: entryIds.length,
newlyQueued,
alreadyQueued,
drained,
};
});
}
function validateColumnMapping(
m: unknown,
): { ok: true; value: SmartsheetColumnMapping } | { ok: false; error: string } {
@@ -32,6 +32,8 @@ import {
saveSmartsheetConfig,
testSmartsheetConnection,
listSmartsheetSyncLog,
enqueueSmartsheetBackfill,
type EnqueueBackfillResult,
type SmartsheetConfigResponse,
type SmartsheetSyncLogEntry,
} from "@/actions/water-log/smartsheet";
@@ -97,6 +99,12 @@ type State = {
loadingLog: boolean;
/** Whether the setup-instructions modal is open. */
setupModalOpen: boolean;
/** Backfill UI state. */
backfill: {
running: boolean;
range: "all" | "30d" | "7d";
result: EnqueueBackfillResult | null;
};
};
type Action =
@@ -118,7 +126,10 @@ type Action =
| { type: "CLEAR_MESSAGE" }
| { type: "REFRESH_LOG"; log: SmartsheetSyncLogEntry[] }
| { type: "OPEN_SETUP_MODAL" }
| { type: "CLOSE_SETUP_MODAL" };
| { type: "CLOSE_SETUP_MODAL" }
| { type: "BACKFILL_START" }
| { type: "BACKFILL_RANGE"; value: "all" | "30d" | "7d" }
| { type: "BACKFILL_DONE"; result: EnqueueBackfillResult };
const EMPTY_MAPPING: SmartsheetColumnMapping = {
entry_id: "",
@@ -146,6 +157,7 @@ const initialState: State = {
recentLog: [],
loadingLog: false,
setupModalOpen: false,
backfill: { running: false, range: "all", result: null },
};
function reducer(state: State, action: Action): State {
@@ -227,6 +239,25 @@ function reducer(state: State, action: Action): State {
return { ...state, setupModalOpen: true };
case "CLOSE_SETUP_MODAL":
return { ...state, setupModalOpen: false };
case "BACKFILL_START":
return {
...state,
backfill: { ...state.backfill, running: true },
};
case "BACKFILL_RANGE":
return {
...state,
backfill: { ...state.backfill, range: action.value },
};
case "BACKFILL_DONE":
return {
...state,
backfill: {
...state.backfill,
running: false,
result: action.result,
},
};
default:
return state;
}
@@ -313,6 +344,50 @@ export default function SmartsheetIntegrationCard({
}
}
async function handleBackfill() {
if (state.backfill.running) return;
if (!state.savedConfig?.configured || !state.enabled) {
// Surface as a notice rather than call into the action just to fail.
dispatch({
type: "BACKFILL_DONE",
result: {
success: false,
error: "Save and enable your Smartsheet config first.",
considered: 0,
newlyQueued: 0,
alreadyQueued: 0,
drained: { synced: 0, skipped: 0, failed: 0, remaining: 0 },
},
});
return;
}
const rangeLabel =
state.backfill.range === "30d"
? "the last 30 days"
: state.backfill.range === "7d"
? "the last 7 days"
: "all time";
if (
!window.confirm(
`Sync ${rangeLabel} of water log entries to Smartsheet? This is safe to re-run — already-pushed entries are deduped by Entry ID.`,
)
) {
return;
}
dispatch({ type: "BACKFILL_START" });
const since =
state.backfill.range === "30d"
? new Date(Date.now() - 30 * 86_400_000)
: state.backfill.range === "7d"
? new Date(Date.now() - 7 * 86_400_000)
: undefined;
const result = await enqueueSmartsheetBackfill(brandId, { since });
dispatch({ type: "BACKFILL_DONE", result });
// Refresh the log so the drain results show up immediately.
const log = await listSmartsheetSyncLog(brandId, 10);
dispatch({ type: "REFRESH_LOG", log });
}
if (state.loading) {
return (
<Card title="Smartsheet Integration" subtitle="Loading…">
@@ -570,6 +645,8 @@ export default function SmartsheetIntegrationCard({
</div>
</Card>
<BackfillCard state={state.backfill} onRun={handleBackfill} onRangeChange={(v) => dispatch({ type: "BACKFILL_RANGE", value: v })} />
<Card
title="Recent activity"
subtitle="Last 10 sync attempts. Use this to spot token / sheet issues quickly."
@@ -730,6 +807,123 @@ function TestResultPanel({ test }: { test: TestState }) {
);
}
// ── Backfill card ──────────────────────────────────────────────────────────
type BackfillState = {
running: boolean;
range: "all" | "30d" | "7d";
result: EnqueueBackfillResult | null;
};
function BackfillCard({
state,
onRun,
onRangeChange,
}: {
state: BackfillState;
onRun: () => void;
onRangeChange: (value: "all" | "30d" | "7d") => void;
}) {
const result = state.result;
// Success block — counts from the inline drain that ran after enqueue.
const showResult = result && result.success;
const showError = result && !result.success;
return (
<Card
title="Backfill historical entries"
subtitle="Push existing water log entries to your Smartsheet. Safe to re-run — already-pushed entries are deduped by Entry ID."
>
<div className="space-y-3">
<div className="flex flex-wrap items-center gap-3">
<label
htmlFor="smartsheet-backfill-range"
className="text-sm font-medium text-[#1d1d1f]"
>
Range
</label>
<select
id="smartsheet-backfill-range"
value={state.range}
onChange={(e) =>
onRangeChange(e.target.value as "all" | "30d" | "7d")
}
className="rounded-lg border border-[#d4d9d3] bg-white px-3 py-2 text-sm outline-none focus:border-[#1a4d2e] focus:ring-1 focus:ring-[#1a4d2e]"
disabled={state.running}
>
<option value="all">All time</option>
<option value="30d">Last 30 days</option>
<option value="7d">Last 7 days</option>
</select>
<button
type="button"
onClick={onRun}
disabled={state.running}
className="rounded-lg border border-[#1a4d2e] bg-white px-4 py-2 text-sm font-semibold text-[#1a4d2e] transition hover:bg-[#1a4d2e] hover:text-white disabled:opacity-50"
>
{state.running ? "Running backfill…" : "Run backfill"}
</button>
</div>
{state.running && (
<p className="text-xs text-[#5a5d5a]">
Counting entries, enqueueing, and draining one batch inline. Any
remaining entries will be drained by the cron (every 15 minutes).
</p>
)}
{showError && (
<div className="rounded-lg border border-[#a4452b] bg-[#a4452b]/5 px-3 py-2 text-xs">
<p className="font-semibold text-[#a4452b]"> {result.error}</p>
</div>
)}
{showResult && result && (
<div className="rounded-lg border border-[#1a4d2e] bg-[#1a4d2e]/5 px-3 py-2 text-xs">
<p className="font-semibold text-[#1a4d2e]">
Found {result.considered}{" "}
{result.considered === 1 ? "entry" : "entries"} matching the
range
{result.considered > 0 && (
<>
{" "}
({result.newlyQueued} new, {result.alreadyQueued} already
queued).
</>
)}
.
</p>
{(result.drained.synced +
result.drained.skipped +
result.drained.failed >
0 || result.drained.remaining > 0) && (
<p className="mt-1 text-[#1d1d1f]">
Inline drain: {result.drained.synced} synced /{" "}
{result.drained.skipped} skipped / {result.drained.failed}{" "}
failed.
{result.drained.remaining > 0 && (
<>
{" "}
<strong className="text-[#1a4d2e]">
{result.drained.remaining} still queued
</strong>{" "}
they&apos;ll drain on the next cron tick (within 15
minutes).
</>
)}
{result.drained.remaining === 0 &&
result.considered > 0 && (
<> All entries are now in Smartsheet.</>
)}
</p>
)}
</div>
)}
</div>
</Card>
);
}
// ── Setup instructions modal ───────────────────────────────────────────────
const FIELD_SCHEMA: Array<{