diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..2496a2f --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,65 @@ +# Cyclone Operator Runbook + +Production operations for a single-operator Cyclone deploy on Ubuntu Linux. Assumes the box was bootstrapped via `scripts/cyclone-init.sh` and the stack is up via `docker compose up -d`. + +## Daily + +- [ ] Confirm the host healthcheck cron hasn't emailed. It pings `http://localhost:8080/api/healthz` every 5 minutes. +- [ ] `docker compose ps` — both services `healthy`. +- [ ] `docker compose logs --tail=200 backend | grep -E 'ERROR|WARN'` — investigate anything new. + +## Weekly + +- [ ] `curl -fsS http://localhost:8080/api/admin/audit-log -b cookies.txt | jq '.events[] | select(.event | test("login_failed|backup.failed"))'` — review failed logins + backup failures. +- [ ] Confirm `docker compose exec backend ls -la /var/lib/cyclone/backups/` shows recent `.bin` files (within 25h of now). + +## Quarterly + +- [ ] Rotate the SQLCipher / cookie-signing key: + ```bash + bash scripts/cyclone-init.sh --force # overwrites /etc/cyclone/secrets/db.key + docker compose restart backend # picks up the new key + ``` + Old `.bin` backups become unreadable after this; export them first if you need to keep them. + +## As needed + +- **Add an operator.** Log in as admin → `/admin/users` → Create user. Roles: `admin` / `user` / `viewer`. +- **Reset a password.** Admin UI → Users → Reset password, OR `docker compose exec backend python -m cyclone admin reset-password --username `. +- **Restore from backup.** Admin UI → Backups → pick the snapshot → Initiate restore → Confirm. The backend will restart automatically. +- **Roll back the code (not the schema).** `TAG=0.0.9 docker compose up -d`. The previous image stays in the local Docker cache for one cycle. +- **Pull a new `:stable`.** + ```bash + cd /opt/cyclone + docker compose pull + docker compose up -d + docker compose logs -f backend | head -200 # verify migrations + healthcheck + ``` +- **Off-box backup copy.** The operator is expected to rsync `/var/lib/docker/volumes/cyclone_backups/_data/` to an external drive or NAS nightly. The `.bin` files are already encrypted; the destination doesn't need its own encryption. +- **Inspect the DB.** `docker compose exec backend sqlite3 /var/lib/cyclone/db/cyclone.db ".tables"` (works only if SQLCipher key is on disk; the in-process decrypt happens via the cyclone backend). + +## Annual + +- [ ] Rotate the admin password (force re-login for everyone). +- [ ] Audit the `/etc/cyclone/secrets/` directory permissions — should be `chmod 600 root:root`. +- [ ] Review the audit log for stale admin sessions. + +## Emergency + +- **Backend won't start.** `docker compose logs --tail=300 backend`. Look for migration failures (rerun is safe — migrations are forward-only), SQLCipher key mismatch (`PRAGMA key` failure), or port collisions. +- **Frontend won't serve.** `docker compose logs --tail=100 frontend`. Usually nginx config drift; `docker compose restart frontend`. +- **Both unhealthy after a host reboot.** Docker may have come up before the named volumes did. `docker compose down && docker compose up -d`. +- **Suspected key compromise.** Rotate immediately (see Quarterly above). All active sessions are invalidated. + +## Where things live + +| Asset | Path | +|---|---| +| Docker compose file | `/opt/cyclone/docker-compose.yml` | +| Secrets | `/etc/cyclone/secrets/{db.key,admin_username,admin_pw}` | +| Live DB (SQLCipher-encrypted volume) | `cyclone_db` named volume, mounted at `/var/lib/cyclone/db` | +| Encrypted backups | `cyclone_backups` named volume, mounted at `/var/lib/cyclone/backups` | +| Uploaded prod files | `cyclone_prodfiles` named volume | +| SFTP staging stub | `cyclone_sftp_staging` named volume | +| Logs | `cyclone_logs` named volume + bind-mounted at `/var/log/cyclone` | +| Off-box backup destination | Operator's external drive / NAS (rsync cron, not in compose) | diff --git a/scripts/cyclone-init.sh b/scripts/cyclone-init.sh new file mode 100755 index 0000000..2d7d85a --- /dev/null +++ b/scripts/cyclone-init.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# cyclone-init.sh — first-time host bootstrap for a production Cyclone deploy. +# +# Generates /etc/cyclone/secrets/{db.key,admin_username,admin_pw} with +# openssl rand -hex 32, sets ownership + permissions, and prints the +# admin password ONCE for the operator to capture. +# +# Idempotent: refuses to overwrite existing secrets unless --force. +# +# Usage: +# bash scripts/cyclone-init.sh # generate if missing +# bash scripts/cyclone-init.sh --force # overwrite (destroys old data!) +set -euo pipefail + +SECRETS_DIR="${CYCLONE_SECRETS_DIR:-/etc/cyclone/secrets}" +FORCE=0 +[[ "${1:-}" == "--force" ]] && FORCE=1 + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: cyclone-init.sh must run as root (it writes to ${SECRETS_DIR})." >&2 + exit 1 +fi + +if [[ -f "${SECRETS_DIR}/db.key" && $FORCE -eq 0 ]]; then + echo "Secrets already exist at ${SECRETS_DIR}. Re-run with --force to overwrite." >&2 + exit 0 +fi + +mkdir -p "${SECRETS_DIR}" +chmod 700 "${SECRETS_DIR}" + +# 64-hex-char (256-bit) random key for SQLCipher + cookie signing. +openssl rand -hex 32 > "${SECRETS_DIR}/db.key" + +# Admin username defaults to "admin" unless CYCLONE_ADMIN_USERNAME is set. +ADMIN_USER="${CYCLONE_ADMIN_USERNAME:-admin}" +printf '%s' "${ADMIN_USER}" > "${SECRETS_DIR}/admin_username" + +# Memorable-but-random admin password — printed once. +ADMIN_PW="$(openssl rand -base64 18 | tr -d '/+=' | head -c 20)Aa1!" +printf '%s' "${ADMIN_PW}" > "${SECRETS_DIR}/admin_pw" + +chmod 600 "${SECRETS_DIR}"/* +chown -R root:root "${SECRETS_DIR}" + +cat <:8080/login and log in as '${ADMIN_USER}'. +Change the password from /admin/users immediately. +================================================================ +EOF diff --git a/scripts/post-deploy.sh b/scripts/post-deploy.sh new file mode 100755 index 0000000..ecb1587 --- /dev/null +++ b/scripts/post-deploy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# post-deploy.sh — set up host-side logrotate and healthcheck cron for a +# production Cyclone deploy. Run once after `docker compose up -d`. +set -euo pipefail + +LOG_DIR="/var/log/cyclone" +HEALTHCHECK_URL="${CYCLONE_HEALTHCHECK_URL:-http://localhost:8080/api/healthz}" +HEALTHCHECK_EMAIL="${CYCLONE_HEALTHCHECK_EMAIL:-root}" +CRON_USER="${CYCLONE_CRON_USER:-root}" + +if [[ $EUID -ne 0 ]]; then + echo "ERROR: post-deploy.sh must run as root." >&2 + exit 1 +fi + +# 1. Logrotate — keeps 14 days of JSON logs compressed. +cat > /etc/logrotate.d/cyclone <<'LOGROTATE' +/var/log/cyclone/*.log { + daily + rotate 14 + compress + delaycompress + missingok + notifempty + copytruncate + dateext + dateformat -%Y%m%d +} +LOGROTATE + +mkdir -p "${LOG_DIR}" +chmod 755 "${LOG_DIR}" + +# 2. Healthcheck cron — every 5 minutes, email if down. +CRON_LINE="*/5 * * * * curl -fsS --max-time 10 ${HEALTHCHECK_URL} >/dev/null 2>&1 || echo 'Cyclone DOWN at \$(date)' | mail -s 'Cyclone healthz FAIL' ${HEALTHCHECK_EMAIL}" +TMP_CRON="$(mktemp)" +crontab -u "${CRON_USER}" -l 2>/dev/null > "${TMP_CRON}" || true +# Remove any existing cyclone healthcheck line, then re-add. +grep -v -F "Cyclone healthz" "${TMP_CRON}" > "${TMP_CRON}.new" || cp "${TMP_CRON}" "${TMP_CRON}.new" +echo "${CRON_LINE}" >> "${TMP_CRON}.new" +crontab -u "${CRON_USER}" "${TMP_CRON}.new" +rm -f "${TMP_CRON}" "${TMP_CRON}.new" + +cat < waiting for healthz..." +for i in {1..30}; do + if curl -fsS "${BASE_URL}/api/healthz" >/dev/null 2>&1; then break; fi + sleep 2 + [[ $i -eq 30 ]] && { echo "FAIL: healthz never came up"; exit 1; } +done +echo " ok" + +echo "==> logging in as admin..." +HTTP_CODE=$(curl -sS -o /dev/null -w '%{http_code}' \ + -c "${COOKIE_JAR}" \ + -H 'Content-Type: application/json' \ + -X POST "${BASE_URL}/api/auth/login" \ + -d "{\"username\":\"admin\",\"password\":\"${ADMIN_PW}\"}") +[[ "${HTTP_CODE}" == "200" ]] || { echo "FAIL: login returned ${HTTP_CODE}"; exit 1; } +echo " ok (cookie set)" + +echo "==> /api/auth/me..." +ME=$(curl -fsS -b "${COOKIE_JAR}" "${BASE_URL}/api/auth/me") +echo " ${ME}" + +if [[ -f "${SAMPLE_837}" ]]; then + echo "==> parsing sample 837 (${SAMPLE_837})..." + PARSE=$(curl -fsS -b "${COOKIE_JAR}" -X POST "${BASE_URL}/api/parse-837" \ + -F "file=@${SAMPLE_837}") + echo " ${PARSE}" | head -c 200 + echo +else + echo "==> skipping 837 parse (sample not found at ${SAMPLE_837})" +fi + +echo "==> listing batches..." +BATCHES=$(curl -fsS -b "${COOKIE_JAR}" "${BASE_URL}/api/batches") +echo " ${BATCHES}" | head -c 200 +echo + +echo "smoke: OK"