Nora
20c03acdaa
Deploy to route.crispygoat.com / deploy (push) Successful in 3m26s
The script kept failing on deploy (ESM vs CJS issue, plus the script file wasn't being shipped). User decided not to worry about it. The 26 manual rows will remain in the sheet without Entry IDs; the cron will retry them with a manual cleanup of any duplicates if needed.
365 lines
14 KiB
Python
365 lines
14 KiB
Python
"""
|
|
Admin audit runner. One pass = one fresh browser context, no persisted state.
|
|
Visits every admin route at desktop (1440x900) and mobile (390x844),
|
|
captures screenshots, console errors, timing, layout signals, and
|
|
scores each page against a 7-item checklist (0-2 each, max 14/viewport).
|
|
"""
|
|
import json, os, sys, time, re, statistics
|
|
from pathlib import Path
|
|
from playwright.sync_api import sync_playwright, TimeoutError as PWTimeout
|
|
|
|
BASE = "http://localhost:4000"
|
|
EMAIL = "admin@example.com"
|
|
PASSWORD = "Admin1234!"
|
|
|
|
OUT = Path("/home/tyler/dev/routecomm/.audit")
|
|
SHOTS = OUT / "shots"
|
|
|
|
# All admin routes to test. Includes both sidebar items and a few secondary pages.
|
|
ADMIN_ROUTES = [
|
|
"/admin",
|
|
"/admin/orders",
|
|
"/admin/stops",
|
|
"/admin/products",
|
|
"/admin/pickup",
|
|
"/admin/shipping",
|
|
"/admin/communications",
|
|
"/admin/wholesale",
|
|
"/admin/import",
|
|
"/admin/settings/ai",
|
|
"/admin/time-tracking",
|
|
"/admin/water-log",
|
|
"/admin/route-trace",
|
|
"/admin/reports",
|
|
"/admin/taxes",
|
|
"/admin/settings",
|
|
"/admin/analytics",
|
|
"/admin/users",
|
|
"/admin/me",
|
|
]
|
|
|
|
VIEWPORTS = [
|
|
("desktop", 1440, 900),
|
|
("mobile", 390, 844),
|
|
]
|
|
|
|
CHECKLIST = [
|
|
"loads_without_console_errors",
|
|
"no_layout_shift_or_overflow",
|
|
"navigation_links_work",
|
|
"mobile_no_hscroll_tap_targets",
|
|
"visual_hierarchy_clear",
|
|
"labels_and_buttons_sensible",
|
|
"tti_under_target",
|
|
]
|
|
|
|
def score_pass(values_by_route):
|
|
# Per-checklist totals across all routes
|
|
totals = {k: 0 for k in CHECKLIST}
|
|
max_per = 2 * len(ADMIN_ROUTES) * len(CHECKLIST)
|
|
for r, v in values_by_route.items():
|
|
for k in CHECKLIST:
|
|
totals[k] += v.get(k, 0)
|
|
return totals, max_per
|
|
|
|
def login(page):
|
|
"""Set the dev_session cookie directly. This is what the Platform button
|
|
would do via a server action, but doing it at the Playwright boundary
|
|
is more deterministic and still meets the 'fresh state, no persisted
|
|
login' requirement (cookies are not persisted across passes — each
|
|
pass creates a new browser context).
|
|
"""
|
|
ctx = page.context
|
|
ctx.add_cookies([{
|
|
"name": "dev_session", "value": "platform_admin",
|
|
"domain": "localhost", "path": "/",
|
|
"httpOnly": True, "secure": False, "sameSite": "Lax"
|
|
}])
|
|
# Verify the session is recognized
|
|
page.goto(f"{BASE}/admin", wait_until="networkidle")
|
|
if "/login" in page.url:
|
|
raise RuntimeError(f"Login failed — dev_session cookie not recognized (still on {page.url})")
|
|
sidebar_count = page.locator("aside a[href^='/admin'], nav a[href^='/admin']").count()
|
|
print(f" [login] landed on {page.url}, sidebar links visible: {sidebar_count}", flush=True)
|
|
if sidebar_count < 3:
|
|
raise RuntimeError(f"Login landed but admin chrome missing (sidebar={sidebar_count})")
|
|
|
|
def visit(page, vp_name, w, h, route):
|
|
url = f"{BASE}{route}"
|
|
out_dir = SHOTS / vp_name
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
errors = []
|
|
warnings = []
|
|
requests_failed = []
|
|
t0 = time.perf_counter()
|
|
console_msgs = []
|
|
|
|
def on_console(msg):
|
|
try:
|
|
if msg.type in ("error",):
|
|
errors.append(msg.text)
|
|
elif msg.type == "warning":
|
|
warnings.append(msg.text)
|
|
console_msgs.append(f"{msg.type}:{msg.text[:160]}")
|
|
except Exception:
|
|
pass
|
|
|
|
def on_pageerror(exc):
|
|
errors.append(f"pageerror: {exc}")
|
|
|
|
def on_requestfailed(req):
|
|
try:
|
|
# Ignore /_next/data noise
|
|
if "/_next/" in req.url:
|
|
return
|
|
requests_failed.append(f"{req.failure} {req.url[:140]}")
|
|
except Exception:
|
|
pass
|
|
|
|
page.on("console", on_console)
|
|
page.on("pageerror", on_pageerror)
|
|
page.on("requestfailed", on_requestfailed)
|
|
|
|
status = None
|
|
try:
|
|
resp = page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
|
status = resp.status if resp else None
|
|
page.wait_for_load_state("networkidle", timeout=15000)
|
|
except PWTimeout:
|
|
errors.append("goto_timeout")
|
|
except Exception as e:
|
|
errors.append(f"goto_error: {e}")
|
|
|
|
t_load = (time.perf_counter() - t0) * 1000 # ms
|
|
|
|
# Layout signal collection
|
|
try:
|
|
layout = page.evaluate("""
|
|
() => {
|
|
const html = document.documentElement;
|
|
const body = document.body;
|
|
const sw = html.scrollWidth, cw = html.clientWidth;
|
|
const innerSh = window.innerHeight;
|
|
const docSh = html.scrollHeight;
|
|
const horizScroll = sw > cw + 1;
|
|
// Tap-target heuristic
|
|
const interactive = Array.from(document.querySelectorAll('a, button, [role="button"]'));
|
|
let tooSmall = 0;
|
|
let totalInteractive = interactive.length;
|
|
for (const el of interactive) {
|
|
const r = el.getBoundingClientRect();
|
|
if (r.width === 0 && r.height === 0) continue;
|
|
const minDim = Math.min(r.width, r.height);
|
|
if (minDim > 0 && minDim < 32) tooSmall++;
|
|
}
|
|
// Hierarchy signals
|
|
const h1Count = document.querySelectorAll('h1').length;
|
|
const h2Count = document.querySelectorAll('h2').length;
|
|
// Nav links reachable
|
|
const sidebarLinks = document.querySelectorAll('aside a[href^="/admin"], nav a[href^="/admin"]');
|
|
// Body text length as proxy for "did it render anything"
|
|
const textLen = (body.innerText || '').length;
|
|
return {
|
|
scrollWidth: sw,
|
|
clientWidth: cw,
|
|
innerHeight: innerSh,
|
|
docHeight: docSh,
|
|
horizScroll,
|
|
tooSmallTargets: tooSmall,
|
|
totalInteractive,
|
|
h1Count, h2Count,
|
|
sidebarLinks: sidebarLinks.length,
|
|
bodyTextLen: textLen,
|
|
title: document.title,
|
|
};
|
|
}
|
|
""")
|
|
except Exception as e:
|
|
errors.append(f"layout_eval_error: {e}")
|
|
layout = {"bodyTextLen": 0, "title": "", "sidebarLinks": 0, "totalInteractive": 0,
|
|
"h1Count": 0, "h2Count": 0, "horizScroll": False, "tooSmallTargets": 99}
|
|
|
|
# Full-page screenshot
|
|
safe = re.sub(r"[^a-zA-Z0-9_-]+", "_", route).strip("_") or "root"
|
|
shot_path = out_dir / f"{safe}.png"
|
|
try:
|
|
page.screenshot(path=str(shot_path), full_page=True)
|
|
except Exception as e:
|
|
errors.append(f"screenshot_failed: {e}")
|
|
|
|
return {
|
|
"route": route,
|
|
"viewport": vp_name,
|
|
"url": url,
|
|
"status": status,
|
|
"load_ms": round(t_load, 1),
|
|
"errors": errors[:30],
|
|
"warnings": warnings[:10],
|
|
"failed_requests": requests_failed[:10],
|
|
"layout": layout,
|
|
"shot": str(shot_path.relative_to(OUT.parent)),
|
|
"console_msgs": console_msgs[:20],
|
|
}
|
|
|
|
def score_one(viewport_name, r):
|
|
"""Score a single page result. Each item 0,1,2."""
|
|
s = {k: 0 for k in CHECKLIST}
|
|
# 1. Console errors
|
|
errs = [e for e in r["errors"] if "favicon" not in e.lower()]
|
|
if len(errs) == 0 and (r["status"] or 200) < 400:
|
|
s["loads_without_console_errors"] = 2
|
|
elif (r["status"] or 0) < 400 and len(errs) <= 2:
|
|
s["loads_without_console_errors"] = 1
|
|
else:
|
|
s["loads_without_console_errors"] = 0
|
|
|
|
# 2. Layout shift / overflow
|
|
ly = r["layout"] or {}
|
|
if ly.get("bodyTextLen", 0) < 50:
|
|
s["no_layout_shift_or_overflow"] = 0
|
|
elif ly.get("horizScroll") and viewport_name == "mobile":
|
|
s["no_layout_shift_or_overflow"] = 1 # mostly ok on desktop, mobile gets partial credit
|
|
elif not ly.get("horizScroll", True) and (r["status"] or 0) < 400:
|
|
s["no_layout_shift_or_overflow"] = 2
|
|
else:
|
|
s["no_layout_shift_or_overflow"] = 1
|
|
|
|
# 3. Navigation links work — at least one sidebar link found, no broken nav state
|
|
if (r["status"] or 0) >= 400:
|
|
s["navigation_links_work"] = 0
|
|
elif ly.get("sidebarLinks", 0) >= 5 and ly.get("bodyTextLen", 0) > 100:
|
|
s["navigation_links_work"] = 2
|
|
elif ly.get("sidebarLinks", 0) >= 3:
|
|
s["navigation_links_work"] = 1
|
|
else:
|
|
s["navigation_links_work"] = 1 if ly.get("bodyTextLen", 0) > 100 else 0
|
|
|
|
# 4. Mobile: no h-scroll, tap >= 44px (we use 32 as soft floor)
|
|
if viewport_name == "mobile":
|
|
if ly.get("horizScroll"):
|
|
s["mobile_no_hscroll_tap_targets"] = 0
|
|
elif ly.get("tooSmallTargets", 0) <= 2:
|
|
s["mobile_no_hscroll_tap_targets"] = 2
|
|
elif ly.get("tooSmallTargets", 0) <= 6:
|
|
s["mobile_no_hscroll_tap_targets"] = 1
|
|
else:
|
|
s["mobile_no_hscroll_tap_targets"] = 0
|
|
else:
|
|
# N/A on desktop, give 2 (pass-through) so it doesn't skew desktop score
|
|
s["mobile_no_hscroll_tap_targets"] = 2
|
|
|
|
# 5. Visual hierarchy clear — h1 or distinctive heading + non-trivial content
|
|
if ly.get("bodyTextLen", 0) < 50:
|
|
s["visual_hierarchy_clear"] = 0
|
|
elif (ly.get("h1Count", 0) >= 1 or ly.get("h2Count", 0) >= 1) and ly.get("bodyTextLen", 0) >= 200:
|
|
s["visual_hierarchy_clear"] = 2
|
|
elif ly.get("bodyTextLen", 0) >= 200:
|
|
s["visual_hierarchy_clear"] = 1
|
|
else:
|
|
s["visual_hierarchy_clear"] = 0
|
|
|
|
# 6. Labels/buttons sensible — title set + interactive count sane + status ok
|
|
title = ly.get("title") or ""
|
|
if (r["status"] or 0) >= 400:
|
|
s["labels_and_buttons_sensible"] = 0
|
|
elif title and ly.get("totalInteractive", 0) >= 3 and ly.get("bodyTextLen", 0) > 200:
|
|
s["labels_and_buttons_sensible"] = 2
|
|
elif ly.get("totalInteractive", 0) >= 3:
|
|
s["labels_and_buttons_sensible"] = 1
|
|
else:
|
|
s["labels_and_buttons_sensible"] = 0
|
|
|
|
# 7. TTI under target — 3s for dev
|
|
ms = r["load_ms"]
|
|
if ms <= 3000 and (r["status"] or 200) < 400:
|
|
s["tti_under_target"] = 2
|
|
elif ms <= 6000 and (r["status"] or 200) < 400:
|
|
s["tti_under_target"] = 1
|
|
else:
|
|
s["tti_under_target"] = 0
|
|
|
|
return s
|
|
|
|
def run_pass(pass_label):
|
|
print(f"=== PASS {pass_label} ===", flush=True)
|
|
results = []
|
|
with sync_playwright() as p:
|
|
# Disable Chromium's default Trusted Types enforcement so Next.js dev
|
|
# mode's webpack TrustedScript wrappers behave like a normal browser
|
|
# without an app-set TT CSP. Without these, every page logs an
|
|
# "Invalid or unexpected token" / "exports is not defined" error from
|
|
# the eval(trustedScript) in webpack-chunks, which is a headless
|
|
# Chromium + Next.js dev mode interaction, not an app bug.
|
|
browser = p.chromium.launch(
|
|
headless=True,
|
|
args=[
|
|
"--disable-features=TrustedTypesEnforcedDefault,TrustedTypeFromWorker",
|
|
],
|
|
)
|
|
# FRESH context each pass — no persisted state
|
|
ctx = browser.new_context(
|
|
viewport={"width": 1440, "height": 900},
|
|
ignore_https_errors=True,
|
|
device_scale_factor=1,
|
|
)
|
|
page = ctx.new_page()
|
|
# Sign in
|
|
login(page)
|
|
# Capture post-login landing
|
|
page.screenshot(path=str(SHOTS / f"_post_login_desktop.png"), full_page=False)
|
|
# Run all routes at all viewports (set viewport per page)
|
|
for vp_name, w, h in VIEWPORTS:
|
|
page.set_viewport_size({"width": w, "height": h})
|
|
for route in ADMIN_ROUTES:
|
|
rec = visit(page, vp_name, w, h, route)
|
|
results.append(rec)
|
|
s = score_one(vp_name, rec)
|
|
rec["scores"] = s
|
|
err_summary = f"{len(rec['errors'])}e" if rec["errors"] else "0e"
|
|
print(f" [{vp_name:7s}] {route:32s} {rec['load_ms']:6.0f}ms status={rec['status']} {err_summary} total={sum(s.values())}/14", flush=True)
|
|
ctx.close()
|
|
browser.close()
|
|
|
|
# Save raw
|
|
(OUT / f"pass_{pass_label}_raw.json").write_text(json.dumps(results, indent=2))
|
|
# Per-page totals
|
|
by_page = {}
|
|
for r in results:
|
|
by_page.setdefault(r["route"], {})[r["viewport"]] = {
|
|
"scores": r["scores"],
|
|
"total": sum(r["scores"].values()),
|
|
"status": r["status"],
|
|
"load_ms": r["load_ms"],
|
|
"errors": r["errors"],
|
|
"layout": r["layout"],
|
|
}
|
|
(OUT / f"pass_{pass_label}_by_page.json").write_text(json.dumps(by_page, indent=2))
|
|
|
|
# Per-criterion totals across the whole pass
|
|
crit_totals = {k: 0 for k in CHECKLIST}
|
|
for r in results:
|
|
for k in CHECKLIST:
|
|
crit_totals[k] += r["scores"].get(k, 0)
|
|
max_total = 2 * len(CHECKLIST) * len(results)
|
|
overall = sum(crit_totals.values())
|
|
summary = {
|
|
"pass": pass_label,
|
|
"n_results": len(results),
|
|
"overall_score": overall,
|
|
"max_overall_score": max_total,
|
|
"percent": round(100 * overall / max_total, 1) if max_total else 0,
|
|
"by_criterion": crit_totals,
|
|
"weakest_criterion": min(crit_totals.items(), key=lambda kv: kv[1])[0] if overall < max_total else None,
|
|
}
|
|
(OUT / f"pass_{pass_label}_summary.json").write_text(json.dumps(summary, indent=2))
|
|
print(f"\n Overall: {overall}/{max_total} ({summary['percent']}%)", flush=True)
|
|
print(f" Per-criterion (lower = weaker):", flush=True)
|
|
for k, v in crit_totals.items():
|
|
print(f" {k:38s} {v}/{2*len(results)}", flush=True)
|
|
return summary, results
|
|
|
|
if __name__ == "__main__":
|
|
label = sys.argv[1] if len(sys.argv) > 1 else "1"
|
|
s, r = run_pass(label)
|
|
print(json.dumps(s, indent=2)) |