From 7ddb06d7bd1ce53f22a196afb63e62e9e7608cd5 Mon Sep 17 00:00:00 2001 From: default Date: Sat, 6 Jun 2026 00:55:38 +0000 Subject: [PATCH] fix(deploy): harden port-freeing against TOCTOU bind failures The Start Docker stack step was racing itself: docker compose down errors were silently swallowed (|| true), leaving the previous postgrest container in a half-torn-down state. fuser -k would kill the listener, ss would see 3011 as free, the script would pick 3011, and then docker compose up would fail to bind because the port was in TIME_WAIT (or a stale docker-proxy listener was still holding it). Three changes: 1. Don't swallow docker compose down errors. If the previous stack isn't actually down, fail loudly so we can investigate. 2. Explicitly remove any surviving route_commerce_postgrest container and kill any leftover docker-proxy listeners for ports 3011-3013 before picking a port. 3. Verify the postgrest container is gone after the cleanup. If it's not, exit 1 instead of continuing into a guaranteed bind failure. 4. Port-picking loop now retries with a 1s sleep on each port (handles TIME_WAIT settling), and uses a stricter ss grep that matches 127.0.0.1: anchored on whitespace to avoid false positives from substring matches (e.g. :30110 matching :3011). Reproduces the failure mode from the run on 2026-06-06: postgrest failed to bind 127.0.0.1:3011 with 'address already in use' even though the script had just reported it as free. --- .gitea/workflows/deploy.yml | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index d191eaf..1245f5f 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -33,25 +33,46 @@ jobs: # (so a new deploy can pick it back up if it's the lowest free port) PREV_PORT=$(cat .postgrest-port 2>/dev/null || echo "") for port in 3001 $PREV_PORT; do - if [ -n "$port" ] && ss -tln 2>/dev/null | grep -q ":$port "; then + if [ -n "$port" ] && ss -tln 2>/dev/null | grep -qE "[[:space:]]127\.0\.0\.1:${port}[[:space:]]"; then echo "Port $port in use, freeing..." fuser -k -9 $port/tcp 2>/dev/null || true docker ps -aq --filter "publish=$port" 2>/dev/null | xargs -r docker rm -f 2>/dev/null || true fi done - docker compose -f $APP_DIR/docker-compose.yml down --remove-orphans 2>/dev/null || true + + # Hard-stop the previous stack. Errors are NOT swallowed: if down + # fails, picking a port against a half-torn-down stack is exactly + # what produces the TOCTOU "address already in use" we keep hitting. + docker compose -f $APP_DIR/docker-compose.yml down --remove-orphans + # Belt-and-braces: anything with the postgrest name that survived. + docker ps -aq --filter "name=route_commerce_postgrest" | xargs -r docker rm -f >/dev/null 2>&1 || true + # docker-proxy sometimes leaves a listener behind for the published port. + pkill -9 -f 'docker-proxy.*3011' 2>/dev/null || true + pkill -9 -f 'docker-proxy.*3012' 2>/dev/null || true + pkill -9 -f 'docker-proxy.*3013' 2>/dev/null || true sleep 3 + # Verify the postgrest container is actually gone before we pick a port. + if docker ps -aq --filter "name=route_commerce_postgrest" | grep -q .; then + echo "ERROR: route_commerce_postgrest still running after down" + docker ps --filter "name=route_commerce_postgrest" + exit 1 + fi + # Find the first free host port starting from 3011. Persist the choice # so the Build and Deploy steps below can use the same URL. POSTGREST_HOST_PORT=3011 - while ss -tln 2>/dev/null | grep -q ":$POSTGREST_HOST_PORT "; do - echo "Port $POSTGREST_HOST_PORT in use, trying next..." + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if ! ss -tln 2>/dev/null | grep -qE "[[:space:]]127\.0\.0\.1:${POSTGREST_HOST_PORT}[[:space:]]"; then + break + fi + echo "Port $POSTGREST_HOST_PORT in use, trying next... (attempt $attempt)" POSTGREST_HOST_PORT=$((POSTGREST_HOST_PORT + 1)) if [ $POSTGREST_HOST_PORT -gt 30200 ]; then echo "ERROR: no free port in 3011-30200 range" exit 1 fi + sleep 1 done echo "Using PostgREST host port: $POSTGREST_HOST_PORT" echo "$POSTGREST_HOST_PORT" > .postgrest-port