#!/usr/bin/env bash # # Deploy PIG. Run on the host that serves it. # # ./scripts/deploy.sh # # Deliberately a script rather than automated push-to-deploy. Automating it # would mean putting an SSH key with write access to the production host onto # the CI runner, which is a meaningful escalation for a project this size. CI # proves the commit is sound; a human decides when it ships. # # That constraint still holds under the tag-to-ship flow added later. Nothing # on the CI runner can reach this host: CI publishes an image, and this host # PULLS it (scripts/autodeploy.sh). The direction of the credential is the # whole point — a read-only registry token here, no host credential there. And # a human still decides when it ships, by choosing to create a `release-*` tag. # # PIG_IMAGE unset Build from this working tree, as it always has. # PIG_IMAGE set Pull that published image instead of building, and leave # the checkout exactly where the caller put it. The caller # is responsible for having checked out the matching # commit, because the compose file, the migrations and the # image have to agree. # # Safe to re-run. Migrations are additive and tracked. set -euo pipefail # Resolved BEFORE the re-exec below and carried across it. After the re-exec # `$0` is a copy in /tmp, so `dirname "$0"` would point at the wrong tree. PIG_REPO_ROOT="${PIG_REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" export PIG_REPO_ROOT cd "$PIG_REPO_ROOT" # Run from a copy, because this script rewrites itself. # # `git reset --hard origin/main` below replaces this very file while bash is # still reading it. bash does not slurp a script: it reads incrementally and # remembers a byte OFFSET, so after the reset it resumes at that offset into # different content — which silently skips or splices steps, and at worst # executes a fragment of a line. The 13dec6b deploy hit exactly this: the new # public-origin gate was on disk and never ran, because bash was still # executing the buffered previous version. # # It failed harmlessly that time. It is not guaranteed to. scripts/autodeploy.sh # has always had this guard; deploy.sh needed it for the same reason. if [ "${PIG_DEPLOY_REEXEC:-}" != '1' ]; then _copy=$(mktemp -t pig-deploy.XXXXXX) cat "$0" > "$_copy" PIG_DEPLOY_REEXEC=1 exec bash "$_copy" "$@" fi trap 'rm -f "$0"' EXIT # What compose will run. Must match the `image:` default in docker-compose.yml, # because that is the tag a rollback re-points at the previous image. IMAGE_REF="${PIG_IMAGE:-pig:local}" # Every compose invocation goes through this. sudo's default `env_reset` drops # PIG_IMAGE, so a plain `sudo docker compose` interpolates the `pig:local` # fallback in docker-compose.yml instead of the release tag: the pull then fails # with "pull access denied for pig", and — worse, if the pull is ever made # non-fatal — the migrate, the `up` and the rollback all silently run whatever # `pig:local` happens to be while the log reports the release. Pass it on the # command line via env(1): `sudo -E` and bare `sudo VAR=val` are both refused by # the default sudoers policy, `sudo env VAR=val …` is not. dc() { sudo env PIG_IMAGE="$IMAGE_REF" docker compose -p pig "$@" } # Gate failures come in two kinds and the caller must be able to tell them # apart: scripts/autodeploy.sh reports to the journal what is serving right now. # 1 a gate failed and the previous image was restored — the old release is up # 3 a gate failed and the release under test is STILL LIVE EXIT_STILL_LIVE=3 # Read one key out of .env WITHOUT sourcing it. Sourcing an environment file # executes it, and this one holds every secret the deployment has. env_value() { [ -f .env ] || return 0 sed -n "s/^[[:space:]]*$1=//p" .env | tail -n1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/" } if [ -n "${PIG_IMAGE:-}" ]; then echo "==> Deploying published image $PIG_IMAGE" # No git sync. The caller has already detached this checkout at the tag the # image was built from; resetting to origin/main here would silently deploy a # compose file and a migration set from a different commit than the image. AFTER=$(git rev-parse --short HEAD) echo " tree at $AFTER" else echo "==> Fetching" git fetch -q origin BEFORE=$(git rev-parse --short HEAD) git reset --hard -q origin/main AFTER=$(git rev-parse --short HEAD) if [ "$BEFORE" = "$AFTER" ]; then echo " Already at $AFTER" else echo " $BEFORE -> $AFTER" git --no-pager log --oneline "$BEFORE..$AFTER" | sed 's/^/ /' fi fi echo "==> Backing up the database first" # Cheap insurance. A migration that goes wrong on a database holding real deal # data is not something to discover without a dump in hand. mkdir -p backups BACKUP="backups/pig-$(date +%Y%m%d-%H%M%S).sql.gz" dc exec -T db pg_dump -U pig pig | gzip > "$BACKUP" echo " $BACKUP ($(du -h "$BACKUP" | cut -f1))" if [ -n "${PIG_IMAGE:-}" ]; then echo "==> Pulling" dc pull app else echo "==> Building" dc build app fi echo "==> Starting the database" dc up -d db for _ in $(seq 1 60); do if dc exec -T db pg_isready -U pig -d pig > /dev/null; then break; fi sleep 1 done if ! dc exec -T db pg_isready -U pig -d pig > /dev/null; then echo "ERROR: database did not become ready within 60 seconds" >&2 exit 1 fi echo "==> Migrating before the schema-dependent app starts" # A release may query a newly introduced table during startup. Running the # migration from a one-off container prevents that app from crash-looping # before an `exec`-based migration can reach it. dc run --rm --no-deps app pnpm exec tsx packages/db/src/migrate.ts # The image the current container is running, captured before it is replaced. # Without this a failed release stays live: both gates below used to exit 1 and # leave the broken version serving, which is tolerable when a human is reading # the terminal and an outage when the poller ran this at 04:00. PREVIOUS_IMAGE="" PREVIOUS_CID=$(dc ps -q app 2>/dev/null | head -n1 || true) if [ -n "$PREVIOUS_CID" ]; then PREVIOUS_IMAGE=$(sudo docker inspect -f '{{.Image}}' "$PREVIOUS_CID" 2>/dev/null || true) fi # Restore the previous image and exit non-zero. Deliberately NOT a database # rollback: migrations are additive, so the previous code runs against the new # schema, and the dump taken above is the escape hatch for the case where it # does not. The re-tag makes $IMAGE_REF point back at the old image locally; the # next successful pull moves it forward again. roll_back() { echo " !! $1" dc logs app --tail 40 || true if [ -z "$PREVIOUS_IMAGE" ]; then echo " NO PREVIOUS IMAGE TO ROLL BACK TO — the broken release is live" >&2 exit "$EXIT_STILL_LIVE" fi echo "==> Rolling back to $PREVIOUS_IMAGE" sudo docker tag "$PREVIOUS_IMAGE" "$IMAGE_REF" dc up -d --no-build app for _ in $(seq 1 60); do if curl -sf http://127.0.0.1:8920/api/health > /dev/null; then break; fi sleep 1 done if curl -sf http://127.0.0.1:8920/api/health | grep -q '"ok":true'; then echo " rolled back, previous release is healthy" exit 1 fi # The restore ran but did not come up. Reporting "rolled back" here would tell # the on-call the previous release is serving when nothing is. echo " ROLLBACK ALSO UNHEALTHY — this host needs a human" >&2 exit "$EXIT_STILL_LIVE" } echo "==> Starting the app" dc up -d app echo "==> Waiting for health" for _ in $(seq 1 60); do if curl -sf http://127.0.0.1:8920/api/health > /dev/null; then break; fi sleep 1 done echo "==> Verifying" if curl -sf http://127.0.0.1:8920/api/health | grep -q '"ok":true'; then echo " health ok" else echo " HEALTH CHECK FAILED" roll_back "health check failed" fi # Authentication must be enforced. A deploy that accidentally serves the CRM # unauthenticated is the one failure worth blocking on. CODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8920/api/dashboard) if [ "$CODE" != "401" ]; then echo " UNAUTHENTICATED REQUEST RETURNED $CODE, EXPECTED 401" roll_back "unauthenticated request returned $CODE, expected 401" fi echo " auth enforced" # Everything above proves the container is well. It proves nothing about what # the public actually gets, and there is a failure on this host that every # check so far passes: a Caddy site block missing `bind 10.0.0.2` lands in a # separate server on :443, wins for traffic arriving on that address, and # answers with a valid certificate, HTTP 200 and an EMPTY body. The app is # fine; the request never reached it. So ask the origin, from outside the # compose network, and insist on seeing something the app actually renders. PUBLIC_URL="${PIG_DEPLOY_PUBLIC_URL:-$(env_value PIG_PUBLIC_URL)}" PUBLIC_URL="${PUBLIC_URL:-https://primeintellectgrowth.com}" PUBLIC_MARKER="${PIG_DEPLOY_PUBLIC_MARKER:-