pig/scripts/deploy.sh

#!/usr/bin/env bash
#
# Deploy PIG. Run on the host that serves it.
#
#   ./scripts/deploy.sh
#
# Deliberately a script rather than automated push-to-deploy. Automating it
# would mean putting an SSH key with write access to the production host onto
# the CI runner, which is a meaningful escalation for a project this size. CI
# proves the commit is sound; a human decides when it ships.
#
# That constraint still holds under the tag-to-ship flow added later. Nothing
# on the CI runner can reach this host: CI publishes an image, and this host
# PULLS it (scripts/autodeploy.sh). The direction of the credential is the
# whole point — a read-only registry token here, no host credential there. And
# a human still decides when it ships, by choosing to create a `release-*` tag.
#
#   PIG_IMAGE unset   Build from this working tree, as it always has.
#   PIG_IMAGE set     Pull that published image instead of building, and leave
#                     the checkout exactly where the caller put it. The caller
#                     is responsible for having checked out the matching
#                     commit, because the compose file, the migrations and the
#                     image have to agree.
#
# Safe to re-run. Migrations are additive and tracked.

set -euo pipefail

# Resolved BEFORE the re-exec below and carried across it. After the re-exec
# `$0` is a copy in /tmp, so `dirname "$0"` would point at the wrong tree.
PIG_REPO_ROOT="${PIG_REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
export PIG_REPO_ROOT
cd "$PIG_REPO_ROOT"

# Run from a copy, because this script rewrites itself.
#
# `git reset --hard origin/main` below replaces this very file while bash is
# still reading it. bash does not slurp a script: it reads incrementally and
# remembers a byte OFFSET, so after the reset it resumes at that offset into
# different content — which silently skips or splices steps, and at worst
# executes a fragment of a line. The 13dec6b deploy hit exactly this: the new
# public-origin gate was on disk and never ran, because bash was still
# executing the buffered previous version.
#
# It failed harmlessly that time. It is not guaranteed to. scripts/autodeploy.sh
# has always had this guard; deploy.sh needed it for the same reason.
if [ "${PIG_DEPLOY_REEXEC:-}" != '1' ]; then
  _copy=$(mktemp -t pig-deploy.XXXXXX)
  cat "$0" > "$_copy"
  PIG_DEPLOY_REEXEC=1 exec bash "$_copy" "$@"
fi
trap 'rm -f "$0"' EXIT

# What compose will run. Must match the `image:` default in docker-compose.yml,
# because that is the tag a rollback re-points at the previous image.
IMAGE_REF="${PIG_IMAGE:-pig:local}"

# Which compose profiles are active. Set below, once .env has been read; empty
# means none, which is what compose does by default anyway.
COMPOSE_PROFILES=''

# Every compose invocation goes through this. sudo's default `env_reset` drops
# PIG_IMAGE, so a plain `sudo docker compose` interpolates the `pig:local`
# fallback in docker-compose.yml instead of the release tag: the pull then fails
# with "pull access denied for pig", and — worse, if the pull is ever made
# non-fatal — the migrate, the `up` and the rollback all silently run whatever
# `pig:local` happens to be while the log reports the release. Pass it on the
# command line via env(1): `sudo -E` and bare `sudo VAR=val` are both refused by
# the default sudoers policy, `sudo env VAR=val …` is not.
#
# COMPOSE_PROFILES rides along for the same reason and needs it just as badly:
# the piggy service is profile-gated, and compose ignores a profile-gated
# service SILENTLY — `pull`, `build` and `up` behave as though it were not in
# the file at all, with no warning and a zero exit. Carrying the profile here
# rather than at each call site is what stops one forgotten flag leaving the
# agent on the previous release.
dc() {
  sudo env PIG_IMAGE="$IMAGE_REF" COMPOSE_PROFILES="$COMPOSE_PROFILES" docker compose -p pig "$@"
}

# Gate failures come in two kinds and the caller must be able to tell them
# apart: scripts/autodeploy.sh reports to the journal what is serving right now.
#   1  a gate failed and the previous image was restored — the old release is up
#   3  a gate failed and the release under test is STILL LIVE
EXIT_STILL_LIVE=3

# Read one key out of .env WITHOUT sourcing it. Sourcing an environment file
# executes it, and this one holds every secret the deployment has.
env_value() {
  [ -f .env ] || return 0
  sed -n "s/^[[:space:]]*$1=//p" .env | tail -n1 | sed -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/"
}

# The same words the API accepts, from `envBoolean` in apps/api/src/lib/config.ts.
# If this and that ever disagree, the CRM offers a chat surface backed by a
# container this script never started.
is_true() {
  # Compose drops a whitespace-preceded inline comment before the container ever
  # sees the value, so this has to as well: `PIGGY_ENABLED=true  # on` would
  # otherwise read as false here and true there — the agent offered by the CRM
  # and never deployed, which is the exact fault this whole path exists to end.
  local value=${1%%[[:space:]]#*}
  case "$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" in
    1 | true | yes | on) return 0 ;;
    *) return 1 ;;
  esac
}

# The database this deployment actually has, with the same defaults
# docker-compose.yml interpolates. Hardcoding `pig` here is survivable only
# while nobody sets POSTGRES_USER: on a host that does, the pre-migration dump
# would fail and take the deploy with it — fail-safe, but it fails the deploy
# instead of backing up the database, which is not the intent.
PG_USER="$(env_value POSTGRES_USER)"
PG_USER="${PG_USER:-pig}"
PG_DB="$(env_value POSTGRES_DB)"
PG_DB="${PG_DB:-pig}"

# Piggy is part of the release or it is not; there is no half-deployed state
# worth having. Left out of the pull, the build, the `up` and the rollback, it
# runs the PREVIOUS image against the schema this deploy just migrated — the
# hazard docker-compose.yml's own comment warns about — or does not run at all
# while the deploy reports success.
DEPLOY_SERVICES=(app)
PIGGY_IN_RELEASE=0
if is_true "$(env_value PIGGY_ENABLED)"; then
  COMPOSE_PROFILES='piggy'
  DEPLOY_SERVICES+=(piggy)
  PIGGY_IN_RELEASE=1
  echo "==> Piggy is enabled; it ships with this release"
fi

if [ -n "${PIG_IMAGE:-}" ]; then
  echo "==> Deploying published image $PIG_IMAGE"
  # No git sync. The caller has already detached this checkout at the tag the
  # image was built from; resetting to origin/main here would silently deploy a
  # compose file and a migration set from a different commit than the image.
  AFTER=$(git rev-parse --short HEAD)
  echo "    tree at $AFTER"
else
  echo "==> Fetching"
  git fetch -q origin
  BEFORE=$(git rev-parse --short HEAD)
  git reset --hard -q origin/main
  AFTER=$(git rev-parse --short HEAD)

  if [ "$BEFORE" = "$AFTER" ]; then
    echo "    Already at $AFTER"
  else
    echo "    $BEFORE -> $AFTER"
    git --no-pager log --oneline "$BEFORE..$AFTER" | sed 's/^/    /'
  fi
fi

echo "==> Starting the database"
# Before the dump, not after it. The backup step below `exec`s into this
# container, so on a host where the stack is down — a reboot, a `compose down`,
# a first-ever deploy — the backup was the step that failed, and it took the
# whole release with it. Fail-safe, but it aborted the deploy instead of
# protecting the data, which is the opposite of the intent.
dc up -d db
for _ in $(seq 1 60); do
  if dc exec -T db pg_isready -U "$PG_USER" -d "$PG_DB" > /dev/null; then break; fi
  sleep 1
done
if ! dc exec -T db pg_isready -U "$PG_USER" -d "$PG_DB" > /dev/null; then
  echo "ERROR: database did not become ready within 60 seconds" >&2
  exit 1
fi

echo "==> Backing up the database first"
# This used to be cheap insurance against a bad migration. It is now the thing
# standing between an agent bug and the book: Piggy can WRITE to the CRM —
# accounts, deals, activity — as the signed-in user, so a release that ships a
# broken write tool can corrupt data no migration ever touched. Restoring from
# here is the only undo that exists.
#
# Taken BEFORE the migration and before the new image starts, which is the only
# ordering that gives a dump of the schema the old code wrote.
#
# How many tables there are, so an empty dump can be told apart from a failed
# one. A first deploy legitimately dumps almost nothing; every later deploy
# dumping almost nothing is a fault.
PUBLIC_TABLES=$(dc exec -T db psql -U "$PG_USER" -d "$PG_DB" -tAc \
  "select count(*) from information_schema.tables where table_schema = 'public'" | tr -cd '0-9')
PUBLIC_TABLES="${PUBLIC_TABLES:-0}"
mkdir -p backups
BACKUP="backups/pig-$(date +%Y%m%d-%H%M%S).sql.gz"
dc exec -T db pg_dump -U "$PG_USER" "$PG_DB" | gzip > "$BACKUP"

# pipefail catches a pg_dump that FAILS. It does not catch a pg_dump that
# succeeds and says nothing — a wrong database name, an empty scratch volume
# adopted by mistake — and gzip turns that silence into a plausible-looking
# 20-byte file. A dump nobody can restore from is worse than a missing one,
# because the deploy log claims a backup was taken. Decompressing the whole
# thing verifies the gzip stream as a side effect, and `wc -c` reads to EOF so
# nothing here dies on SIGPIPE.
BACKUP_BYTES=$(gzip -dc "$BACKUP" | wc -c)
if [ "$PUBLIC_TABLES" -gt 0 ] && [ "$BACKUP_BYTES" -lt 4096 ]; then
  echo "ERROR: $PG_DB has $PUBLIC_TABLES tables but $BACKUP decompresses to only" >&2
  echo "       $BACKUP_BYTES bytes. That is not a usable dump. Refusing to migrate" >&2
  echo "       without one, because this dump is the only rollback the data has." >&2
  exit 1
fi
echo "    $BACKUP ($(du -h "$BACKUP" | cut -f1) compressed, $BACKUP_BYTES bytes of SQL, $PUBLIC_TABLES tables)"
if [ "$PUBLIC_TABLES" -eq 0 ]; then
  echo "    (no tables yet — this looks like a first deploy, so a thin dump is expected)"
fi
# Kept next to the checkout on purpose. backups/ is in .gitignore, so
# scripts/autodeploy.sh's `git checkout --detach --force` leaves it alone — a
# dump that the next deploy deletes is not a backup. Nothing here prunes them
# either; they are small, they are the only copy, and a disk-space problem is a
# better problem than a missing dump. Copy them off this host if the data
# matters as much as the uptime does.

if [ -n "${PIG_IMAGE:-}" ]; then
  echo "==> Pulling"
  dc pull "${DEPLOY_SERVICES[@]}"
else
  echo "==> Building"
  dc build "${DEPLOY_SERVICES[@]}"
fi

echo "==> Migrating before the schema-dependent app starts"
# A release may query a newly introduced table during startup. Running the
# migration from a one-off container prevents that app from crash-looping
# before an `exec`-based migration can reach it.
#
# This ordering matters more with every migration that a NEW surface depends
# on. 0014 adds piggy_conversations and piggy_messages, which the API writes on
# every agent turn — and the agent turn is paid for BEFORE the insert, so a
# release that starts ahead of its migration takes a user's question, calls a
# model, bills for it and then fails to persist the answer. `dc up` starts app
# and piggy together below, so both are behind this line.
#
# The image runs it, not the host: the migration set that runs is the one baked
# into $IMAGE_REF, so a published image and its migrations can never disagree.
# `--no-deps` because the database is already up, and letting compose start
# dependencies here would start the OLD app container against an unmigrated
# schema, which is the exact race this step exists to avoid.
dc run --rm --no-deps app pnpm exec tsx packages/db/src/migrate.ts

# The image the current container is running, captured before it is replaced.
# Without this a failed release stays live: both gates below used to exit 1 and
# leave the broken version serving, which is tolerable when a human is reading
# the terminal and an outage when the poller ran this at 04:00.
PREVIOUS_IMAGE=""
PREVIOUS_CID=$(dc ps -q app 2>/dev/null | head -n1 || true)
if [ -n "$PREVIOUS_CID" ]; then
  PREVIOUS_IMAGE=$(sudo docker inspect -f '{{.Image}}' "$PREVIOUS_CID" 2>/dev/null || true)
fi

# Restore the previous image and exit non-zero. Deliberately NOT a database
# rollback: migrations are additive, so the previous code runs against the new
# schema, and the dump taken above is the escape hatch for the case where it
# does not. The re-tag makes $IMAGE_REF point back at the old image locally; the
# next successful pull moves it forward again.
#
# Leaving the schema ahead of the image is the deliberate half of that, and it
# is safe for exactly the reason above. 0014 is the current case: it CREATES
# piggy_conversations and piggy_messages and alters nothing, so an image that
# predates it never names those tables and cannot notice they exist. The rule
# this relies on is a rule about migrations, not about this script — a
# migration that drops a column, renames one, or tightens a constraint breaks
# the previous image and makes this rollback a partial outage. Write additive
# migrations, or plan the rollback with the dump in hand.
roll_back() {
  echo "    !! $1"
  dc logs app --tail 40 || true

  if [ -z "$PREVIOUS_IMAGE" ]; then
    echo "    NO PREVIOUS IMAGE TO ROLL BACK TO — the broken release is live" >&2
    exit "$EXIT_STILL_LIVE"
  fi

  echo "==> Rolling back to $PREVIOUS_IMAGE"
  sudo docker tag "$PREVIOUS_IMAGE" "$IMAGE_REF"
  # Piggy included: app and piggy are one image running two commands, and a
  # rollback that restores only the app leaves the two halves of the same
  # release on different code.
  dc up -d --no-build "${DEPLOY_SERVICES[@]}"

  for _ in $(seq 1 60); do
    if curl -sf http://127.0.0.1:8920/api/health > /dev/null; then break; fi
    sleep 1
  done
  if curl -sf http://127.0.0.1:8920/api/health | grep -q '"ok":true'; then
    echo "    rolled back, previous release is healthy"
    exit 1
  fi
  # The restore ran but did not come up. Reporting "rolled back" here would tell
  # the on-call the previous release is serving when nothing is.
  echo "    ROLLBACK ALSO UNHEALTHY — this host needs a human" >&2
  exit "$EXIT_STILL_LIVE"
}

echo "==> Starting the app"
dc up -d "${DEPLOY_SERVICES[@]}"

echo "==> Waiting for health"
for _ in $(seq 1 60); do
  if curl -sf http://127.0.0.1:8920/api/health > /dev/null; then break; fi
  sleep 1
done

echo "==> Verifying"
if curl -sf http://127.0.0.1:8920/api/health | grep -q '"ok":true'; then
  echo "    health ok"
else
  echo "    HEALTH CHECK FAILED"
  roll_back "health check failed"
fi

# Authentication must be enforced. A deploy that accidentally serves the CRM
# unauthenticated is the one failure worth blocking on.
CODE=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8920/api/dashboard)
if [ "$CODE" != "401" ]; then
  echo "    UNAUTHENTICATED REQUEST RETURNED $CODE, EXPECTED 401"
  roll_back "unauthenticated request returned $CODE, expected 401"
fi
echo "    auth enforced"

if [ "$PIGGY_IN_RELEASE" = '1' ]; then
  echo "==> Verifying Piggy"
  PIGGY_CID=$(dc ps -q piggy 2>/dev/null | head -n1 || true)
  if [ -z "$PIGGY_CID" ]; then
    echo "    PIGGY IS ENABLED IN .env BUT NO PIGGY CONTAINER IS RUNNING"
    roll_back "piggy is enabled but no piggy container is running"
  fi

  # Image IDs, not tags. The failure this catches is a Piggy someone started by
  # hand once and never touched again: it answers to the same tag while running
  # whatever that tag meant on the day, so a tag comparison sees nothing wrong
  # and the old agent goes on writing to a schema three migrations newer.
  WANTED_IMAGE_ID=$(sudo docker image inspect -f '{{.Id}}' "$IMAGE_REF" 2>/dev/null || true)
  PIGGY_IMAGE_ID=$(sudo docker inspect -f '{{.Image}}' "$PIGGY_CID" 2>/dev/null || true)
  if [ -z "$WANTED_IMAGE_ID" ] || [ "$PIGGY_IMAGE_ID" != "$WANTED_IMAGE_ID" ]; then
    echo "    PIGGY IS RUNNING ${PIGGY_IMAGE_ID:-<unknown>}, EXPECTED ${WANTED_IMAGE_ID:-<unknown>}"
    roll_back "piggy is not running $IMAGE_REF"
  fi
  echo "    piggy on $IMAGE_REF"

  # Where the harness is allowed to look at the filesystem, read back from the
  # container that is actually running rather than from the file that was meant
  # to configure it.
  #
  # The harness discovers extensions, skills and context files from its cwd, and
  # Piggy hands it PIGGY_AGENT_DIR as cwd. Inside /app that is the application
  # checkout, so a stray bind mount or a hand-edited compose file would put
  # source code within reach of a CRM agent's prompt — a leak that every other
  # gate here passes cheerfully, because the container is perfectly healthy.
  # Empty is fine: an older compose file simply does not set it, and the
  # in-code default is ~/.pig/piggy-agent.
  PIGGY_AGENT_DIR_LIVE=$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' \
    "$PIGGY_CID" 2>/dev/null | sed -n 's/^PIGGY_AGENT_DIR=//p' | tail -n1)
  case "${PIGGY_AGENT_DIR_LIVE:-}" in
    /app | /app/*)
      echo "    PIGGY_AGENT_DIR IS $PIGGY_AGENT_DIR_LIVE, INSIDE THE CHECKOUT" >&2
      echo "    The agent harness treats that directory as its cwd and reads" >&2
      echo "    context files from it, so this puts repository contents inside a" >&2
      echo "    CRM agent's prompt. Point it at /var/lib/piggy-agent, which the" >&2
      echo "    image creates for exactly this, and redeploy. Not rolled back:" >&2
      echo "    the previous image reads the same compose file." >&2
      exit "$EXIT_STILL_LIVE"
      ;;
  esac

  # `starting` until the first probe answers, so this is a wait, not a poll of
  # something already decided. start_period is 20s and the interval 30s, hence
  # the longer budget than the app's.
  PIGGY_HEALTH=''
  for _ in $(seq 1 90); do
    PIGGY_HEALTH=$(sudo docker inspect \
      -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
      "$PIGGY_CID" 2>/dev/null || true)
    [ "$PIGGY_HEALTH" = 'starting' ] || break
    sleep 1
  done

  if [ "$PIGGY_HEALTH" = 'healthy' ]; then
    echo "    piggy healthy"
  elif [ "$PIGGY_HEALTH" = 'none' ]; then
    # Only reachable from a container created by an older compose file, since
    # this one defines a healthcheck for the service. Worth saying rather than
    # passing quietly, because "no result" is not "well".
    echo "    piggy has no healthcheck to consult; recreate it to get one" >&2
  else
    echo "    PIGGY IS ${PIGGY_HEALTH:-UNKNOWN}" >&2
    dc logs piggy --tail 40 || true
    # Deliberately no rollback, for the same reason the empty-body case below
    # does not: the previous image reads this same .env and fails identically,
    # so restoring it churns the CRM without fixing the agent. The CRM is live
    # and well; the agent the operator asked for is not.
    echo "    The CRM is serving. Piggy is not, and a rollback would not help:" >&2
    echo "    the previous image reads the same .env. Usual causes, in order:" >&2
    echo "      - PRIME_API_KEY missing or rejected (PIGGY_INFERENCE_API_KEY is" >&2
    echo "        still accepted as its alias); Piggy exits at boot and" >&2
    echo "        restart: unless-stopped turns that into a crash loop" >&2
    echo "      - PIGGY_INTERNAL_TOKEN shorter than 32 characters" >&2
    echo "      - PIGGY_AGENT_MODEL naming a model that is not in the catalogue" >&2
    echo "        at apps/piggy/src/agent/models.json" >&2
    echo "    The log above says which; the config error names the key." >&2
    exit "$EXIT_STILL_LIVE"
  fi
fi

# Everything above proves the container is well. It proves nothing about what
# the public actually gets, and there is a failure on this host that every
# check so far passes: a Caddy site block missing `bind 10.0.0.2` lands in a
# separate server on :443, wins for traffic arriving on that address, and
# answers with a valid certificate, HTTP 200 and an EMPTY body. The app is
# fine; the request never reached it. So ask the origin, from outside the
# compose network, and insist on seeing something the app actually renders.
PUBLIC_URL="${PIG_DEPLOY_PUBLIC_URL:-$(env_value PIG_PUBLIC_URL)}"
PUBLIC_URL="${PUBLIC_URL:-https://primeintellectgrowth.com}"
PUBLIC_MARKER="${PIG_DEPLOY_PUBLIC_MARKER:-<div id=\"root\">}"

echo "==> Verifying the public origin ($PUBLIC_URL)"
# Three outcomes, and conflating any two of them has already cost a deploy.
# `|| true` used to fold "curl never got an answer" into the empty string, so a
# host where the public name does not hairpin, or where egress to 443 is
# filtered, failed a completely healthy release — and autodeploy.sh then
# blacklists that digest for good. Keep the transport result separate from the
# body.
PUBLIC_ERR=$(mktemp)
trap 'rm -f "$PUBLIC_ERR"' EXIT
if PUBLIC_BODY=$(curl -sS --max-time 20 --location "${PUBLIC_URL%/}/" 2>"$PUBLIC_ERR"); then
  CURL_STATUS=0
else
  CURL_STATUS=$?
fi
PUBLIC_BYTES=${#PUBLIC_BODY}

if [ "$CURL_STATUS" -ne 0 ]; then
  # Could not connect at all: DNS, TLS, no hairpin, filtered egress. This says
  # nothing about the release, and nothing about the proxy either — from this
  # host the two are indistinguishable. Do not fail a deploy that passed every
  # check that can actually be trusted; set PIG_DEPLOY_REQUIRE_PUBLIC=1 on a
  # host where the public name IS reachable from here and this must block.
  echo "    could not reach the public origin (curl exit $CURL_STATUS)" >&2
  sed 's/^/      /' "$PUBLIC_ERR" >&2
  if [ "${PIG_DEPLOY_REQUIRE_PUBLIC:-0}" = '1' ]; then
    echo "    PIG_DEPLOY_REQUIRE_PUBLIC=1, so this is fatal." >&2
    exit "$EXIT_STILL_LIVE"
  fi
  echo "    Not treated as a failure: the container passed every local gate," >&2
  echo "    and this host may simply not be able to route to its own name." >&2
  echo "    Verify $PUBLIC_URL from somewhere else." >&2
elif [ "$PUBLIC_BYTES" -eq 0 ]; then
  echo "    PUBLIC ORIGIN ANSWERED WITH AN EMPTY BODY" >&2
  echo "    Something terminated TLS and answered, so this is the proxy, not" >&2
  echo "    the release. Check that every Caddy site block on this host has" >&2
  echo "    'bind 10.0.0.2'." >&2
  # Deliberately no rollback: the previous image would fail this same check.
  # Rolling back would churn the service and not fix a proxy fault. The release
  # under test therefore stays live, which is what EXIT_STILL_LIVE tells the
  # poller to say.
  exit "$EXIT_STILL_LIVE"
elif ! printf '%s' "$PUBLIC_BODY" | grep -qF "$PUBLIC_MARKER"; then
  echo "    PUBLIC ORIGIN ANSWERED $PUBLIC_BYTES BYTES WITHOUT '$PUBLIC_MARKER'" >&2
  # This one DOES roll back, unlike the empty-body case above. A proxy fault
  # cannot serve a wrong-but-non-empty page for this hostname; a bad release
  # can — a changed Vite `base`, a Dockerfile that stopped copying
  # apps/web/dist. Every earlier gate passes in that state (the API is well,
  # /api/dashboard still 401s) and only this one fires, so if it is the release
  # then refusing to roll back leaves the broken release serving the public.
  # Restoring the previous image is right when it is, and harmless when it is
  # not: the marker check is the last gate, so the rollback costs one restart.
  roll_back "public origin answered without '$PUBLIC_MARKER'"
else
  echo "    public origin serving PIG ($PUBLIC_BYTES bytes)"
fi

echo "==> Deployed $AFTER"