salience-editor/deploy.sh

#!/bin/sh
set -eu

# rsync wrapper for unreliable network connections
# my home network is spotty and rsync often dies mid-transfer
# this lets me manually retry the failed rsync and continue the deploy
rsync_retry() {
	if ! rsync "$@"; then
		echo ""
		echo "rsync failed (probably network). run this manually:"
		echo "rsync $*"
		echo ""
		printf "press enter after manual rsync completes to continue..."
		read _
	fi
}

# Deployment topology and zero-downtime process
#
# Directory structure on remote:
#   $base/$project/
#     base_port            - starting port number (default 3100)
#     releases/{stamp}_{hash}/
#       dist/              - static assets served by nginx 
#       server/            - node server from frontend/server/entrypoint.express.js
#       api/               - python flask API (salience/, nltk_data/, etc.)
#       assigned_port      - port allocated to frontend for this release
#       assigned_api_port  - port allocated to API for this release
#     current -> releases/{latest}
#     systemd/             - unit files per release (frontend + API)
#
# Zero-downtime deployment:
#   1. rsync new release (frontend/dist/ + frontend/server/ + api/)
#   2. install dependencies (npm for frontend, uv for API)
#   3. allocate two ports (frontend + API via get-next-port.sh)
#   4. generate systemd units for new release with unique ports
#      - frontend: node server
#      - API: gunicorn with 4 workers running Flask app
#   5. start new services, wait for health
#   6. update nginx upstream to point to new ports
#   7. reload nginx (graceful, no dropped connections)
#   8. stop old services
#   9. cleanup old releases (keep 3 most recent)
#
# Port allocation: get-next-port.sh reads base_port and existing
# assigned_port files to find first available port.
# Each release runs independently until cutover.

ssh=deploy-peoplesgrocers-website
base=/home/peoplesgrocers
project=salience
nginx_conf=/etc/nginx/sites-available/$project
service_listen_address=127.221.91.58
local_nginx_snippet="$HOME/src/work/infra/servers/chicago-web01/nginx/snippets/qwik-city-apps/salience.conf"

test -d frontend/dist || { echo 'no frontend/dist/'; exit 1; }
test -d frontend/server || { echo 'no frontend/server/'; exit 1; }
test -d .git || { echo 'not a git repo'; exit 1; }
git diff-index --quiet HEAD || { echo 'git repo dirty'; exit 1; }

hash=$(git rev-parse --short=8 HEAD)
stamp=$(date +%Y-%b-%d-%a-%I-%M%p | tr 'APM' 'apm')
release="${stamp}_${hash}"
service_name="${project}-${release}"

echo "deploying: $project @ $release"
printf "continue? [y/n] "
read ans
test "$ans" = "y" || exit 1

# prepare remote directories
ssh $ssh "mkdir -p $base/$project/{releases,systemd} $base/$project/releases/$release"

# sync both dist and server
echo "syncing dist..."
rsync_retry -tvaz frontend/dist/ $ssh:$base/$project/releases/$release/dist/
echo "syncing server..."
rsync_retry -tvaz frontend/server/ $ssh:$base/$project/releases/$release/server/

# copy server package.json and install dependencies
echo "copying server package.json..."
scp frontend/package.json $ssh:$base/$project/releases/$release/package.json
echo "installing server dependencies..."
ssh $ssh "source ~/.nvm/nvm.sh && cd $base/$project/releases/$release && npm install"

# sync api directory (exclude benchmarks, include specific files/dirs)
echo "syncing api..."
rsync_retry -tvaz \
  --include='salience/' --include='salience/**' \
  --include='nltk_data/' --include='nltk_data/**' \
  --include='pyproject.toml' \
  --include='uv.lock' \
  --include='transcript.txt' \
  --include='README.md' \
  --exclude='*' \
  api/ $ssh:$base/$project/releases/$release/api/

# link to shared models cache
echo "linking to shared models_cache..."
ssh $ssh "mkdir -p $base/$project/shared/models_cache"
ssh $ssh "ln -sfn ../../../shared/models_cache $base/$project/releases/$release/api/models_cache"

echo "installing api dependencies..."
ssh $ssh "cd $base/$project/releases/$release/api && ~/.local/bin/uv sync"

set -x
# determine ports for this release (frontend and api)
port=$(sh get-next-port.sh)
echo "frontend port for this release: $port"
api_port=$(sh get-next-port.sh)
echo "api port for this release: $api_port"

# record port assignments
ssh $ssh "echo $port > $base/$project/releases/$release/assigned_port"
ssh $ssh "echo $api_port > $base/$project/releases/$release/assigned_api_port"

set +x

# generate systemd unit file
unit_content="[Unit]
Description=${project} release ${release}
After=network.target

[Service]
Type=simple
User=peoplesgrocers
WorkingDirectory=$base/$project/releases/$release
Environment=\"PORT=$port\"
Environment=\"ORIGIN=https://peoplesgrocers.com\"
ExecStart=/home/peoplesgrocers/.nvm/versions/node/v24.10.0/bin/node server/entry.express.js
Restart=on-failure
RestartSec=10

[Install]
WantedBy=multi-user.target"

echo "$unit_content" | ssh $ssh "cat > $base/$project/systemd/${service_name}.service"

echo ""
echo "systemd unit created at: $base/$project/systemd/${service_name}.service"
echo ""

# generate systemd unit file for API
api_service_name="${project}-api-${release}"
api_unit_content="[Unit]
Description=${project} API release ${release}
After=network.target

[Service]
Type=simple
User=peoplesgrocers
WorkingDirectory=$base/$project/releases/$release/api
Environment=\"PORT=$api_port\"
ExecStart=$base/$project/releases/$release/api/.venv/bin/gunicorn --bind ${service_listen_address}:$api_port --workers 4 salience:app
Restart=on-failure
RestartSec=10

[Install]
WantedBy=multi-user.target"

echo "$api_unit_content" | ssh $ssh "cat > $base/$project/systemd/${api_service_name}.service"

echo ""
echo "API systemd unit created at: $base/$project/systemd/${api_service_name}.service"
echo ""

# find old services
old_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-' | grep -v 'api' | awk '{print \$1}' | head -1" || true)
if [ -n "$old_service" ]; then
	old_port=$(ssh $ssh "systemctl show $old_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
	echo "old frontend service: $old_service (port $old_port)"
else
	old_port=""
	echo "no old frontend service running"
fi

old_api_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-api-' | awk '{print \$1}' | head -1" || true)
if [ -n "$old_api_service" ]; then
	old_api_port=$(ssh $ssh "systemctl show $old_api_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
	echo "old API service: $old_api_service (port $old_api_port)"
else
	old_api_port=""
	echo "no old API service running"
fi

# Update local nginx snippet with new port
if [ -n "$local_nginx_snippet" ] && [ -f "$local_nginx_snippet" ]; then
	echo "updating local nginx snippet: $local_nginx_snippet"
	if [ -n "$old_port" ]; then
		echo "  changing port $old_port -> $port"
		sed -i.bak "s/${service_listen_address}:${old_port}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
	else
		echo "  setting port to $port"
		sed -i.bak "s/${service_listen_address}:[0-9]\{4,5\}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
	fi
	rm -f "${local_nginx_snippet}.bak"
	echo "nginx snippet updated locally"
else
	echo "warning: local_nginx_snippet not set or file not found"
fi

echo ""
echo "--- run these commands on $ssh ---"
echo ""
echo "# install and start new services"
echo "sudo ln -sf $base/$project/systemd/${service_name}.service /etc/systemd/system/"
echo "sudo ln -sf $base/$project/systemd/${api_service_name}.service /etc/systemd/system/"
echo "sudo systemctl daemon-reload"
echo "sudo systemctl start ${service_name}"
echo "sudo systemctl start ${api_service_name}"
echo ""
echo "# verify services are healthy"
echo "sudo systemctl status ${service_name}"
echo "sudo systemctl status ${api_service_name}"
echo "curl http://${service_listen_address}:$port/"
echo "curl http://${service_listen_address}:$api_port/models"
echo ""
echo "# then deploy your nginx configuration and reload nginx"
echo ""

if [ -n "$old_service" ]; then
	echo "# stop old frontend service"
	echo "sudo systemctl stop $old_service"
	echo ""
fi

if [ -n "$old_api_service" ]; then
	echo "# stop old API service"
	echo "sudo systemctl stop $old_api_service"
	echo ""
fi

echo "# update current symlink"
echo "ln -sfn releases/$release $base/$project/current"
echo ""
echo "--- end commands ---"
echo ""

printf "test health checks? [y/n] "
read ans
if [ "$ans" = "y" ]; then
	echo "testing frontend..."
	ssh $ssh "curl -v http://${service_listen_address}:$port/" || echo "frontend health check failed"
	echo "testing API..."
	ssh $ssh "curl -v http://${service_listen_address}:$api_port/models" || echo "API health check failed"
fi

if [ -n "$old_service" ]; then
	echo ""
	printf "stop old frontend service ($old_service)? [y/n] "
	read ans
	if [ "$ans" = "y" ]; then
		ssh $ssh "sudo systemctl stop $old_service"
		echo "old frontend service stopped"
	fi
fi

if [ -n "$old_api_service" ]; then
	echo ""
	printf "stop old API service ($old_api_service)? [y/n] "
	read ans
	if [ "$ans" = "y" ]; then
		ssh $ssh "sudo systemctl stop $old_api_service"
		echo "old API service stopped"
	fi
fi

echo ""
printf "update current symlink? [y/n] "
read ans
if [ "$ans" = "y" ]; then
	ssh $ssh "ln -sfn releases/$release $base/$project/current"
	echo "current -> $release"
fi

echo ""
echo "cleanup old releases (keep 3):"
old_releases=$(ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p'" || true)
if [ -n "$old_releases" ]; then
	echo "$old_releases"
	printf "remove these? [y/n] "
	read ans
	if [ "$ans" = "y" ]; then
		ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p' | while read r; do
			rm -rf \"\$r\"
			sudo systemctl stop ${project}-\${r} 2>/dev/null || true
			sudo systemctl stop ${project}-api-\${r} 2>/dev/null || true
			sudo rm -f /etc/systemd/system/${project}-\${r}.service
			sudo rm -f /etc/systemd/system/${project}-api-\${r}.service
			rm -f $base/$project/systemd/${project}-\${r}.service
			rm -f $base/$project/systemd/${project}-api-\${r}.service
		done"
		ssh $ssh "sudo systemctl daemon-reload"
		echo "cleanup done"
	fi
else
	echo "no old releases to clean"
fi

echo ""
echo "done: $release (frontend port $port, API port $api_port)"