301 lines
10 KiB
Bash
Executable file
301 lines
10 KiB
Bash
Executable file
#!/bin/sh
|
|
set -eu
|
|
|
|
# rsync wrapper for unreliable network connections
|
|
# my home network is spotty and rsync often dies mid-transfer
|
|
# this lets me manually retry the failed rsync and continue the deploy
|
|
rsync_retry() {
|
|
if ! rsync "$@"; then
|
|
echo ""
|
|
echo "rsync failed (probably network). run this manually:"
|
|
echo "rsync $*"
|
|
echo ""
|
|
printf "press enter after manual rsync completes to continue..."
|
|
read _
|
|
fi
|
|
}
|
|
|
|
# Deployment topology and zero-downtime process
|
|
#
|
|
# Directory structure on remote:
|
|
# $base/$project/
|
|
# base_port - starting port number (default 3100)
|
|
# releases/{stamp}_{hash}/
|
|
# dist/ - static assets served by nginx
|
|
# server/ - node server from frontend/server/entrypoint.express.js
|
|
# api/ - python flask API (salience/, nltk_data/, etc.)
|
|
# assigned_port - port allocated to frontend for this release
|
|
# assigned_api_port - port allocated to API for this release
|
|
# current -> releases/{latest}
|
|
# systemd/ - unit files per release (frontend + API)
|
|
#
|
|
# Zero-downtime deployment:
|
|
# 1. rsync new release (frontend/dist/ + frontend/server/ + api/)
|
|
# 2. install dependencies (npm for frontend, uv for API)
|
|
# 3. allocate two ports (frontend + API via get-next-port.sh)
|
|
# 4. generate systemd units for new release with unique ports
|
|
# - frontend: node server
|
|
# - API: gunicorn with 4 workers running Flask app
|
|
# 5. start new services, wait for health
|
|
# 6. update nginx upstream to point to new ports
|
|
# 7. reload nginx (graceful, no dropped connections)
|
|
# 8. stop old services
|
|
# 9. cleanup old releases (keep 3 most recent)
|
|
#
|
|
# Port allocation: get-next-port.sh reads base_port and existing
|
|
# assigned_port files to find first available port.
|
|
# Each release runs independently until cutover.
|
|
|
|
ssh=deploy-peoplesgrocers-website
|
|
base=/home/peoplesgrocers
|
|
project=salience
|
|
nginx_conf=/etc/nginx/sites-available/$project
|
|
service_listen_address=127.221.91.58
|
|
local_nginx_snippet="$HOME/src/work/infra/servers/chicago-web01/nginx/snippets/qwik-city-apps/salience.conf"
|
|
|
|
test -d frontend/dist || { echo 'no frontend/dist/'; exit 1; }
|
|
test -d frontend/server || { echo 'no frontend/server/'; exit 1; }
|
|
test -d .git || { echo 'not a git repo'; exit 1; }
|
|
git diff-index --quiet HEAD || { echo 'git repo dirty'; exit 1; }
|
|
|
|
hash=$(git rev-parse --short=8 HEAD)
|
|
stamp=$(date +%Y-%b-%d-%a-%I-%M%p | tr 'APM' 'apm')
|
|
release="${stamp}_${hash}"
|
|
service_name="${project}-${release}"
|
|
|
|
echo "deploying: $project @ $release"
|
|
printf "continue? [y/n] "
|
|
read ans
|
|
test "$ans" = "y" || exit 1
|
|
|
|
# prepare remote directories
|
|
ssh $ssh "mkdir -p $base/$project/{releases,systemd} $base/$project/releases/$release"
|
|
|
|
# sync both dist and server
|
|
echo "syncing dist..."
|
|
rsync_retry -tvaz frontend/dist/ $ssh:$base/$project/releases/$release/dist/
|
|
echo "syncing server..."
|
|
rsync_retry -tvaz frontend/server/ $ssh:$base/$project/releases/$release/server/
|
|
|
|
# copy server package.json and install dependencies
|
|
echo "copying server package.json..."
|
|
scp frontend/package.json $ssh:$base/$project/releases/$release/package.json
|
|
echo "installing server dependencies..."
|
|
ssh $ssh "source ~/.nvm/nvm.sh && cd $base/$project/releases/$release && npm install"
|
|
|
|
# sync api directory (exclude benchmarks, include specific files/dirs)
|
|
echo "syncing api..."
|
|
rsync_retry -tvaz \
|
|
--include='salience/' --include='salience/**' \
|
|
--include='nltk_data/' --include='nltk_data/**' \
|
|
--include='pyproject.toml' \
|
|
--include='uv.lock' \
|
|
--include='transcript.txt' \
|
|
--include='README.md' \
|
|
--exclude='*' \
|
|
api/ $ssh:$base/$project/releases/$release/api/
|
|
|
|
# link to shared models cache
|
|
echo "linking to shared models_cache..."
|
|
ssh $ssh "mkdir -p $base/$project/shared/models_cache"
|
|
ssh $ssh "ln -sfn ../../../shared/models_cache $base/$project/releases/$release/api/models_cache"
|
|
|
|
echo "installing api dependencies..."
|
|
ssh $ssh "cd $base/$project/releases/$release/api && ~/.local/bin/uv sync"
|
|
|
|
set -x
|
|
# determine ports for this release (frontend and api)
|
|
port=$(sh get-next-port.sh)
|
|
echo "frontend port for this release: $port"
|
|
api_port=$(sh get-next-port.sh)
|
|
echo "api port for this release: $api_port"
|
|
|
|
# record port assignments
|
|
ssh $ssh "echo $port > $base/$project/releases/$release/assigned_port"
|
|
ssh $ssh "echo $api_port > $base/$project/releases/$release/assigned_api_port"
|
|
|
|
set +x
|
|
|
|
# generate systemd unit file
|
|
unit_content="[Unit]
|
|
Description=${project} release ${release}
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=peoplesgrocers
|
|
WorkingDirectory=$base/$project/releases/$release
|
|
Environment=\"PORT=$port\"
|
|
Environment=\"ORIGIN=https://peoplesgrocers.com\"
|
|
ExecStart=/home/peoplesgrocers/.nvm/versions/node/v24.10.0/bin/node server/entry.express.js
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target"
|
|
|
|
echo "$unit_content" | ssh $ssh "cat > $base/$project/systemd/${service_name}.service"
|
|
|
|
echo ""
|
|
echo "systemd unit created at: $base/$project/systemd/${service_name}.service"
|
|
echo ""
|
|
|
|
# generate systemd unit file for API
|
|
api_service_name="${project}-api-${release}"
|
|
api_unit_content="[Unit]
|
|
Description=${project} API release ${release}
|
|
After=network.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=peoplesgrocers
|
|
WorkingDirectory=$base/$project/releases/$release/api
|
|
Environment=\"PORT=$api_port\"
|
|
ExecStart=$base/$project/releases/$release/api/.venv/bin/gunicorn --bind ${service_listen_address}:$api_port --workers 4 salience:app
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target"
|
|
|
|
echo "$api_unit_content" | ssh $ssh "cat > $base/$project/systemd/${api_service_name}.service"
|
|
|
|
echo ""
|
|
echo "API systemd unit created at: $base/$project/systemd/${api_service_name}.service"
|
|
echo ""
|
|
|
|
# find old services
|
|
old_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-' | grep -v 'api' | awk '{print \$1}' | head -1" || true)
|
|
if [ -n "$old_service" ]; then
|
|
old_port=$(ssh $ssh "systemctl show $old_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
|
|
echo "old frontend service: $old_service (port $old_port)"
|
|
else
|
|
old_port=""
|
|
echo "no old frontend service running"
|
|
fi
|
|
|
|
old_api_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-api-' | awk '{print \$1}' | head -1" || true)
|
|
if [ -n "$old_api_service" ]; then
|
|
old_api_port=$(ssh $ssh "systemctl show $old_api_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
|
|
echo "old API service: $old_api_service (port $old_api_port)"
|
|
else
|
|
old_api_port=""
|
|
echo "no old API service running"
|
|
fi
|
|
|
|
# Update local nginx snippet with new port
|
|
if [ -n "$local_nginx_snippet" ] && [ -f "$local_nginx_snippet" ]; then
|
|
echo "updating local nginx snippet: $local_nginx_snippet"
|
|
if [ -n "$old_port" ]; then
|
|
echo " changing port $old_port -> $port"
|
|
sed -i.bak "s/${service_listen_address}:${old_port}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
|
|
else
|
|
echo " setting port to $port"
|
|
sed -i.bak "s/${service_listen_address}:[0-9]\{4,5\}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
|
|
fi
|
|
rm -f "${local_nginx_snippet}.bak"
|
|
echo "nginx snippet updated locally"
|
|
else
|
|
echo "warning: local_nginx_snippet not set or file not found"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- run these commands on $ssh ---"
|
|
echo ""
|
|
echo "# install and start new services"
|
|
echo "sudo ln -sf $base/$project/systemd/${service_name}.service /etc/systemd/system/"
|
|
echo "sudo ln -sf $base/$project/systemd/${api_service_name}.service /etc/systemd/system/"
|
|
echo "sudo systemctl daemon-reload"
|
|
echo "sudo systemctl start ${service_name}"
|
|
echo "sudo systemctl start ${api_service_name}"
|
|
echo ""
|
|
echo "# verify services are healthy"
|
|
echo "sudo systemctl status ${service_name}"
|
|
echo "sudo systemctl status ${api_service_name}"
|
|
echo "curl http://${service_listen_address}:$port/"
|
|
echo "curl http://${service_listen_address}:$api_port/models"
|
|
echo ""
|
|
echo "# then deploy your nginx configuration and reload nginx"
|
|
echo ""
|
|
|
|
if [ -n "$old_service" ]; then
|
|
echo "# stop old frontend service"
|
|
echo "sudo systemctl stop $old_service"
|
|
echo ""
|
|
fi
|
|
|
|
if [ -n "$old_api_service" ]; then
|
|
echo "# stop old API service"
|
|
echo "sudo systemctl stop $old_api_service"
|
|
echo ""
|
|
fi
|
|
|
|
echo "# update current symlink"
|
|
echo "ln -sfn releases/$release $base/$project/current"
|
|
echo ""
|
|
echo "--- end commands ---"
|
|
echo ""
|
|
|
|
printf "test health checks? [y/n] "
|
|
read ans
|
|
if [ "$ans" = "y" ]; then
|
|
echo "testing frontend..."
|
|
ssh $ssh "curl -v http://${service_listen_address}:$port/" || echo "frontend health check failed"
|
|
echo "testing API..."
|
|
ssh $ssh "curl -v http://${service_listen_address}:$api_port/models" || echo "API health check failed"
|
|
fi
|
|
|
|
if [ -n "$old_service" ]; then
|
|
echo ""
|
|
printf "stop old frontend service ($old_service)? [y/n] "
|
|
read ans
|
|
if [ "$ans" = "y" ]; then
|
|
ssh $ssh "sudo systemctl stop $old_service"
|
|
echo "old frontend service stopped"
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$old_api_service" ]; then
|
|
echo ""
|
|
printf "stop old API service ($old_api_service)? [y/n] "
|
|
read ans
|
|
if [ "$ans" = "y" ]; then
|
|
ssh $ssh "sudo systemctl stop $old_api_service"
|
|
echo "old API service stopped"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
printf "update current symlink? [y/n] "
|
|
read ans
|
|
if [ "$ans" = "y" ]; then
|
|
ssh $ssh "ln -sfn releases/$release $base/$project/current"
|
|
echo "current -> $release"
|
|
fi
|
|
|
|
echo ""
|
|
echo "cleanup old releases (keep 3):"
|
|
old_releases=$(ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p'" || true)
|
|
if [ -n "$old_releases" ]; then
|
|
echo "$old_releases"
|
|
printf "remove these? [y/n] "
|
|
read ans
|
|
if [ "$ans" = "y" ]; then
|
|
ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p' | while read r; do
|
|
rm -rf \"\$r\"
|
|
sudo systemctl stop ${project}-\${r} 2>/dev/null || true
|
|
sudo systemctl stop ${project}-api-\${r} 2>/dev/null || true
|
|
sudo rm -f /etc/systemd/system/${project}-\${r}.service
|
|
sudo rm -f /etc/systemd/system/${project}-api-\${r}.service
|
|
rm -f $base/$project/systemd/${project}-\${r}.service
|
|
rm -f $base/$project/systemd/${project}-api-\${r}.service
|
|
done"
|
|
ssh $ssh "sudo systemctl daemon-reload"
|
|
echo "cleanup done"
|
|
fi
|
|
else
|
|
echo "no old releases to clean"
|
|
fi
|
|
|
|
echo ""
|
|
echo "done: $release (frontend port $port, API port $api_port)"
|