feat: create deployment scripts
This commit is contained in:
parent
78297efe5c
commit
8d5bce4bfb
GPG key ID:
D460CD54A9E3AB86
22 changed files with 2697 additions and 74 deletions
301
deploy.sh
Executable file
301
deploy.sh
Executable file
|
|
@ -0,0 +1,301 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# rsync wrapper for unreliable network connections
|
||||
# my home network is spotty and rsync often dies mid-transfer
|
||||
# this lets me manually retry the failed rsync and continue the deploy
|
||||
rsync_retry() {
|
||||
if ! rsync "$@"; then
|
||||
echo ""
|
||||
echo "rsync failed (probably network). run this manually:"
|
||||
echo "rsync $*"
|
||||
echo ""
|
||||
printf "press enter after manual rsync completes to continue..."
|
||||
read _
|
||||
fi
|
||||
}
|
||||
|
||||
# Deployment topology and zero-downtime process
|
||||
#
|
||||
# Directory structure on remote:
|
||||
# $base/$project/
|
||||
# base_port - starting port number (default 3100)
|
||||
# releases/{stamp}_{hash}/
|
||||
# dist/ - static assets served by nginx
|
||||
# server/ - node server from frontend/server/entrypoint.express.js
|
||||
# api/ - python flask API (salience/, nltk_data/, etc.)
|
||||
# assigned_port - port allocated to frontend for this release
|
||||
# assigned_api_port - port allocated to API for this release
|
||||
# current -> releases/{latest}
|
||||
# systemd/ - unit files per release (frontend + API)
|
||||
#
|
||||
# Zero-downtime deployment:
|
||||
# 1. rsync new release (frontend/dist/ + frontend/server/ + api/)
|
||||
# 2. install dependencies (npm for frontend, uv for API)
|
||||
# 3. allocate two ports (frontend + API via get-next-port.sh)
|
||||
# 4. generate systemd units for new release with unique ports
|
||||
# - frontend: node server
|
||||
# - API: gunicorn with 4 workers running Flask app
|
||||
# 5. start new services, wait for health
|
||||
# 6. update nginx upstream to point to new ports
|
||||
# 7. reload nginx (graceful, no dropped connections)
|
||||
# 8. stop old services
|
||||
# 9. cleanup old releases (keep 3 most recent)
|
||||
#
|
||||
# Port allocation: get-next-port.sh reads base_port and existing
|
||||
# assigned_port files to find first available port.
|
||||
# Each release runs independently until cutover.
|
||||
|
||||
ssh=deploy-peoplesgrocers-website
|
||||
base=/home/peoplesgrocers
|
||||
project=salience
|
||||
nginx_conf=/etc/nginx/sites-available/$project
|
||||
service_listen_address=127.10.143.212
|
||||
local_nginx_snippet="$HOME/src/work/infra/servers/chicago-web01/nginx/snippets/qwik-city-apps/salience.conf"
|
||||
|
||||
test -d frontend/dist || { echo 'no frontend/dist/'; exit 1; }
|
||||
test -d frontend/server || { echo 'no frontend/server/'; exit 1; }
|
||||
test -d .git || { echo 'not a git repo'; exit 1; }
|
||||
git diff-index --quiet HEAD || { echo 'git repo dirty'; exit 1; }
|
||||
|
||||
hash=$(git rev-parse --short=8 HEAD)
|
||||
stamp=$(date +%Y-%b-%d-%a-%I-%M%p | tr 'APM' 'apm')
|
||||
release="${stamp}_${hash}"
|
||||
service_name="${project}-${release}"
|
||||
|
||||
echo "deploying: $project @ $release"
|
||||
printf "continue? [y/n] "
|
||||
read ans
|
||||
test "$ans" = "y" || exit 1
|
||||
|
||||
# prepare remote directories
|
||||
ssh $ssh "mkdir -p $base/$project/{releases,systemd} $base/$project/releases/$release"
|
||||
|
||||
# sync both dist and server
|
||||
echo "syncing dist..."
|
||||
rsync_retry -tvaz frontend/dist/ $ssh:$base/$project/releases/$release/dist/
|
||||
echo "syncing server..."
|
||||
rsync_retry -tvaz frontend/server/ $ssh:$base/$project/releases/$release/server/
|
||||
|
||||
# copy server package.json and install dependencies
|
||||
echo "copying server package.json..."
|
||||
scp frontend/package.json $ssh:$base/$project/releases/$release/package.json
|
||||
echo "installing server dependencies..."
|
||||
ssh $ssh "source ~/.nvm/nvm.sh && cd $base/$project/releases/$release && npm install"
|
||||
|
||||
# sync api directory (exclude benchmarks, include specific files/dirs)
|
||||
echo "syncing api..."
|
||||
rsync_retry -tvaz \
|
||||
--include='salience/' --include='salience/**' \
|
||||
--include='nltk_data/' --include='nltk_data/**' \
|
||||
--include='pyproject.toml' \
|
||||
--include='uv.lock' \
|
||||
--include='transcript.txt' \
|
||||
--include='README.md' \
|
||||
--exclude='*' \
|
||||
api/ $ssh:$base/$project/releases/$release/api/
|
||||
|
||||
# link to shared models cache
|
||||
echo "linking to shared models_cache..."
|
||||
ssh $ssh "mkdir -p $base/$project/shared/models_cache"
|
||||
ssh $ssh "ln -sfn ../../../shared/models_cache $base/$project/releases/$release/api/models_cache"
|
||||
|
||||
echo "installing api dependencies..."
|
||||
ssh $ssh "cd $base/$project/releases/$release/api && ~/.local/bin/uv sync"
|
||||
|
||||
set -x
|
||||
# determine ports for this release (frontend and api)
|
||||
port=$(sh get-next-port.sh)
|
||||
echo "frontend port for this release: $port"
|
||||
api_port=$(sh get-next-port.sh)
|
||||
echo "api port for this release: $api_port"
|
||||
|
||||
# record port assignments
|
||||
ssh $ssh "echo $port > $base/$project/releases/$release/assigned_port"
|
||||
ssh $ssh "echo $api_port > $base/$project/releases/$release/assigned_api_port"
|
||||
|
||||
set +x
|
||||
|
||||
# generate systemd unit file
|
||||
unit_content="[Unit]
|
||||
Description=${project} release ${release}
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=peoplesgrocers
|
||||
WorkingDirectory=$base/$project/releases/$release
|
||||
Environment=\"PORT=$port\"
|
||||
Environment=\"ORIGIN=https://peoplesgrocers.com\"
|
||||
ExecStart=/home/peoplesgrocers/.nvm/versions/node/v24.10.0/bin/node server/entry.express.js
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target"
|
||||
|
||||
echo "$unit_content" | ssh $ssh "cat > $base/$project/systemd/${service_name}.service"
|
||||
|
||||
echo ""
|
||||
echo "systemd unit created at: $base/$project/systemd/${service_name}.service"
|
||||
echo ""
|
||||
|
||||
# generate systemd unit file for API
|
||||
api_service_name="${project}-api-${release}"
|
||||
api_unit_content="[Unit]
|
||||
Description=${project} API release ${release}
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=peoplesgrocers
|
||||
WorkingDirectory=$base/$project/releases/$release/api
|
||||
Environment=\"PORT=$api_port\"
|
||||
ExecStart=$base/$project/releases/$release/api/.venv/bin/gunicorn --bind ${service_listen_address}:$api_port --workers 4 salience:app
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target"
|
||||
|
||||
echo "$api_unit_content" | ssh $ssh "cat > $base/$project/systemd/${api_service_name}.service"
|
||||
|
||||
echo ""
|
||||
echo "API systemd unit created at: $base/$project/systemd/${api_service_name}.service"
|
||||
echo ""
|
||||
|
||||
# find old services
|
||||
old_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-' | grep -v 'api' | awk '{print \$1}' | head -1" || true)
|
||||
if [ -n "$old_service" ]; then
|
||||
old_port=$(ssh $ssh "systemctl show $old_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
|
||||
echo "old frontend service: $old_service (port $old_port)"
|
||||
else
|
||||
old_port=""
|
||||
echo "no old frontend service running"
|
||||
fi
|
||||
|
||||
old_api_service=$(ssh $ssh "systemctl list-units --type=service --state=running | grep '^${project}-api-' | awk '{print \$1}' | head -1" || true)
|
||||
if [ -n "$old_api_service" ]; then
|
||||
old_api_port=$(ssh $ssh "systemctl show $old_api_service --property=Environment" | sed -n 's/.*PORT=\([0-9]*\).*/\1/p')
|
||||
echo "old API service: $old_api_service (port $old_api_port)"
|
||||
else
|
||||
old_api_port=""
|
||||
echo "no old API service running"
|
||||
fi
|
||||
|
||||
# Update local nginx snippet with new port
|
||||
if [ -n "$local_nginx_snippet" ] && [ -f "$local_nginx_snippet" ]; then
|
||||
echo "updating local nginx snippet: $local_nginx_snippet"
|
||||
if [ -n "$old_port" ]; then
|
||||
echo " changing port $old_port -> $port"
|
||||
sed -i.bak "s/${service_listen_address}:${old_port}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
|
||||
else
|
||||
echo " setting port to $port"
|
||||
sed -i.bak "s/${service_listen_address}:[0-9]\{4,5\}/${service_listen_address}:${port}/g" "$local_nginx_snippet"
|
||||
fi
|
||||
rm -f "${local_nginx_snippet}.bak"
|
||||
echo "nginx snippet updated locally"
|
||||
else
|
||||
echo "warning: local_nginx_snippet not set or file not found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- run these commands on $ssh ---"
|
||||
echo ""
|
||||
echo "# install and start new services"
|
||||
echo "sudo ln -sf $base/$project/systemd/${service_name}.service /etc/systemd/system/"
|
||||
echo "sudo ln -sf $base/$project/systemd/${api_service_name}.service /etc/systemd/system/"
|
||||
echo "sudo systemctl daemon-reload"
|
||||
echo "sudo systemctl start ${service_name}"
|
||||
echo "sudo systemctl start ${api_service_name}"
|
||||
echo ""
|
||||
echo "# verify services are healthy"
|
||||
echo "sudo systemctl status ${service_name}"
|
||||
echo "sudo systemctl status ${api_service_name}"
|
||||
echo "curl http://${service_listen_address}:$port/"
|
||||
echo "curl http://${service_listen_address}:$api_port/models"
|
||||
echo ""
|
||||
echo "# then deploy your nginx configuration and reload nginx"
|
||||
echo ""
|
||||
|
||||
if [ -n "$old_service" ]; then
|
||||
echo "# stop old frontend service"
|
||||
echo "sudo systemctl stop $old_service"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ -n "$old_api_service" ]; then
|
||||
echo "# stop old API service"
|
||||
echo "sudo systemctl stop $old_api_service"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "# update current symlink"
|
||||
echo "ln -sfn releases/$release $base/$project/current"
|
||||
echo ""
|
||||
echo "--- end commands ---"
|
||||
echo ""
|
||||
|
||||
printf "test health checks? [y/n] "
|
||||
read ans
|
||||
if [ "$ans" = "y" ]; then
|
||||
echo "testing frontend..."
|
||||
ssh $ssh "curl -v http://${service_listen_address}:$port/" || echo "frontend health check failed"
|
||||
echo "testing API..."
|
||||
ssh $ssh "curl -v http://${service_listen_address}:$api_port/models" || echo "API health check failed"
|
||||
fi
|
||||
|
||||
if [ -n "$old_service" ]; then
|
||||
echo ""
|
||||
printf "stop old frontend service ($old_service)? [y/n] "
|
||||
read ans
|
||||
if [ "$ans" = "y" ]; then
|
||||
ssh $ssh "sudo systemctl stop $old_service"
|
||||
echo "old frontend service stopped"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$old_api_service" ]; then
|
||||
echo ""
|
||||
printf "stop old API service ($old_api_service)? [y/n] "
|
||||
read ans
|
||||
if [ "$ans" = "y" ]; then
|
||||
ssh $ssh "sudo systemctl stop $old_api_service"
|
||||
echo "old API service stopped"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
printf "update current symlink? [y/n] "
|
||||
read ans
|
||||
if [ "$ans" = "y" ]; then
|
||||
ssh $ssh "ln -sfn releases/$release $base/$project/current"
|
||||
echo "current -> $release"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "cleanup old releases (keep 3):"
|
||||
old_releases=$(ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p'" || true)
|
||||
if [ -n "$old_releases" ]; then
|
||||
echo "$old_releases"
|
||||
printf "remove these? [y/n] "
|
||||
read ans
|
||||
if [ "$ans" = "y" ]; then
|
||||
ssh $ssh "cd $base/$project/releases && ls -t | sed -n '4,\$p' | while read r; do
|
||||
rm -rf \"\$r\"
|
||||
sudo systemctl stop ${project}-\${r} 2>/dev/null || true
|
||||
sudo systemctl stop ${project}-api-\${r} 2>/dev/null || true
|
||||
sudo rm -f /etc/systemd/system/${project}-\${r}.service
|
||||
sudo rm -f /etc/systemd/system/${project}-api-\${r}.service
|
||||
rm -f $base/$project/systemd/${project}-\${r}.service
|
||||
rm -f $base/$project/systemd/${project}-api-\${r}.service
|
||||
done"
|
||||
ssh $ssh "sudo systemctl daemon-reload"
|
||||
echo "cleanup done"
|
||||
fi
|
||||
else
|
||||
echo "no old releases to clean"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "done: $release (frontend port $port, API port $api_port)"
|
||||
Loading…
Add table
Add a link
Reference in a new issue