118 lines
4 KiB
Bash
118 lines
4 KiB
Bash
|
#!/bin/sh
|
||
|
|
set -eu
|
||
|
|
set -o pipefail
|
||
|
|
|
||
|
|
# Deployment topology with Nomad
|
||
|
|
#
|
||
|
|
# Directory structure on remote:
|
||
|
|
# $base/$project/
|
||
|
|
# releases/{stamp}_{hash}/
|
||
|
|
# salience/ - python package
|
||
|
|
# transcript.txt - default text file
|
||
|
|
# pyproject.toml - dependencies
|
||
|
|
# .venv/ - virtual environment (created by uv)
|
||
|
|
# job.nomad.hcl - nomad job definition for this release
|
||
|
|
#
|
||
|
|
# Zero-downtime deployment with Nomad:
|
||
|
|
# 1. rsync new release (salience/ + pyproject.toml + transcript.txt)
|
||
|
|
# 2. uv sync dependencies
|
||
|
|
# 3. generate job file with release path
|
||
|
|
# 4. nomad job run (triggers blue-green deployment)
|
||
|
|
# 5. nomad waits for health checks to pass
|
||
|
|
# 6. nomad auto-promotes new allocation
|
||
|
|
# 7. old allocation enters graceful shutdown (30s kill_timeout)
|
||
|
|
# 8. consul-template updates nginx config (via service tags)
|
||
|
|
# 9. cleanup old releases (keep 5 most recent)
|
||
|
|
|
||
|
|
ssh=deploy-peoplesgrocers-website
|
||
|
|
base=/home/peoplesgrocers
|
||
|
|
project=salience-editor-api
|
||
|
|
|
||
|
|
#git diff-index --quiet HEAD || { echo 'git repo dirty'; exit 1; }
|
||
|
|
|
||
|
|
hash=$(git rev-parse --short=8 HEAD)
|
||
|
|
stamp=$(date +%Y-%b-%d-%a-%I_%M%p | tr 'APM' 'apm')
|
||
|
|
release="${stamp}-${hash}"
|
||
|
|
|
||
|
|
echo "deploying: $project @ $release"
|
||
|
|
printf "continue? [y/n] "
|
||
|
|
read ans
|
||
|
|
test "$ans" = "y" || exit 1
|
||
|
|
|
||
|
|
# prepare remote directories
|
||
|
|
ssh $ssh "mkdir -p $base/$project/releases/$release"
|
||
|
|
|
||
|
|
# sync all files using rclone (handles poor network connections better)
|
||
|
|
echo "syncing release files..."
|
||
|
|
temp_dir=$(mktemp -d)
|
||
|
|
trap "rm -rf $temp_dir" EXIT INT TERM
|
||
|
|
|
||
|
|
# Copy files to temp directory for single rclone transfer
|
||
|
|
rsync -a \
|
||
|
|
--exclude '__pycache__' \
|
||
|
|
--exclude '*.swp' \
|
||
|
|
salience pyproject.toml transcript.txt README.md \
|
||
|
|
"$temp_dir/"
|
||
|
|
test -f uv.lock && cp uv.lock "$temp_dir/"
|
||
|
|
rclone copy "$temp_dir/" "${ssh}:$base/$project/releases/$release/" \
|
||
|
|
--progress --retries 10 --checksum
|
||
|
|
|
||
|
|
rm -rf "$temp_dir"
|
||
|
|
echo "installing dependencies with uv..."
|
||
|
|
ssh $ssh "cd $base/$project/releases/$release && ~/.local/bin/uv sync --link-mode symlink"
|
||
|
|
|
||
|
|
# generate nomad job file with release path
|
||
|
|
echo "generating nomad job file..."
|
||
|
|
release_path="$base/$project/releases/$release"
|
||
|
|
job_file="$base/$project/releases/$release/job.nomad.hcl"
|
||
|
|
|
||
|
|
# Use envsubst with whitelist to only replace our variables, not Nomad runtime variables
|
||
|
|
export RELEASE_PLACEHOLDER="$release"
|
||
|
|
export RELEASE_PATH="$release_path"
|
||
|
|
envsubst '$RELEASE_PLACEHOLDER $RELEASE_PATH' < salience-editor-api.nomad.hcl | ssh $ssh "cat > $job_file"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "nomad job file created at: $job_file"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# submit job to nomad
|
||
|
|
echo "submitting job to nomad..."
|
||
|
|
deployment_id=$(ssh $ssh "source ~/.local/bin/env && nomad job run $job_file | grep -oE 'Deployment ID = [a-f0-9-]+' | awk '{print \$4}'" )
|
||
|
|
|
||
|
|
if [ -n "$deployment_id" ]; then
|
||
|
|
echo "deployment started: $deployment_id"
|
||
|
|
echo ""
|
||
|
|
echo "monitoring deployment..."
|
||
|
|
|
||
|
|
# Monitor deployment status
|
||
|
|
ssh $ssh "source ~/.local/bin/env && nomad deployment status $deployment_id"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
printf "watch deployment progress? [y/n] "
|
||
|
|
read ans
|
||
|
|
if [ "$ans" = "y" ]; then
|
||
|
|
ssh $ssh "source ~/.local/bin/env && watch -n 2 'nomad deployment status $deployment_id'"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo "warning: could not extract deployment ID"
|
||
|
|
echo "check deployment status manually with: nomad job status $project"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "done: $release"
|
||
|
|
echo ""
|
||
|
|
echo "Next steps:"
|
||
|
|
echo "- Nomad will automatically promote the deployment after health checks pass"
|
||
|
|
echo "- Consul-template will update nginx config based on healthy service instances"
|
||
|
|
echo "- Old allocation will gracefully shutdown (30s timeout for in-flight requests)"
|
||
|
|
echo "- Run ./cleanup-old-releases.sh to remove old releases (keeps 5 most recent)"
|
||
|
|
echo ""
|
||
|
|
if [ -n "$deployment_id" ]; then
|
||
|
|
echo "Monitor deployment:"
|
||
|
|
echo " nomad deployment status $deployment_id"
|
||
|
|
echo " watch -n 2 'nomad deployment status $deployment_id'"
|
||
|
|
echo " nomad job allocs $project"
|
||
|
|
echo ""
|
||
|
|
fi
|
||
|
|
echo "Check service health:"
|
||
|
|
echo " curl http://localhost:15500/v1/health/service/$project | jq"
|