CI/CD Pipeline Improvement - Add automated rollback on deployment failure and Telegram notifications for CI/deploy status. Changes: - scripts/deploy-prod.sh: Add rollback_save_image(), rollback_restore_image(), and telegram_notify() functions - scripts/deploy-prod.sh: Save current Docker image before building new one - scripts/deploy-prod.sh: Rollback to previous image on health check failure - .gitea/workflows/ci.yaml: Add Telegram notifications for CI failures - memento-note/eslint.config.mjs: Disable experimental React Compiler rules Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
189 lines
7.4 KiB
Bash
Executable File
189 lines
7.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Déploiement prod sur 192.168.1.190 — exécuter SUR LE SERVEUR (runner docker-host).
|
||
set -euo pipefail
|
||
|
||
# Telegram notification function
|
||
telegram_notify() {
|
||
local status="$1"
|
||
local message="$2"
|
||
local bot_token="${TELEGRAM_BOT_TOKEN:-}"
|
||
local chat_id="${TELEGRAM_CHAT_ID:-}"
|
||
|
||
if [ -z "$bot_token" ] || [ -z "$chat_id" ]; then
|
||
echo "TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID not set — skipping notification"
|
||
return 0
|
||
fi
|
||
|
||
local emoji="✅"
|
||
if [ "$status" = "failure" ]; then
|
||
emoji="❌"
|
||
elif [ "$status" = "rollback" ]; then
|
||
emoji="🔄"
|
||
fi
|
||
|
||
local commit_msg=$(git log -1 --pretty=format:"%s" 2>/dev/null || echo "unknown")
|
||
# Escape markdown special characters in commit message
|
||
commit_msg=$(echo "$commit_msg" | sed 's/[_*`[\]()~>#+=|{}.!\\-]/\\&/g')
|
||
local commit_short="${GIT_COMMIT:0:8}"
|
||
|
||
curl -s -X POST "https://api.telegram.org/bot${bot_token}/sendMessage" \
|
||
-d "chat_id=${chat_id}" \
|
||
-d "text=${emoji} Momento Deploy ${status}
|
||
Commit: ${commit_short}
|
||
Message: ${commit_msg}
|
||
Details: ${message}" \
|
||
-d "parse_mode=Markdown" > /dev/null 2>&1 || echo "Telegram notification failed"
|
||
}
|
||
|
||
# Rollback functions
|
||
rollback_save_image() {
|
||
echo "=== Saving current image for rollback ==="
|
||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^memento-memento-note:latest"; then
|
||
docker tag memento-memento-note:latest memento-memento-note:rollback || echo "WARN: Failed to tag rollback image"
|
||
echo "Rollback image saved: memento-memento-note:rollback"
|
||
else
|
||
echo "WARN: No current memento-note image found to save"
|
||
fi
|
||
}
|
||
|
||
rollback_restore_image() {
|
||
echo "=== Rolling back to previous image ==="
|
||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^memento-memento-note:rollback"; then
|
||
docker tag memento-memento-note:rollback memento-memento-note:latest || echo "ERROR: Failed to restore rollback image"
|
||
docker compose up -d --force-recreate memento-note || echo "ERROR: Failed to restart container"
|
||
echo "Rollback completed"
|
||
else
|
||
echo "ERROR: No rollback image found"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
ROOT="${DEPLOY_ROOT:-/opt/memento}"
|
||
ARTIFACT_TGZ="${ARTIFACT_TGZ:-}"
|
||
EXPECTED_COMMIT="${EXPECTED_COMMIT:-}"
|
||
|
||
# Health check configuration: 24 iterations × 5 seconds = 2 minutes total timeout
|
||
# This allows Next.js cold start time while keeping feedback fast
|
||
HEALTH_CHECK_MAX_ITERATIONS=24
|
||
HEALTH_CHECK_SLEEP_SECONDS=5
|
||
|
||
cd "$ROOT"
|
||
git config --global --add safe.directory "$ROOT" 2>/dev/null || true
|
||
git fetch origin main
|
||
git reset --hard origin/main
|
||
export GIT_COMMIT="$(git rev-parse HEAD)"
|
||
echo "=== Deploy $GIT_COMMIT ==="
|
||
|
||
if [ -n "$EXPECTED_COMMIT" ] && [ "$GIT_COMMIT" != "$EXPECTED_COMMIT" ]; then
|
||
echo "ERROR: git HEAD $GIT_COMMIT != expected $EXPECTED_COMMIT"
|
||
exit 1
|
||
fi
|
||
|
||
docker compose up -d postgres
|
||
for i in $(seq 1 30); do
|
||
docker compose exec -T postgres pg_isready -U "${POSTGRES_USER:-memento}" >/dev/null 2>&1 && break
|
||
[ "$i" -eq 30 ] && { echo "Postgres not ready"; exit 1; }
|
||
sleep 2
|
||
done
|
||
|
||
docker compose exec -T postgres psql -U "${POSTGRES_USER:-memento}" -d "${POSTGRES_DB:-memento}" -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null
|
||
|
||
if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q '^memento-note/prisma/migrations/'; then
|
||
DUMP_FILE="/opt/memento/backups/pre-migrate-$(date +%Y%m%d-%H%M%S).sql.gz"
|
||
mkdir -p /opt/memento/backups
|
||
docker compose exec -T postgres pg_dump -U "${POSTGRES_USER:-memento}" -d "${POSTGRES_DB:-memento}" --clean --if-exists | gzip > "$DUMP_FILE"
|
||
DUMP_SIZE=$(stat -c%s "$DUMP_FILE")
|
||
if [ "$DUMP_SIZE" -lt 1048576 ]; then
|
||
echo "ERROR: Dump too small ($DUMP_SIZE bytes)"
|
||
exit 1
|
||
fi
|
||
echo "Backup: $DUMP_FILE ($(( DUMP_SIZE / 1024 ))KB)"
|
||
else
|
||
echo "No migration change — skip pre-migrate dump"
|
||
fi
|
||
|
||
export COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1
|
||
export GIT_COMMIT
|
||
|
||
# Save current image for rollback before building new one
|
||
rollback_save_image
|
||
|
||
if [ -n "$ARTIFACT_TGZ" ] && [ -f "$ARTIFACT_TGZ" ]; then
|
||
echo "=== Fast image (CI artifact) ==="
|
||
tar xzf "$ARTIFACT_TGZ" -C memento-note/
|
||
export MEMENTO_DOCKERFILE=Dockerfile.prebuilt
|
||
export MEMENTO_SOCKET_DOCKERFILE=Dockerfile.socket.prebuilt
|
||
else
|
||
echo "=== Full docker build (no artifact) ==="
|
||
export MEMENTO_DOCKERFILE=Dockerfile
|
||
export MEMENTO_SOCKET_DOCKERFILE=Dockerfile.socket
|
||
fi
|
||
|
||
docker compose build memento-note
|
||
docker compose build memento-socket
|
||
|
||
if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q '^mcp-server/'; then
|
||
docker compose build mcp-server
|
||
fi
|
||
|
||
docker compose up -d --remove-orphans --force-recreate memento-note
|
||
docker compose up -d memento-socket
|
||
docker compose up -d mcp-server 2>/dev/null || true
|
||
|
||
# Redémarrer les exporters monitoring pour appliquer les configs à jour
|
||
if docker ps --format '{{.Names}}' | grep -q "^memento-grafana$"; then
|
||
echo "=== Updating monitoring exporters ==="
|
||
env $(cat /opt/memento/.env.docker | grep -v '^#' | xargs) \
|
||
docker compose -f monitoring/docker-compose.monitoring.yml up -d \
|
||
postgres-exporter cadvisor node-exporter redis-exporter 2>&1 || true
|
||
fi
|
||
|
||
echo "=== Migrations (Prisma CLI via node, pas npx) ==="
|
||
if docker compose exec -T memento-note test -f ./node_modules/prisma/build/index.js 2>/dev/null; then
|
||
docker compose exec -T memento-note node ./node_modules/prisma/build/index.js migrate deploy
|
||
else
|
||
echo "WARN: prisma CLI absent de l'image — les migrations tournent au démarrage (entrypoint) ou via psql manuel"
|
||
fi
|
||
|
||
nginx -t 2>/dev/null && systemctl reload nginx 2>/dev/null || true
|
||
|
||
for i in $(seq 1 "$HEALTH_CHECK_MAX_ITERATIONS"); do
|
||
BODY=$(docker compose exec -T memento-note node -e "require('http').get('http://localhost:3000/api/build-info',r=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>{console.log(d);process.exit(0)})}).on('error',()=>process.exit(1))" 2>/dev/null || true)
|
||
ACTUAL=$(echo "$BODY" | jq -r '.commit // empty' 2>/dev/null || true)
|
||
if [ "$ACTUAL" = "$GIT_COMMIT" ]; then
|
||
echo "OK build-info commit=$ACTUAL"
|
||
if docker ps --format '{{.Names}}' | grep -q "^memento-grafana$"; then
|
||
# Ne recréer Prometheus que si sa config a changé (préserve l'historique TSDB)
|
||
PROM_CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -E '^monitoring/(prometheus\.yml|alerts\.yml)' || true)
|
||
GRAFANA_CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -E '^monitoring/' || true)
|
||
if [ -n "$PROM_CHANGED" ]; then
|
||
echo "=== Prometheus config changed — recreating ==="
|
||
docker compose -f monitoring/docker-compose.monitoring.yml up -d --force-recreate prometheus
|
||
else
|
||
echo "=== Prometheus config unchanged — keeping TSDB history ==="
|
||
docker compose -f monitoring/docker-compose.monitoring.yml up -d prometheus
|
||
fi
|
||
if [ -n "$GRAFANA_CHANGED" ]; then
|
||
echo "=== Grafana config changed — recreating ==="
|
||
docker compose -f monitoring/docker-compose.monitoring.yml up -d --force-recreate grafana
|
||
else
|
||
echo "=== Grafana config unchanged — keeping state ==="
|
||
docker compose -f monitoring/docker-compose.monitoring.yml up -d grafana
|
||
fi
|
||
fi
|
||
docker compose ps
|
||
telegram_notify "success" "Deployment successful — app is healthy"
|
||
exit 0
|
||
fi
|
||
sleep "$HEALTH_CHECK_SLEEP_SECONDS"
|
||
done
|
||
|
||
echo "ERROR: build-info=$ACTUAL expected=$GIT_COMMIT"
|
||
docker logs memento-web --tail=40
|
||
|
||
# Rollback on health check failure
|
||
rollback_restore_image
|
||
telegram_notify "rollback" "Health check failed — rolled back to previous version"
|
||
|
||
exit 1
|