feat: add NAS backup, verification, and DR scripts

scripts/disaster-recovery.sh

@@ -1,9 +1,16 @@
 #!/bin/bash
 # ==============================================================================
-# Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook (V2)
+# Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook (V3)
 # ==============================================================================
-# Packages app configs (.env, docker-compose), database backups, and NPM
-# configs, and exports them to LOCAL, NAS, or remote SCP storage.
+# Archives app configs (.env, docker-compose), database backup, and exports
+# to the NAS at 192.168.1.146.
+#
+# On RESTORE: deploys app on the new server and automatically updates NPM
+# (192.168.1.184) to reroute traffic via API — no manual intervention needed.
+#
+# Usage:
+#   ./disaster-recovery.sh --backup              # Create DR archive → NAS
+#   ./disaster-recovery.sh --restore <archive>  # Restore on THIS machine
 # ==============================================================================
 
 set -euo pipefail
@@ -31,49 +38,54 @@ if [ -f "${ENV_FILE}" ]; then
     set +a
 fi
 
-# Config Defaults & Type Resolution
-BACKUP_DEST_TYPE="${BACKUP_DEST_TYPE:-LOCAL}" # LOCAL, NAS, SCP
-BACKUP_DEST_PATH="${BACKUP_DEST_PATH:-${PROJECT_ROOT}/backups}"
-DR_RETENTION_DAYS=${DR_RETENTION_DAYS:-14}
+# NAS SSH (même config que backup-to-nas.sh)
+NAS_HOST="${NAS_HOST:-192.168.1.146}"
+NAS_USER="${NAS_USER:-wordly-backup}"
+NAS_PATH="${NAS_PATH:-/volume1/backups/wordly}"
+NAS_SSH_PORT="${NAS_SSH_PORT:-22}"
+NAS_SSH_KEY="${NAS_SSH_KEY:-/root/.ssh/wordly_nas_key}"
+BACKUP_DEST_PATH="${NAS_PATH}/snapshots"
+DR_RETENTION_DAYS=${DR_RETENTION_DAYS:-30}
 
-# SCP Configuration
-SCP_HOST="${SCP_HOST:-}"
-SCP_USER="${SCP_USER:-}"
-SCP_KEY_PATH="${SCP_KEY_PATH:-~/.ssh/id_rsa}"
-SCP_PORT="${SCP_PORT:-22}"
-SCP_DEST_PATH="${SCP_DEST_PATH:-/var/backups/wordly}"
+# IP of THIS server (used during restore to configure NPM failover)
+SERVER_IP="${SERVER_IP:-}"
 
-# NPM Configuration directories
-NPM_DATA_DIR="${NPM_DATA_DIR:-}"
-NPM_LETSENCRYPT_DIR="${NPM_LETSENCRYPT_DIR:-}"
+# Telegram
+TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
+TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID:-}"
 
 # ==============================================================================
-# DESTINATION PREPARATION
+# SEND TELEGRAM NOTIFICATION
+# ==============================================================================
+send_telegram() {
+    local message="$1"
+    if [ -n "${TELEGRAM_BOT_TOKEN}" ] && [ -n "${TELEGRAM_CHAT_ID}" ]; then
+        curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+            -d "chat_id=${TELEGRAM_CHAT_ID}" \
+            -d "text=${message}" \
+            -d "parse_mode=Markdown" \
+            >/dev/null 2>&1 || true
+    fi
+}
+
+# ==============================================================================
+# DESTINATION PREPARATION (backup mode)
 # ==============================================================================
 prepare_destination() {
-    if [ "${BACKUP_DEST_TYPE}" = "NAS" ] || [ "${BACKUP_DEST_TYPE}" = "LOCAL" ]; then
-        if [ ! -d "${BACKUP_DEST_PATH}" ]; then
-            mkdir -p "${BACKUP_DEST_PATH}" 2>/dev/null || true
-        fi
-        
-        if [ ! -w "${BACKUP_DEST_PATH}" ]; then
-            log_warning "Backup destination path '${BACKUP_DEST_PATH}' is not writable. Falling back to local backups."
-            BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
-            BACKUP_DEST_TYPE="LOCAL"
-            mkdir -p "${BACKUP_DEST_PATH}/dr"
-        fi
-        DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
-        mkdir -p "${DR_LOCAL_DIR}"
-    elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
-        if [ -z "${SCP_HOST}" ] || [ -z "${SCP_USER}" ]; then
-            log_error "SCP backup selected but SCP_HOST or SCP_USER is not configured in .env."
-            log_warning "Falling back to LOCAL backup directory."
-            BACKUP_DEST_TYPE="LOCAL"
-            BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
-            DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
-            mkdir -p "${DR_LOCAL_DIR}"
-        fi
+    local ssh_cmd="ssh -i ${NAS_SSH_KEY} -p ${NAS_SSH_PORT} -o BatchMode=yes -o ConnectTimeout=10"
+
+    log "Vérification de la connectivité SSH vers le NAS ${NAS_HOST}..."
+    if ! ${ssh_cmd} "${NAS_USER}@${NAS_HOST}" "echo OK" >/dev/null 2>&1; then
+        log_error "Impossible de joindre le NAS ${NAS_HOST} via SSH."
+        log_error "Lancez d'abord : sudo bash scripts/setup-nas.sh"
+        exit 1
     fi
+
+    # S'assurer que le dossier snapshots existe sur le NAS
+    ${ssh_cmd} "${NAS_USER}@${NAS_HOST}" \
+        "mkdir -p ${NAS_PATH}/snapshots" 2>/dev/null || true
+
+    log_success "NAS SSH OK — Destination : ${NAS_USER}@${NAS_HOST}:${NAS_PATH}/snapshots"
 }
 
 # ==============================================================================
@@ -127,22 +139,11 @@ perform_backup() {
     mkdir -p "${packing_dir}/db_backup"
     cp "${latest_db_backup}" "${packing_dir}/db_backup/"
 
-    # 5. Pack Nginx Proxy Manager (NPM) configs if configured
-    local has_npm_data=false
-    if [ -n "${NPM_DATA_DIR}" ] && [ -d "${NPM_DATA_DIR}" ]; then
-        log "Packaging Nginx Proxy Manager /data directory..."
-        cp -r "${NPM_DATA_DIR}" "${packing_dir}/npm_data"
-        has_npm_data=true
-    fi
-    if [ -n "${NPM_LETSENCRYPT_DIR}" ] && [ -d "${NPM_LETSENCRYPT_DIR}" ]; then
-        log "Packaging Nginx Proxy Manager /etc/letsencrypt directory..."
-        cp -r "${NPM_LETSENCRYPT_DIR}" "${packing_dir}/npm_letsencrypt"
-        has_npm_data=true
-    fi
-
-    if [ "${has_npm_data}" = "false" ]; then
-        log_warning "NPM directories (NPM_DATA_DIR / NPM_LETSENCRYPT_DIR) not configured or not found. Skipping NPM config packaging."
-    fi
+    # 5. Note: NPM config is NOT backed up here.
+    #    NPM runs on its own dedicated server (192.168.1.184) and is stable.
+    #    Only the forward_host IP needs to change during failover, which is
+    #    done automatically via the NPM API by npm-failover.sh during restore.
+    log "NPM is on dedicated server 192.168.1.184 — no NPM config to backup."
 
     # 6. Compress DR Archive
     local dr_archive_name="wordly_dr_${TIMESTAMP}.tar.gz"
@@ -160,44 +161,47 @@ perform_backup() {
     local size
     size=$(du -h "${local_archive_path}" | cut -f1)
 
-    # 7. Route to Destination
-    if [ "${BACKUP_DEST_TYPE}" = "LOCAL" ] || [ "${BACKUP_DEST_TYPE}" = "NAS" ]; then
-        local dest_path="${DR_LOCAL_DIR}/${dr_archive_name}"
-        mv "${local_archive_path}" "${dest_path}"
-        log_success "DR archive created successfully (${size}) at: ${dest_path}"
-        
-        # Retention
-        log "Applying retention policy (pruning files older than ${DR_RETENTION_DAYS} days)..."
-        find "${DR_LOCAL_DIR}" -name "wordly_dr_*.tar.gz" -mtime +"${DR_RETENTION_DAYS}" -exec rm -f {} \;
-        
-    elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
-        log "Transferring DR archive to remote server via SCP (${SCP_USER}@${SCP_HOST}:${SCP_PORT})..."
-        
-        # Test connection & Create remote directory if not exists
-        if ! ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" "mkdir -p ${SCP_DEST_PATH}" 2>/dev/null; then
-            log_error "SSH connection to ${SCP_USER}@${SCP_HOST} failed. Saving archive locally instead."
-            mkdir -p "${PROJECT_ROOT}/backups/dr"
-            mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
-            log_warning "DR backup saved locally at: ${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
-            exit 1
-        fi
+    # 7. Envoyer l'archive sur le NAS via rsync SSH
+    local ssh_cmd="ssh -i ${NAS_SSH_KEY} -p ${NAS_SSH_PORT} -o BatchMode=yes -o ConnectTimeout=30"
+    local dest_path="${BACKUP_DEST_PATH}/${dr_archive_name}"
 
-        # SCP copy
-        if scp -P "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${local_archive_path}" "${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"; then
-            log_success "DR archive transferred successfully to ${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"
-            rm -f "${local_archive_path}"
-            
-            # Remote retention prune
-            log "Applying remote retention policy on backup server..."
-            ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" \
-                "find ${SCP_DEST_PATH} -name 'wordly_dr_*.tar.gz' -mtime +${DR_RETENTION_DAYS} -exec rm -f {} \;" || true
-        else
-            log_error "SCP file transfer failed. Retaining local backup."
-            mkdir -p "${PROJECT_ROOT}/backups/dr"
-            mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
-        fi
+    log "Transfert de l'archive DR vers le NAS via rsync SSH..."
+    if ! rsync -az \
+        -e "ssh -i ${NAS_SSH_KEY} -p ${NAS_SSH_PORT} -o BatchMode=yes -o ConnectTimeout=30" \
+        "${local_archive_path}" \
+        "${NAS_USER}@${NAS_HOST}:${BACKUP_DEST_PATH}/${dr_archive_name}"; then
+        log_error "rsync SSH vers le NAS a échoué !"
+        log_warning "Archive conservée localement : ${local_archive_path}"
+        send_telegram "🚨 *Wordly DR Backup FAILED*
+rsync NAS échoué : ${NAS_HOST}
+Fichier local : ${local_archive_path}
+Date: $(date '+%Y-%m-%d %H:%M:%S')"
+        exit 1
     fi
 
+    rm -f "${local_archive_path}"
+    log_success "Archive DR transférée (${size}) → ${NAS_USER}@${NAS_HOST}:${dest_path}"
+
+    # Retention policy sur le NAS
+    log "Rotation des archives (>${DR_RETENTION_DAYS} jours) sur le NAS..."
+    ${ssh_cmd} "${NAS_USER}@${NAS_HOST}" \
+        "find ${BACKUP_DEST_PATH} -name 'wordly_dr_*.tar.gz' -mtime +${DR_RETENTION_DAYS} -delete 2>/dev/null; echo OK" | grep -q "OK" || true
+
+    # Sync scripts
+    if command -v rsync &>/dev/null; then
+        rsync -az \
+            -e "ssh -i ${NAS_SSH_KEY} -p ${NAS_SSH_PORT} -o BatchMode=yes" \
+            --exclude="__pycache__" \
+            "${SCRIPT_DIR}/" \
+            "${NAS_USER}@${NAS_HOST}:${NAS_PATH}/scripts/" 2>/dev/null || true
+    fi
+
+    send_telegram "✅ *Wordly.art DR Backup OK*
+Archive: \`${dr_archive_name}\`
+Taille: ${size}
+NAS: \`${dest_path}\`
+Date: $(date '+%Y-%m-%d %H:%M:%S')"
+
     log_success "Disaster Recovery backup complete."
 }
 
@@ -250,22 +254,7 @@ perform_restore() {
     source "${PROJECT_ROOT}/.env"
     set +a
 
-    # Restore NPM configs to their target directories if present in the package
-    if [ -d "${PROJECT_ROOT}/npm_data" ] && [ -n "${NPM_DATA_DIR}" ]; then
-        log "Restoring NPM /data directory..."
-        mkdir -p "$(dirname "${NPM_DATA_DIR}")"
-        rm -rf "${NPM_DATA_DIR}"
-        mv "${PROJECT_ROOT}/npm_data" "${NPM_DATA_DIR}"
-    fi
-
-    if [ -d "${PROJECT_ROOT}/npm_letsencrypt" ] && [ -n "${NPM_LETSENCRYPT_DIR}" ]; then
-        log "Restoring NPM /etc/letsencrypt directory..."
-        mkdir -p "$(dirname "${NPM_LETSENCRYPT_DIR}")"
-        rm -rf "${NPM_LETSENCRYPT_DIR}"
-        mv "${PROJECT_ROOT}/npm_letsencrypt" "${NPM_LETSENCRYPT_DIR}"
-    fi
-
-    log_success "Docker configurations, env keys, and NPM configurations restored."
+    log_success "Docker configurations and env keys restored."
 
     # Boot Docker Compose Services
     log "Spinning up Docker containers (database, redis, backend, frontend, NPM if configured)..."
@@ -328,10 +317,63 @@ perform_restore() {
     log "Restarting application backend..."
     ${compose_cmd} restart backend
 
+    # HTTP Health check (wait up to 3 minutes)
+    log "Waiting for application health check (max 180s)..."
+    local app_url="http://localhost:8001/health"
+    local health_ok=false
+    for i in $(seq 1 36); do
+        local http_code
+        http_code=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 --max-time 5 "${app_url}" 2>/dev/null || echo "000")
+        if [ "${http_code}" = "200" ]; then
+            health_ok=true
+            log_success "App is healthy (HTTP 200) after $((i * 5))s"
+            break
+        fi
+        echo "  Health check attempt ${i}/36... (HTTP ${http_code})"
+        sleep 5
+    done
+
+    if [ "${health_ok}" = "false" ]; then
+        log_error "App did NOT become healthy within 180s!"
+        log_error "NPM failover will NOT be triggered automatically."
+        log_error "Investigate: docker compose logs backend"
+        send_telegram "🚨 *Wordly.art DR FAILED — App unhealthy*
+Serveur: \`$(hostname -I | awk '{print $1}')\`
+Date: $(date '+%Y-%m-%d %H:%M:%S')
+Action: vérifiez les logs Docker"
+        exit 1
+    fi
+
+    # ==============================================================================
+    # NPM AUTOMATIC FAILOVER
+    # ==============================================================================
+    log "App is healthy. Triggering NPM failover..."
+    local this_server_ip
+    this_server_ip="${SERVER_IP:-$(hostname -I | awk '{print $1}')}"
+
+    if bash "${SCRIPT_DIR}/npm-failover.sh" --target-ip "${this_server_ip}"; then
+        log_success "NPM now routes traffic to this server (${this_server_ip})"
+        send_telegram "✅ *Wordly.art DR COMPLET*
+Serveur actif: \`${this_server_ip}\`
+NPM redirigé automatiquement
+Date: $(date '+%Y-%m-%d %H:%M:%S')"
+    else
+        log_error "NPM failover script FAILED."
+        log_warning "Manual failover required:"
+        log_warning "  → Go to http://192.168.1.184:81"
+        log_warning "  → Edit proxy host for ${NPM_PROXY_HOST_DOMAIN:-wordly.art}"
+        log_warning "  → Change Forward Hostname to: ${this_server_ip}"
+        send_telegram "⚠️ *Wordly.art DR — NPM manuel requis*
+App OK sur: \`${this_server_ip}\`
+NPM failover automatique a échoué
+Action: http://192.168.1.184:81 → modifier Forward Host"
+    fi
+
     log_success "=========================================================================="
     log_success "DISASTER RECOVERY SYSTEM RESTORE COMPLETE!"
     log_success "=========================================================================="
-    log "Your application and reverse-proxy routes are restored."
+    log_success "  App: http://${this_server_ip}:8001/health"
+    log_success "  NPM: http://192.168.1.184:81"
     echo ""
 }