#!/bin/bash # ============================================================================== # Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook (V2) # ============================================================================== # Packages app configs (.env, docker-compose), database backups, and NPM # configs, and exports them to LOCAL, NAS, or remote SCP storage. # ============================================================================== set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "[DR ${TIMESTAMP}] $1"; } log_success() { echo -e "[DR ${TIMESTAMP}] ${GREEN}$1${NC}"; } log_warning() { echo -e "[DR ${TIMESTAMP}] ${YELLOW}WARNING: $1${NC}"; } log_error() { echo -e "[DR ${TIMESTAMP}] ${RED}ERROR: $1${NC}"; } # Sourcing .env ENV_FILE="${PROJECT_ROOT}/.env" if [ -f "${ENV_FILE}" ]; then set -a source "${ENV_FILE}" set +a fi # Config Defaults & Type Resolution BACKUP_DEST_TYPE="${BACKUP_DEST_TYPE:-LOCAL}" # LOCAL, NAS, SCP BACKUP_DEST_PATH="${BACKUP_DEST_PATH:-${PROJECT_ROOT}/backups}" DR_RETENTION_DAYS=${DR_RETENTION_DAYS:-14} # SCP Configuration SCP_HOST="${SCP_HOST:-}" SCP_USER="${SCP_USER:-}" SCP_KEY_PATH="${SCP_KEY_PATH:-~/.ssh/id_rsa}" SCP_PORT="${SCP_PORT:-22}" SCP_DEST_PATH="${SCP_DEST_PATH:-/var/backups/wordly}" # NPM Configuration directories NPM_DATA_DIR="${NPM_DATA_DIR:-}" NPM_LETSENCRYPT_DIR="${NPM_LETSENCRYPT_DIR:-}" # ============================================================================== # DESTINATION PREPARATION # ============================================================================== prepare_destination() { if [ "${BACKUP_DEST_TYPE}" = "NAS" ] || [ "${BACKUP_DEST_TYPE}" = "LOCAL" ]; then if [ ! -d "${BACKUP_DEST_PATH}" ]; then mkdir -p "${BACKUP_DEST_PATH}" 2>/dev/null || true fi if [ ! -w "${BACKUP_DEST_PATH}" ]; then log_warning "Backup destination path '${BACKUP_DEST_PATH}' is not writable. Falling back to local backups." BACKUP_DEST_PATH="${PROJECT_ROOT}/backups" BACKUP_DEST_TYPE="LOCAL" mkdir -p "${BACKUP_DEST_PATH}/dr" fi DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr" mkdir -p "${DR_LOCAL_DIR}" elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then if [ -z "${SCP_HOST}" ] || [ -z "${SCP_USER}" ]; then log_error "SCP backup selected but SCP_HOST or SCP_USER is not configured in .env." log_warning "Falling back to LOCAL backup directory." BACKUP_DEST_TYPE="LOCAL" BACKUP_DEST_PATH="${PROJECT_ROOT}/backups" DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr" mkdir -p "${DR_LOCAL_DIR}" fi fi } # ============================================================================== # BACKUP ACTION # ============================================================================== perform_backup() { prepare_destination log "Starting Disaster Recovery backup (Destination Mode: ${BACKUP_DEST_TYPE})..." # 1. Trigger DB Backup log "Triggering database dump..." if ! bash "${SCRIPT_DIR}/backup-database.sh" --full; then log_error "Database backup failed. Aborting DR packaging." exit 1 fi # 2. Locate DB Backup file local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}" local latest_db_backup latest_db_backup=$(ls -t "${local_backup_dir}/daily/"*.gz 2>/dev/null | head -n 1 || true) if [ -z "${latest_db_backup}" ]; then log_error "Could not find database backup file." exit 1 fi log "Database backup file loaded: $(basename "${latest_db_backup}")" # 3. Create temp packaging folder local packing_dir="${PROJECT_ROOT}/temp_dr_pack_${TIMESTAMP}" mkdir -p "${packing_dir}" # 4. Pack Configurations log "Packing application configuration (.env & docker-compose)..." if [ -f "${PROJECT_ROOT}/.env" ]; then cp "${PROJECT_ROOT}/.env" "${packing_dir}/.env.production" fi for f in docker-compose.yml docker-compose.local.yml docker-compose.monitoring.yml docker-compose.dev.yml; do if [ -f "${PROJECT_ROOT}/${f}" ]; then cp "${PROJECT_ROOT}/${f}" "${packing_dir}/" fi done if [ -d "${PROJECT_ROOT}/docker" ]; then cp -r "${PROJECT_ROOT}/docker" "${packing_dir}/" fi if [ -d "${PROJECT_ROOT}/scripts" ]; then cp -r "${PROJECT_ROOT}/scripts" "${packing_dir}/" fi mkdir -p "${packing_dir}/db_backup" cp "${latest_db_backup}" "${packing_dir}/db_backup/" # 5. Pack Nginx Proxy Manager (NPM) configs if configured local has_npm_data=false if [ -n "${NPM_DATA_DIR}" ] && [ -d "${NPM_DATA_DIR}" ]; then log "Packaging Nginx Proxy Manager /data directory..." cp -r "${NPM_DATA_DIR}" "${packing_dir}/npm_data" has_npm_data=true fi if [ -n "${NPM_LETSENCRYPT_DIR}" ] && [ -d "${NPM_LETSENCRYPT_DIR}" ]; then log "Packaging Nginx Proxy Manager /etc/letsencrypt directory..." cp -r "${NPM_LETSENCRYPT_DIR}" "${packing_dir}/npm_letsencrypt" has_npm_data=true fi if [ "${has_npm_data}" = "false" ]; then log_warning "NPM directories (NPM_DATA_DIR / NPM_LETSENCRYPT_DIR) not configured or not found. Skipping NPM config packaging." fi # 6. Compress DR Archive local dr_archive_name="wordly_dr_${TIMESTAMP}.tar.gz" local local_archive_path="${PROJECT_ROOT}/${dr_archive_name}" log "Compressing configurations, database, and NPM data into DR archive..." tar -czf "${local_archive_path}" -C "${packing_dir}" . rm -rf "${packing_dir}" if [ ! -f "${local_archive_path}" ] || [ ! -s "${local_archive_path}" ]; then log_error "Failed to compress archive." exit 1 fi local size size=$(du -h "${local_archive_path}" | cut -f1) # 7. Route to Destination if [ "${BACKUP_DEST_TYPE}" = "LOCAL" ] || [ "${BACKUP_DEST_TYPE}" = "NAS" ]; then local dest_path="${DR_LOCAL_DIR}/${dr_archive_name}" mv "${local_archive_path}" "${dest_path}" log_success "DR archive created successfully (${size}) at: ${dest_path}" # Retention log "Applying retention policy (pruning files older than ${DR_RETENTION_DAYS} days)..." find "${DR_LOCAL_DIR}" -name "wordly_dr_*.tar.gz" -mtime +"${DR_RETENTION_DAYS}" -exec rm -f {} \; elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then log "Transferring DR archive to remote server via SCP (${SCP_USER}@${SCP_HOST}:${SCP_PORT})..." # Test connection & Create remote directory if not exists if ! ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" "mkdir -p ${SCP_DEST_PATH}" 2>/dev/null; then log_error "SSH connection to ${SCP_USER}@${SCP_HOST} failed. Saving archive locally instead." mkdir -p "${PROJECT_ROOT}/backups/dr" mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}" log_warning "DR backup saved locally at: ${PROJECT_ROOT}/backups/dr/${dr_archive_name}" exit 1 fi # SCP copy if scp -P "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${local_archive_path}" "${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"; then log_success "DR archive transferred successfully to ${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}" rm -f "${local_archive_path}" # Remote retention prune log "Applying remote retention policy on backup server..." ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" \ "find ${SCP_DEST_PATH} -name 'wordly_dr_*.tar.gz' -mtime +${DR_RETENTION_DAYS} -exec rm -f {} \;" || true else log_error "SCP file transfer failed. Retaining local backup." mkdir -p "${PROJECT_ROOT}/backups/dr" mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}" fi fi log_success "Disaster Recovery backup complete." } # ============================================================================== # RESTORE ACTION # ============================================================================== perform_restore() { local dr_package="$1" if [ -z "${dr_package}" ]; then log_error "No DR package archive specified." echo "Usage: $0 --restore " exit 1 fi if [ ! -f "${dr_package}" ]; then log_error "DR Archive file not found: ${dr_package}" exit 1 fi echo "" log_warning "RESTORE DISASTER RECOVERY PACKAGE - THIS WILL OVERWRITE ENVIRONMENT CONFIGURATIONS, DATABASES, AND NPM FILES!" echo " Archive: ${dr_package}" echo "" read -p "Type 'RESTORE-ALL' to confirm complete system restore: " confirm_val if [ "${confirm_val}" != "RESTORE-ALL" ]; then log "System restore cancelled." exit 0 fi log "Extracting DR archive contents..." # Safety backup of existing .env if [ -f "${PROJECT_ROOT}/.env" ]; then cp "${PROJECT_ROOT}/.env" "${PROJECT_ROOT}/.env.bak_before_dr_restore_${TIMESTAMP}" log "Created backup of existing .env: .env.bak_before_dr_restore_${TIMESTAMP}" fi # Extract all tar -xzf "${dr_package}" -C "${PROJECT_ROOT}" # Restore .env if [ -f "${PROJECT_ROOT}/.env.production" ]; then mv "${PROJECT_ROOT}/.env.production" "${PROJECT_ROOT}/.env" log "Restored .env configuration" fi # Reload variables from restored .env set -a source "${PROJECT_ROOT}/.env" set +a # Restore NPM configs to their target directories if present in the package if [ -d "${PROJECT_ROOT}/npm_data" ] && [ -n "${NPM_DATA_DIR}" ]; then log "Restoring NPM /data directory..." mkdir -p "$(dirname "${NPM_DATA_DIR}")" rm -rf "${NPM_DATA_DIR}" mv "${PROJECT_ROOT}/npm_data" "${NPM_DATA_DIR}" fi if [ -d "${PROJECT_ROOT}/npm_letsencrypt" ] && [ -n "${NPM_LETSENCRYPT_DIR}" ]; then log "Restoring NPM /etc/letsencrypt directory..." mkdir -p "$(dirname "${NPM_LETSENCRYPT_DIR}")" rm -rf "${NPM_LETSENCRYPT_DIR}" mv "${PROJECT_ROOT}/npm_letsencrypt" "${NPM_LETSENCRYPT_DIR}" fi log_success "Docker configurations, env keys, and NPM configurations restored." # Boot Docker Compose Services log "Spinning up Docker containers (database, redis, backend, frontend, NPM if configured)..." local compose_cmd="docker compose" if ! docker compose version &>/dev/null; then compose_cmd="docker-compose" fi ${compose_cmd} up -d # Locate the embedded database backup local db_backup_archive db_backup_archive=$(ls "${PROJECT_ROOT}/db_backup/"*.gz 2>/dev/null | head -n 1 || true) if [ -z "${db_backup_archive}" ]; then log_error "Database backup archive not found inside the DR package extraction." exit 1 fi log "Database backup located: $(basename "${db_backup_archive}")" # Wait for database container to be healthy (PostgreSQL) local db_type="sqlite" if [[ "${DATABASE_URL:-}" =~ ^postgres ]]; then db_type="postgres" fi if [ "${db_type}" = "postgres" ]; then local postgres_container="${POSTGRES_CONTAINER:-wordly-postgres}" log "Waiting for PostgreSQL container (${postgres_container}) to be healthy..." for i in $(seq 1 30); do if docker inspect --format='{{.State.Health.Status}}' "${postgres_container}" 2>/dev/null | grep -q "healthy"; then log_success "Database container is healthy." break fi echo " Waiting for database... ($i/30)" sleep 2 done else sleep 2 fi # Restore the database using the database backup script log "Triggering database restore..." local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}" mkdir -p "${local_backup_dir}/daily" cp "${db_backup_archive}" "${local_backup_dir}/daily/" local db_archive_filename db_archive_filename=$(basename "${db_backup_archive}") # Run DB restore log "Restoring DB contents..." bash "${SCRIPT_DIR}/backup-database.sh" --restore "${db_archive_filename}" # Clean up extracted temporary folder rm -rf "${PROJECT_ROOT}/db_backup" # Restart app to clear connection caches log "Restarting application backend..." ${compose_cmd} restart backend log_success "==========================================================================" log_success "DISASTER RECOVERY SYSTEM RESTORE COMPLETE!" log_success "==========================================================================" log "Your application and reverse-proxy routes are restored." echo "" } # ============================================================================== # MAIN ENTRY # ============================================================================== main() { case "${1:-}" in --backup) perform_backup ;; --restore) perform_restore "${2:-}" ;; *) echo "Wordly Disaster Recovery Utility (V2)" echo "Usage:" echo " $0 --backup # Package configs, db dump, NPM configurations, and export" echo " $0 --restore # Extract and restore full stack on new machine" exit 1 ;; esac } main "$@"