360 lines
14 KiB
Bash
Executable File
360 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================
|
|
# Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook (V2)
|
|
# ==============================================================================
|
|
# Packages app configs (.env, docker-compose), database backups, and NPM
|
|
# configs, and exports them to LOCAL, NAS, or remote SCP storage.
|
|
# ==============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
log() { echo -e "[DR ${TIMESTAMP}] $1"; }
|
|
log_success() { echo -e "[DR ${TIMESTAMP}] ${GREEN}$1${NC}"; }
|
|
log_warning() { echo -e "[DR ${TIMESTAMP}] ${YELLOW}WARNING: $1${NC}"; }
|
|
log_error() { echo -e "[DR ${TIMESTAMP}] ${RED}ERROR: $1${NC}"; }
|
|
|
|
# Sourcing .env
|
|
ENV_FILE="${PROJECT_ROOT}/.env"
|
|
if [ -f "${ENV_FILE}" ]; then
|
|
set -a
|
|
source "${ENV_FILE}"
|
|
set +a
|
|
fi
|
|
|
|
# Config Defaults & Type Resolution
|
|
BACKUP_DEST_TYPE="${BACKUP_DEST_TYPE:-LOCAL}" # LOCAL, NAS, SCP
|
|
BACKUP_DEST_PATH="${BACKUP_DEST_PATH:-${PROJECT_ROOT}/backups}"
|
|
DR_RETENTION_DAYS=${DR_RETENTION_DAYS:-14}
|
|
|
|
# SCP Configuration
|
|
SCP_HOST="${SCP_HOST:-}"
|
|
SCP_USER="${SCP_USER:-}"
|
|
SCP_KEY_PATH="${SCP_KEY_PATH:-~/.ssh/id_rsa}"
|
|
SCP_PORT="${SCP_PORT:-22}"
|
|
SCP_DEST_PATH="${SCP_DEST_PATH:-/var/backups/wordly}"
|
|
|
|
# NPM Configuration directories
|
|
NPM_DATA_DIR="${NPM_DATA_DIR:-}"
|
|
NPM_LETSENCRYPT_DIR="${NPM_LETSENCRYPT_DIR:-}"
|
|
|
|
# ==============================================================================
|
|
# DESTINATION PREPARATION
|
|
# ==============================================================================
|
|
prepare_destination() {
|
|
if [ "${BACKUP_DEST_TYPE}" = "NAS" ] || [ "${BACKUP_DEST_TYPE}" = "LOCAL" ]; then
|
|
if [ ! -d "${BACKUP_DEST_PATH}" ]; then
|
|
mkdir -p "${BACKUP_DEST_PATH}" 2>/dev/null || true
|
|
fi
|
|
|
|
if [ ! -w "${BACKUP_DEST_PATH}" ]; then
|
|
log_warning "Backup destination path '${BACKUP_DEST_PATH}' is not writable. Falling back to local backups."
|
|
BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
|
|
BACKUP_DEST_TYPE="LOCAL"
|
|
mkdir -p "${BACKUP_DEST_PATH}/dr"
|
|
fi
|
|
DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
|
|
mkdir -p "${DR_LOCAL_DIR}"
|
|
elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
|
|
if [ -z "${SCP_HOST}" ] || [ -z "${SCP_USER}" ]; then
|
|
log_error "SCP backup selected but SCP_HOST or SCP_USER is not configured in .env."
|
|
log_warning "Falling back to LOCAL backup directory."
|
|
BACKUP_DEST_TYPE="LOCAL"
|
|
BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
|
|
DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
|
|
mkdir -p "${DR_LOCAL_DIR}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================
|
|
# BACKUP ACTION
|
|
# ==============================================================================
|
|
perform_backup() {
|
|
prepare_destination
|
|
log "Starting Disaster Recovery backup (Destination Mode: ${BACKUP_DEST_TYPE})..."
|
|
|
|
# 1. Trigger DB Backup
|
|
log "Triggering database dump..."
|
|
if ! bash "${SCRIPT_DIR}/backup-database.sh" --full; then
|
|
log_error "Database backup failed. Aborting DR packaging."
|
|
exit 1
|
|
fi
|
|
|
|
# 2. Locate DB Backup file
|
|
local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}"
|
|
local latest_db_backup
|
|
latest_db_backup=$(ls -t "${local_backup_dir}/daily/"*.gz 2>/dev/null | head -n 1 || true)
|
|
|
|
if [ -z "${latest_db_backup}" ]; then
|
|
log_error "Could not find database backup file."
|
|
exit 1
|
|
fi
|
|
log "Database backup file loaded: $(basename "${latest_db_backup}")"
|
|
|
|
# 3. Create temp packaging folder
|
|
local packing_dir="${PROJECT_ROOT}/temp_dr_pack_${TIMESTAMP}"
|
|
mkdir -p "${packing_dir}"
|
|
|
|
# 4. Pack Configurations
|
|
log "Packing application configuration (.env & docker-compose)..."
|
|
if [ -f "${PROJECT_ROOT}/.env" ]; then
|
|
cp "${PROJECT_ROOT}/.env" "${packing_dir}/.env.production"
|
|
fi
|
|
|
|
for f in docker-compose.yml docker-compose.local.yml docker-compose.monitoring.yml docker-compose.dev.yml; do
|
|
if [ -f "${PROJECT_ROOT}/${f}" ]; then
|
|
cp "${PROJECT_ROOT}/${f}" "${packing_dir}/"
|
|
fi
|
|
done
|
|
|
|
if [ -d "${PROJECT_ROOT}/docker" ]; then
|
|
cp -r "${PROJECT_ROOT}/docker" "${packing_dir}/"
|
|
fi
|
|
if [ -d "${PROJECT_ROOT}/scripts" ]; then
|
|
cp -r "${PROJECT_ROOT}/scripts" "${packing_dir}/"
|
|
fi
|
|
|
|
mkdir -p "${packing_dir}/db_backup"
|
|
cp "${latest_db_backup}" "${packing_dir}/db_backup/"
|
|
|
|
# 5. Pack Nginx Proxy Manager (NPM) configs if configured
|
|
local has_npm_data=false
|
|
if [ -n "${NPM_DATA_DIR}" ] && [ -d "${NPM_DATA_DIR}" ]; then
|
|
log "Packaging Nginx Proxy Manager /data directory..."
|
|
cp -r "${NPM_DATA_DIR}" "${packing_dir}/npm_data"
|
|
has_npm_data=true
|
|
fi
|
|
if [ -n "${NPM_LETSENCRYPT_DIR}" ] && [ -d "${NPM_LETSENCRYPT_DIR}" ]; then
|
|
log "Packaging Nginx Proxy Manager /etc/letsencrypt directory..."
|
|
cp -r "${NPM_LETSENCRYPT_DIR}" "${packing_dir}/npm_letsencrypt"
|
|
has_npm_data=true
|
|
fi
|
|
|
|
if [ "${has_npm_data}" = "false" ]; then
|
|
log_warning "NPM directories (NPM_DATA_DIR / NPM_LETSENCRYPT_DIR) not configured or not found. Skipping NPM config packaging."
|
|
fi
|
|
|
|
# 6. Compress DR Archive
|
|
local dr_archive_name="wordly_dr_${TIMESTAMP}.tar.gz"
|
|
local local_archive_path="${PROJECT_ROOT}/${dr_archive_name}"
|
|
|
|
log "Compressing configurations, database, and NPM data into DR archive..."
|
|
tar -czf "${local_archive_path}" -C "${packing_dir}" .
|
|
rm -rf "${packing_dir}"
|
|
|
|
if [ ! -f "${local_archive_path}" ] || [ ! -s "${local_archive_path}" ]; then
|
|
log_error "Failed to compress archive."
|
|
exit 1
|
|
fi
|
|
|
|
local size
|
|
size=$(du -h "${local_archive_path}" | cut -f1)
|
|
|
|
# 7. Route to Destination
|
|
if [ "${BACKUP_DEST_TYPE}" = "LOCAL" ] || [ "${BACKUP_DEST_TYPE}" = "NAS" ]; then
|
|
local dest_path="${DR_LOCAL_DIR}/${dr_archive_name}"
|
|
mv "${local_archive_path}" "${dest_path}"
|
|
log_success "DR archive created successfully (${size}) at: ${dest_path}"
|
|
|
|
# Retention
|
|
log "Applying retention policy (pruning files older than ${DR_RETENTION_DAYS} days)..."
|
|
find "${DR_LOCAL_DIR}" -name "wordly_dr_*.tar.gz" -mtime +"${DR_RETENTION_DAYS}" -exec rm -f {} \;
|
|
|
|
elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
|
|
log "Transferring DR archive to remote server via SCP (${SCP_USER}@${SCP_HOST}:${SCP_PORT})..."
|
|
|
|
# Test connection & Create remote directory if not exists
|
|
if ! ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" "mkdir -p ${SCP_DEST_PATH}" 2>/dev/null; then
|
|
log_error "SSH connection to ${SCP_USER}@${SCP_HOST} failed. Saving archive locally instead."
|
|
mkdir -p "${PROJECT_ROOT}/backups/dr"
|
|
mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
|
|
log_warning "DR backup saved locally at: ${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
|
|
exit 1
|
|
fi
|
|
|
|
# SCP copy
|
|
if scp -P "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${local_archive_path}" "${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"; then
|
|
log_success "DR archive transferred successfully to ${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"
|
|
rm -f "${local_archive_path}"
|
|
|
|
# Remote retention prune
|
|
log "Applying remote retention policy on backup server..."
|
|
ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" \
|
|
"find ${SCP_DEST_PATH} -name 'wordly_dr_*.tar.gz' -mtime +${DR_RETENTION_DAYS} -exec rm -f {} \;" || true
|
|
else
|
|
log_error "SCP file transfer failed. Retaining local backup."
|
|
mkdir -p "${PROJECT_ROOT}/backups/dr"
|
|
mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
|
|
fi
|
|
fi
|
|
|
|
log_success "Disaster Recovery backup complete."
|
|
}
|
|
|
|
# ==============================================================================
|
|
# RESTORE ACTION
|
|
# ==============================================================================
|
|
perform_restore() {
|
|
local dr_package="$1"
|
|
|
|
if [ -z "${dr_package}" ]; then
|
|
log_error "No DR package archive specified."
|
|
echo "Usage: $0 --restore <path_to_archive.tar.gz>"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "${dr_package}" ]; then
|
|
log_error "DR Archive file not found: ${dr_package}"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
log_warning "RESTORE DISASTER RECOVERY PACKAGE - THIS WILL OVERWRITE ENVIRONMENT CONFIGURATIONS, DATABASES, AND NPM FILES!"
|
|
echo " Archive: ${dr_package}"
|
|
echo ""
|
|
read -p "Type 'RESTORE-ALL' to confirm complete system restore: " confirm_val
|
|
if [ "${confirm_val}" != "RESTORE-ALL" ]; then
|
|
log "System restore cancelled."
|
|
exit 0
|
|
fi
|
|
|
|
log "Extracting DR archive contents..."
|
|
|
|
# Safety backup of existing .env
|
|
if [ -f "${PROJECT_ROOT}/.env" ]; then
|
|
cp "${PROJECT_ROOT}/.env" "${PROJECT_ROOT}/.env.bak_before_dr_restore_${TIMESTAMP}"
|
|
log "Created backup of existing .env: .env.bak_before_dr_restore_${TIMESTAMP}"
|
|
fi
|
|
|
|
# Extract all
|
|
tar -xzf "${dr_package}" -C "${PROJECT_ROOT}"
|
|
|
|
# Restore .env
|
|
if [ -f "${PROJECT_ROOT}/.env.production" ]; then
|
|
mv "${PROJECT_ROOT}/.env.production" "${PROJECT_ROOT}/.env"
|
|
log "Restored .env configuration"
|
|
fi
|
|
|
|
# Reload variables from restored .env
|
|
set -a
|
|
source "${PROJECT_ROOT}/.env"
|
|
set +a
|
|
|
|
# Restore NPM configs to their target directories if present in the package
|
|
if [ -d "${PROJECT_ROOT}/npm_data" ] && [ -n "${NPM_DATA_DIR}" ]; then
|
|
log "Restoring NPM /data directory..."
|
|
mkdir -p "$(dirname "${NPM_DATA_DIR}")"
|
|
rm -rf "${NPM_DATA_DIR}"
|
|
mv "${PROJECT_ROOT}/npm_data" "${NPM_DATA_DIR}"
|
|
fi
|
|
|
|
if [ -d "${PROJECT_ROOT}/npm_letsencrypt" ] && [ -n "${NPM_LETSENCRYPT_DIR}" ]; then
|
|
log "Restoring NPM /etc/letsencrypt directory..."
|
|
mkdir -p "$(dirname "${NPM_LETSENCRYPT_DIR}")"
|
|
rm -rf "${NPM_LETSENCRYPT_DIR}"
|
|
mv "${PROJECT_ROOT}/npm_letsencrypt" "${NPM_LETSENCRYPT_DIR}"
|
|
fi
|
|
|
|
log_success "Docker configurations, env keys, and NPM configurations restored."
|
|
|
|
# Boot Docker Compose Services
|
|
log "Spinning up Docker containers (database, redis, backend, frontend, NPM if configured)..."
|
|
local compose_cmd="docker compose"
|
|
if ! docker compose version &>/dev/null; then
|
|
compose_cmd="docker-compose"
|
|
fi
|
|
|
|
${compose_cmd} up -d
|
|
|
|
# Locate the embedded database backup
|
|
local db_backup_archive
|
|
db_backup_archive=$(ls "${PROJECT_ROOT}/db_backup/"*.gz 2>/dev/null | head -n 1 || true)
|
|
|
|
if [ -z "${db_backup_archive}" ]; then
|
|
log_error "Database backup archive not found inside the DR package extraction."
|
|
exit 1
|
|
fi
|
|
|
|
log "Database backup located: $(basename "${db_backup_archive}")"
|
|
|
|
# Wait for database container to be healthy (PostgreSQL)
|
|
local db_type="sqlite"
|
|
if [[ "${DATABASE_URL:-}" =~ ^postgres ]]; then
|
|
db_type="postgres"
|
|
fi
|
|
|
|
if [ "${db_type}" = "postgres" ]; then
|
|
local postgres_container="${POSTGRES_CONTAINER:-wordly-postgres}"
|
|
log "Waiting for PostgreSQL container (${postgres_container}) to be healthy..."
|
|
for i in $(seq 1 30); do
|
|
if docker inspect --format='{{.State.Health.Status}}' "${postgres_container}" 2>/dev/null | grep -q "healthy"; then
|
|
log_success "Database container is healthy."
|
|
break
|
|
fi
|
|
echo " Waiting for database... ($i/30)"
|
|
sleep 2
|
|
done
|
|
else
|
|
sleep 2
|
|
fi
|
|
|
|
# Restore the database using the database backup script
|
|
log "Triggering database restore..."
|
|
local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}"
|
|
mkdir -p "${local_backup_dir}/daily"
|
|
cp "${db_backup_archive}" "${local_backup_dir}/daily/"
|
|
|
|
local db_archive_filename
|
|
db_archive_filename=$(basename "${db_backup_archive}")
|
|
|
|
# Run DB restore
|
|
log "Restoring DB contents..."
|
|
bash "${SCRIPT_DIR}/backup-database.sh" --restore "${db_archive_filename}"
|
|
|
|
# Clean up extracted temporary folder
|
|
rm -rf "${PROJECT_ROOT}/db_backup"
|
|
|
|
# Restart app to clear connection caches
|
|
log "Restarting application backend..."
|
|
${compose_cmd} restart backend
|
|
|
|
log_success "=========================================================================="
|
|
log_success "DISASTER RECOVERY SYSTEM RESTORE COMPLETE!"
|
|
log_success "=========================================================================="
|
|
log "Your application and reverse-proxy routes are restored."
|
|
echo ""
|
|
}
|
|
|
|
# ==============================================================================
|
|
# MAIN ENTRY
|
|
# ==============================================================================
|
|
main() {
|
|
case "${1:-}" in
|
|
--backup)
|
|
perform_backup
|
|
;;
|
|
--restore)
|
|
perform_restore "${2:-}"
|
|
;;
|
|
*)
|
|
echo "Wordly Disaster Recovery Utility (V2)"
|
|
echo "Usage:"
|
|
echo " $0 --backup # Package configs, db dump, NPM configurations, and export"
|
|
echo " $0 --restore <archive.tar.gz> # Extract and restore full stack on new machine"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|