sepehr/office_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Robustness: Implement multi-destination backups (LOCAL, NAS, SCP) and backup/restore of NPM configurations

Browse Source
This commit is contained in:
sepehr
2026-06-07 09:50:51 +02:00
parent c7299228cd
commit 80b49ee354
2 changed files with 230 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

245
scripts/disaster-recovery.sh
View File

@@ -1,13 +1,9 @@
#!/bin/bash
# ==============================================================================
# Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook
# Wordly.art - Disaster Recovery (DR) Backup & Restore Playbook (V2)
# ==============================================================================
# Packages configuration files (.env, docker-compose) and database backups
# into a single archive, and automates restorations on new machines.
#
# Usage:
# ./disaster-recovery.sh --backup # Package and backup configs + DB to NAS
# ./disaster-recovery.sh --restore FILE.tar.gz # Extract configs, boot Docker, restore DB
# Packages app configs (.env, docker-compose), database backups, and NPM
# configs, and exports them to LOCAL, NAS, or remote SCP storage.
# ==============================================================================
set -euo pipefail
@@ -27,7 +23,7 @@ log_success() { echo -e "[DR ${TIMESTAMP}] ${GREEN}$1${NC}"; }
log_warning() { echo -e "[DR ${TIMESTAMP}] ${YELLOW}WARNING: $1${NC}"; }
log_error() { echo -e "[DR ${TIMESTAMP}] ${RED}ERROR: $1${NC}"; }
# Sourcing .env for path defaults
# Sourcing .env
ENV_FILE="${PROJECT_ROOT}/.env"
if [ -f "${ENV_FILE}" ]; then
set -a
@@ -35,107 +31,174 @@ if [ -f "${ENV_FILE}" ]; then
set +a
fi
# Configuration defaults
NAS_BACKUP_DIR="${NAS_BACKUP_DIR:-/mnt/nas-backups/wordly}"
# Config Defaults & Type Resolution
BACKUP_DEST_TYPE="${BACKUP_DEST_TYPE:-LOCAL}" # LOCAL, NAS, SCP
BACKUP_DEST_PATH="${BACKUP_DEST_PATH:-${PROJECT_ROOT}/backups}"
DR_RETENTION_DAYS=${DR_RETENTION_DAYS:-14}
# Fallback to local backups folder if NAS directory is not writable
if [ ! -d "${NAS_BACKUP_DIR}" ]; then
mkdir -p "${NAS_BACKUP_DIR}" 2>/dev/null || true
fi
# SCP Configuration
SCP_HOST="${SCP_HOST:-}"
SCP_USER="${SCP_USER:-}"
SCP_KEY_PATH="${SCP_KEY_PATH:-~/.ssh/id_rsa}"
SCP_PORT="${SCP_PORT:-22}"
SCP_DEST_PATH="${SCP_DEST_PATH:-/var/backups/wordly}"
if [ ! -w "${NAS_BACKUP_DIR}" ]; then
log_warning "NAS backup directory '${NAS_BACKUP_DIR}' is not writable or mounted. Falling back to local backups folder."
NAS_BACKUP_DIR="${PROJECT_ROOT}/backups"
fi
# NPM Configuration directories
NPM_DATA_DIR="${NPM_DATA_DIR:-}"
NPM_LETSENCRYPT_DIR="${NPM_LETSENCRYPT_DIR:-}"
DR_BACKUP_DIR="${NAS_BACKUP_DIR}/dr"
DR_RETENTION_DAYS=14
# ==============================================================================
# DESTINATION PREPARATION
# ==============================================================================
prepare_destination() {
if [ "${BACKUP_DEST_TYPE}" = "NAS" ] || [ "${BACKUP_DEST_TYPE}" = "LOCAL" ]; then
if [ ! -d "${BACKUP_DEST_PATH}" ]; then
mkdir -p "${BACKUP_DEST_PATH}" 2>/dev/null || true
fi
if [ ! -w "${BACKUP_DEST_PATH}" ]; then
log_warning "Backup destination path '${BACKUP_DEST_PATH}' is not writable. Falling back to local backups."
BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
BACKUP_DEST_TYPE="LOCAL"
mkdir -p "${BACKUP_DEST_PATH}/dr"
fi
DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
mkdir -p "${DR_LOCAL_DIR}"
elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
if [ -z "${SCP_HOST}" ] || [ -z "${SCP_USER}" ]; then
log_error "SCP backup selected but SCP_HOST or SCP_USER is not configured in .env."
log_warning "Falling back to LOCAL backup directory."
BACKUP_DEST_TYPE="LOCAL"
BACKUP_DEST_PATH="${PROJECT_ROOT}/backups"
DR_LOCAL_DIR="${BACKUP_DEST_PATH}/dr"
mkdir -p "${DR_LOCAL_DIR}"
fi
fi
}
# ==============================================================================
# BACKUP ACTION
# ==============================================================================
perform_backup() {
log "Initiating complete Disaster Recovery backup..."
prepare_destination
log "Starting Disaster Recovery backup (Destination Mode: ${BACKUP_DEST_TYPE})..."
# 1. Trigger DB Backup first
log "Triggering database backup..."
# 1. Trigger DB Backup
log "Triggering database dump..."
if ! bash "${SCRIPT_DIR}/backup-database.sh" --full; then
log_error "Database backup failed. Aborting DR package."
log_error "Database backup failed. Aborting DR packaging."
exit 1
fi
# 2. Locate the latest DB backup
# Check default backup directory (resolved from .env or script fallback)
local local_backup_dir
local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}"
# 2. Locate DB Backup file
local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}"
local latest_db_backup
latest_db_backup=$(ls -t "${local_backup_dir}/daily/"*.gz 2>/dev/null | head -n 1 || true)
if [ -z "${latest_db_backup}" ]; then
log_error "Could not locate the generated database backup file in ${local_backup_dir}/daily/."
log_error "Could not find database backup file."
exit 1
fi
log "Latest database backup located: $(basename "${latest_db_backup}")"
log "Database backup file loaded: $(basename "${latest_db_backup}")"
# 3. Create temp packing folder
# 3. Create temp packaging folder
local packing_dir="${PROJECT_ROOT}/temp_dr_pack_${TIMESTAMP}"
mkdir -p "${packing_dir}"
# 4. Copy configurations
log "Packaging configuration files..."
# 4. Pack Configurations
log "Packing application configuration (.env & docker-compose)..."
if [ -f "${PROJECT_ROOT}/.env" ]; then
cp "${PROJECT_ROOT}/.env" "${packing_dir}/.env.production"
else
log_warning "No .env file found at project root. Continuing without it."
fi
# Copy docker-compose files
for f in docker-compose.yml docker-compose.local.yml docker-compose.monitoring.yml docker-compose.dev.yml; do
if [ -f "${PROJECT_ROOT}/${f}" ]; then
cp "${PROJECT_ROOT}/${f}" "${packing_dir}/"
fi
done
# Copy docker directory (Prometheus, Grafana, Nginx configs, Dockerfiles)
if [ -d "${PROJECT_ROOT}/docker" ]; then
cp -r "${PROJECT_ROOT}/docker" "${packing_dir}/"
fi
# Copy scripts directory (so restore scripts are present in the package)
if [ -d "${PROJECT_ROOT}/scripts" ]; then
cp -r "${PROJECT_ROOT}/scripts" "${packing_dir}/"
fi
# Copy the DB backup archive
mkdir -p "${packing_dir}/db_backup"
cp "${latest_db_backup}" "${packing_dir}/db_backup/"
# 5. Compress Everything
mkdir -p "${DR_BACKUP_DIR}"
# 5. Pack Nginx Proxy Manager (NPM) configs if configured
local has_npm_data=false
if [ -n "${NPM_DATA_DIR}" ] && [ -d "${NPM_DATA_DIR}" ]; then
log "Packaging Nginx Proxy Manager /data directory..."
cp -r "${NPM_DATA_DIR}" "${packing_dir}/npm_data"
has_npm_data=true
fi
if [ -n "${NPM_LETSENCRYPT_DIR}" ] && [ -d "${NPM_LETSENCRYPT_DIR}" ]; then
log "Packaging Nginx Proxy Manager /etc/letsencrypt directory..."
cp -r "${NPM_LETSENCRYPT_DIR}" "${packing_dir}/npm_letsencrypt"
has_npm_data=true
fi
if [ "${has_npm_data}" = "false" ]; then
log_warning "NPM directories (NPM_DATA_DIR / NPM_LETSENCRYPT_DIR) not configured or not found. Skipping NPM config packaging."
fi
# 6. Compress DR Archive
local dr_archive_name="wordly_dr_${TIMESTAMP}.tar.gz"
local dr_archive_path="${DR_BACKUP_DIR}/${dr_archive_name}"
local local_archive_path="${PROJECT_ROOT}/${dr_archive_name}"
log "Compressing configurations and database into DR package..."
tar -czf "${dr_archive_path}" -C "${packing_dir}" .
# Clean up temp packaging folder
log "Compressing configurations, database, and NPM data into DR archive..."
tar -czf "${local_archive_path}" -C "${packing_dir}" .
rm -rf "${packing_dir}"
if [ -f "${dr_archive_path}" ] && [ -s "${dr_archive_path}" ]; then
local size
size=$(du -h "${dr_archive_path}" | cut -f1)
log_success "Disaster Recovery backup package created: ${dr_archive_name} (${size})"
log_success "Stored securely at: ${dr_archive_path}"
# 6. Apply retention cleanups
log "Cleaning up old DR packages (retention: ${DR_RETENTION_DAYS} days)..."
find "${DR_BACKUP_DIR}" -name "wordly_dr_*.tar.gz" -mtime +"${DR_RETENTION_DAYS}" -exec rm -f {} \;
log_success "Disaster Recovery backup complete."
else
log_error "DR Archive compression failed."
if [ ! -f "${local_archive_path}" ] || [ ! -s "${local_archive_path}" ]; then
log_error "Failed to compress archive."
exit 1
fi
local size
size=$(du -h "${local_archive_path}" | cut -f1)
# 7. Route to Destination
if [ "${BACKUP_DEST_TYPE}" = "LOCAL" ] || [ "${BACKUP_DEST_TYPE}" = "NAS" ]; then
local dest_path="${DR_LOCAL_DIR}/${dr_archive_name}"
mv "${local_archive_path}" "${dest_path}"
log_success "DR archive created successfully (${size}) at: ${dest_path}"
# Retention
log "Applying retention policy (pruning files older than ${DR_RETENTION_DAYS} days)..."
find "${DR_LOCAL_DIR}" -name "wordly_dr_*.tar.gz" -mtime +"${DR_RETENTION_DAYS}" -exec rm -f {} \;
elif [ "${BACKUP_DEST_TYPE}" = "SCP" ]; then
log "Transferring DR archive to remote server via SCP (${SCP_USER}@${SCP_HOST}:${SCP_PORT})..."
# Test connection & Create remote directory if not exists
if ! ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" "mkdir -p ${SCP_DEST_PATH}" 2>/dev/null; then
log_error "SSH connection to ${SCP_USER}@${SCP_HOST} failed. Saving archive locally instead."
mkdir -p "${PROJECT_ROOT}/backups/dr"
mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
log_warning "DR backup saved locally at: ${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
exit 1
fi
# SCP copy
if scp -P "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${local_archive_path}" "${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"; then
log_success "DR archive transferred successfully to ${SCP_USER}@${SCP_HOST}:${SCP_DEST_PATH}/${dr_archive_name}"
rm -f "${local_archive_path}"
# Remote retention prune
log "Applying remote retention policy on backup server..."
ssh -p "${SCP_PORT}" -i "${SCP_KEY_PATH}" -o StrictHostKeyChecking=no "${SCP_USER}@${SCP_HOST}" \
"find ${SCP_DEST_PATH} -name 'wordly_dr_*.tar.gz' -mtime +${DR_RETENTION_DAYS} -exec rm -f {} \;" || true
else
log_error "SCP file transfer failed. Retaining local backup."
mkdir -p "${PROJECT_ROOT}/backups/dr"
mv "${local_archive_path}" "${PROJECT_ROOT}/backups/dr/${dr_archive_name}"
fi
fi
log_success "Disaster Recovery backup complete."
}
# ==============================================================================
@@ -147,20 +210,17 @@ perform_restore() {
if [ -z "${dr_package}" ]; then
log_error "No DR package archive specified."
echo "Usage: $0 --restore <path_to_archive.tar.gz>"
echo "Available archives in ${DR_BACKUP_DIR}:"
ls -lh "${DR_BACKUP_DIR}"/wordly_dr_*.tar.gz 2>/dev/null || echo " (none)"
exit 1
fi
if [ ! -f "${dr_package}" ]; then
log_error "Archive file not found: ${dr_package}"
log_error "DR Archive file not found: ${dr_package}"
exit 1
fi
echo ""
log_warning "RESTORE DISASTER RECOVERY PACKAGE - THIS WILL OVERWRITE ENVIRONMENT CONFIGURATIONS AND DATABASES!"
log_warning "RESTORE DISASTER RECOVERY PACKAGE - THIS WILL OVERWRITE ENVIRONMENT CONFIGURATIONS, DATABASES, AND NPM FILES!"
echo " Archive: ${dr_package}"
echo " Target : Current Server Host (Workspace)"
echo ""
read -p "Type 'RESTORE-ALL' to confirm complete system restore: " confirm_val
if [ "${confirm_val}" != "RESTORE-ALL" ]; then
@@ -169,17 +229,17 @@ perform_restore() {
fi
log "Extracting DR archive contents..."
# Create safety backup of existing .env before overwrite
# Safety backup of existing .env
if [ -f "${PROJECT_ROOT}/.env" ]; then
cp "${PROJECT_ROOT}/.env" "${PROJECT_ROOT}/.env.bak_before_dr_restore_${TIMESTAMP}"
log "Created backup of existing .env: .env.bak_before_dr_restore_${TIMESTAMP}"
fi
# Extract configs directly into project root
# Extract all
tar -xzf "${dr_package}" -C "${PROJECT_ROOT}"
# Restore .env from packaged .env.production
# Restore .env
if [ -f "${PROJECT_ROOT}/.env.production" ]; then
mv "${PROJECT_ROOT}/.env.production" "${PROJECT_ROOT}/.env"
log "Restored .env configuration"
@@ -190,22 +250,30 @@ perform_restore() {
source "${PROJECT_ROOT}/.env"
set +a
log_success "Docker and configurations extracted successfully."
# Boot Docker Compose Services
log "Spinning up Docker containers (database, redis, backend, frontend)..."
if ! command -v docker-compose &>/dev/null && ! docker compose version &>/dev/null; then
log_error "docker-compose is not installed. Please install Docker first."
exit 1
# Restore NPM configs to their target directories if present in the package
if [ -d "${PROJECT_ROOT}/npm_data" ] && [ -n "${NPM_DATA_DIR}" ]; then
log "Restoring NPM /data directory..."
mkdir -p "$(dirname "${NPM_DATA_DIR}")"
rm -rf "${NPM_DATA_DIR}"
mv "${PROJECT_ROOT}/npm_data" "${NPM_DATA_DIR}"
fi
# Try running docker compose
if [ -d "${PROJECT_ROOT}/npm_letsencrypt" ] && [ -n "${NPM_LETSENCRYPT_DIR}" ]; then
log "Restoring NPM /etc/letsencrypt directory..."
mkdir -p "$(dirname "${NPM_LETSENCRYPT_DIR}")"
rm -rf "${NPM_LETSENCRYPT_DIR}"
mv "${PROJECT_ROOT}/npm_letsencrypt" "${NPM_LETSENCRYPT_DIR}"
fi
log_success "Docker configurations, env keys, and NPM configurations restored."
# Boot Docker Compose Services
log "Spinning up Docker containers (database, redis, backend, frontend, NPM if configured)..."
local compose_cmd="docker compose"
if ! docker compose version &>/dev/null; then
compose_cmd="docker-compose"
fi
# Start services in detached mode
${compose_cmd} up -d
# Locate the embedded database backup
@@ -242,7 +310,6 @@ perform_restore() {
# Restore the database using the database backup script
log "Triggering database restore..."
# Make sure backups/daily folder exists and copy the db backup there for backup-database.sh to see it
local local_backup_dir="${BACKUP_DIR:-${PROJECT_ROOT}/backups}"
mkdir -p "${local_backup_dir}/daily"
cp "${db_backup_archive}" "${local_backup_dir}/daily/"
@@ -251,13 +318,10 @@ perform_restore() {
db_archive_filename=$(basename "${db_backup_archive}")
# Run DB restore
# Sourcing backup-database.sh with the file name
# We pass the confirmation non-interactively using YES or mock prompt if needed,
# but backup-database.sh reads YES/RESTORE. Let's make it easy:
log "Restoring DB contents... (You will need to type 'RESTORE' if prompted)"
log "Restoring DB contents..."
bash "${SCRIPT_DIR}/backup-database.sh" --restore "${db_archive_filename}"
# Clean up extracted folders
# Clean up extracted temporary folder
rm -rf "${PROJECT_ROOT}/db_backup"
# Restart app to clear connection caches
@@ -267,10 +331,7 @@ perform_restore() {
log_success "=========================================================================="
log_success "DISASTER RECOVERY SYSTEM RESTORE COMPLETE!"
log_success "=========================================================================="
log "Your application has been restored and started."
log "Next Steps:"
log "1. Verify the service is online: curl http://localhost:8000/health"
log "2. Update your Nginx Proxy Manager (NPM) domains to point to this server's IP."
log "Your application and reverse-proxy routes are restored."
echo ""
}
@@ -286,9 +347,9 @@ main() {
perform_restore "${2:-}"
;;
*)
echo "Wordly Disaster Recovery Utility"
echo "Wordly Disaster Recovery Utility (V2)"
echo "Usage:"
echo " $0 --backup # Package and copy configs + database to NAS"
echo " $0 --backup # Package configs, db dump, NPM configurations, and export"
echo " $0 --restore <archive.tar.gz> # Extract and restore full stack on new machine"
exit 1
;;