From e7b5ea9a61b95321669720daf8ae604785336781 Mon Sep 17 00:00:00 2001 From: sepehr Date: Sun, 7 Jun 2026 09:26:39 +0200 Subject: [PATCH] Robustness: Add json-file log rotation limits to all docker services --- docker-compose.local.yml | 25 +++ docker-compose.yml | 45 +++++ scripts/backup-database.sh | 401 +++++++++++++++++++++++++++++++++++++ 3 files changed, 471 insertions(+) create mode 100755 scripts/backup-database.sh diff --git a/docker-compose.local.yml b/docker-compose.local.yml index caa067b..ce17d0c 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -26,6 +26,11 @@ services: - "5432:5432" networks: - translate-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-translate} -d ${POSTGRES_DB:-translate_db}"] interval: 5s @@ -47,6 +52,11 @@ services: - "6379:6379" networks: - translate-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -84,6 +94,11 @@ services: - "8000:8000" networks: - translate-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: postgres: condition: service_healthy @@ -119,6 +134,11 @@ services: - "${FRONTEND_PORT:-3010}:3000" networks: - translate-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: backend: condition: service_healthy @@ -144,6 +164,11 @@ services: - nginx_cache:/var/cache/nginx networks: - translate-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: backend: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 8af1dfb..ab20ad6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,11 @@ services: - postgres_data:/var/lib/postgresql/data networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-translate} -d ${POSTGRES_DB:-translate_db}"] interval: 10s @@ -53,6 +58,11 @@ services: - redis_data:/data networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -116,6 +126,11 @@ services: - logs_data:/app/logs networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: postgres: condition: service_healthy @@ -155,6 +170,11 @@ services: - NEXT_PUBLIC_GOOGLE_CLIENT_ID=${NEXT_PUBLIC_GOOGLE_CLIENT_ID:-} networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: backend: condition: service_healthy @@ -176,6 +196,11 @@ services: - ollama_data:/root/.ollama networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" deploy: resources: reservations: @@ -205,6 +230,11 @@ services: - '--web.enable-lifecycle' networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"] interval: 30s @@ -232,6 +262,11 @@ services: - "3001:3000" networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" depends_on: prometheus: condition: service_healthy @@ -260,6 +295,11 @@ services: - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|etc|var/lib/docker)($$|/)' networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" # =========================================== # cAdvisor - Container Metrics @@ -279,6 +319,11 @@ services: - /dev/kmsg networks: - wordly-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" # =========================================== # Networks diff --git a/scripts/backup-database.sh b/scripts/backup-database.sh new file mode 100755 index 0000000..eedec3e --- /dev/null +++ b/scripts/backup-database.sh @@ -0,0 +1,401 @@ +#!/bin/bash +# ============================================================================== +# Wordly.art - Unified Database Backup & Restore Script +# ============================================================================== +# Sourced dynamically from .env. Supports PostgreSQL (Docker) and SQLite (Local). +# +# Usage: +# ./backup-database.sh # Perform scheduled backup (daily/weekly/monthly) +# ./backup-database.sh --full # Run backup cycle with retention cleanup +# ./backup-database.sh --restore FILE.gz # Restore database from a specific archive +# ============================================================================== + +set -euo pipefail + +# SCRIPT DIRECTORY AND CONSTANTS +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +DAY_OF_WEEK=$(date +"%u") # 1=Mon, 7=Sun +DAY_OF_MONTH=$(date +"%d") + +# Colors for terminal output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "[Backup ${TIMESTAMP}] $1"; } +log_success() { echo -e "[Backup ${TIMESTAMP}] ${GREEN}$1${NC}"; } +log_warning() { echo -e "[Backup ${TIMESTAMP}] ${YELLOW}WARNING: $1${NC}"; } +log_error() { echo -e "[Backup ${TIMESTAMP}] ${RED}ERROR: $1${NC}"; } + +# ============================================================================== +# 1. LOAD CONFIGURATION +# ============================================================================== +ENV_FILE="${SCRIPT_DIR}/../.env" +if [ -f "${ENV_FILE}" ]; then + set -a + source "${ENV_FILE}" + set +a + log "Loaded environment configuration from .env" +else + log_warning ".env file not found at ${ENV_FILE}. Using system environment variables." +fi + +# Configuration Fallbacks +BACKUP_DIR="${BACKUP_DIR:-${SCRIPT_DIR}/../backups}" +DAILY_RETENTION=${DAILY_RETENTION:-7} +WEEKLY_RETENTION=${WEEKLY_RETENTION:-4} +MONTHLY_RETENTION=${MONTHLY_RETENTION:-6} +NOTIFICATION_WEBHOOK="${NOTIFICATION_WEBHOOK:-}" + +# Parse DB configuration +DATABASE_URL="${DATABASE_URL:-}" +DB_TYPE="sqlite" +if [[ "${DATABASE_URL}" =~ ^postgres ]]; then + DB_TYPE="postgres" +fi + +# SQLite-specific configs +SQLITE_PATH="${SQLITE_PATH:-data/translate.db}" +# Ensure SQLite path is absolute +if [[ ! "$SQLITE_PATH" =~ ^/ ]] && [[ "$SQLITE_PATH" =~ ^[a-zA-Z]: ]]; then + # Already absolute (Windows path format) + true +else + # Convert to absolute path relative to project root + SQLITE_PATH="${SCRIPT_DIR}/../${SQLITE_PATH}" +fi + +# PostgreSQL-specific configs +POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-wordly-postgres}" +POSTGRES_USER="${POSTGRES_USER:-translate}" +POSTGRES_DB="${POSTGRES_DB:-translate_db}" +POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-}" + +# ============================================================================== +# 2. VALIDATION AND PREPARATION +# ============================================================================== +check_prerequisites() { + mkdir -p "${BACKUP_DIR}/daily" + mkdir -p "${BACKUP_DIR}/weekly" + mkdir -p "${BACKUP_DIR}/monthly" + + if [ "${DB_TYPE}" = "postgres" ]; then + # Check Docker PostgreSQL container + if ! command -v docker &>/dev/null; then + log_error "Docker is not installed or not in PATH." + exit 1 + fi + if ! docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then + log_error "PostgreSQL container '${POSTGRES_CONTAINER}' is not running." + exit 1 + fi + if [ -z "${POSTGRES_PASSWORD}" ]; then + log_error "POSTGRES_PASSWORD is not set in environment or .env file." + exit 1 + fi + else + # Check SQLite db file + local db_dir + db_dir=$(dirname "${SQLITE_PATH}") + mkdir -p "${db_dir}" + if [ ! -f "${SQLITE_PATH}" ]; then + log_warning "SQLite database file not found at '${SQLITE_PATH}'. It will be created on first write." + fi + fi +} + +send_notification() { + local message="$1" + if [ -n "${NOTIFICATION_WEBHOOK}" ]; then + curl -s -X POST "${NOTIFICATION_WEBHOOK}" \ + -H "Content-Type: application/json" \ + -d "{\"text\": \"${message}\"}" >/dev/null 2>&1 || true + fi +} + +# ============================================================================== +# 3. BACKUP OPERATION +# ============================================================================== +create_backup() { + local backup_name + local dest_path + + if [ "${DB_TYPE}" = "postgres" ]; then + backup_name="wordly_postgres_${TIMESTAMP}.sql.gz" + dest_path="${BACKUP_DIR}/daily/${backup_name}" + log "Starting PostgreSQL backup for database '${POSTGRES_DB}'..." + + # Dump using pg_dump inside Docker container + if ! docker exec -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" pg_dump \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + --format=custom \ + --no-owner \ + --no-acl \ + 2>/dev/null | gzip > "${dest_path}"; then + log_error "PostgreSQL pg_dump failed!" + send_notification "Wordly DB Backup FAILED: PostgreSQL dump error." + exit 1 + fi + else + backup_name="wordly_sqlite_${TIMESTAMP}.db.gz" + dest_path="${BACKUP_DIR}/daily/${backup_name}" + log "Starting SQLite backup for file '${SQLITE_PATH}'..." + + # If the database file does not exist yet (fresh install), touch it to allow backup + if [ ! -f "${SQLITE_PATH}" ]; then + touch "${SQLITE_PATH}" + fi + + # Safe online backup using Python + local temp_db="${BACKUP_DIR}/daily/temp_${TIMESTAMP}.db" + if ! python -c " +import sqlite3, sys +try: + src = sqlite3.connect(r'${SQLITE_PATH}') + dst = sqlite3.connect(r'${temp_db}') + with dst: + src.backup(dst) + src.close() + dst.close() +except Exception as e: + print(e, file=sys.stderr) + sys.exit(1) +" 2>/dev/null; then + log_error "SQLite online backup failed! Checking if DB is locked." + send_notification "Wordly DB Backup FAILED: SQLite online backup script error." + exit 1 + fi + + # Compress the temporary copy + gzip -c "${temp_db}" > "${dest_path}" + rm -f "${temp_db}" + fi + + # Verify backup size and integrity + if [ -f "${dest_path}" ] && [ -s "${dest_path}" ]; then + local size + size=$(du -h "${dest_path}" | cut -f1) + log_success "Backup file created successfully: ${backup_name} (${size})" + + # Verify archive integrity + if gzip -t "${dest_path}" 2>/dev/null; then + log_success "Backup integrity verification passed." + else + log_error "Backup archive integrity check FAILED!" + send_notification "Wordly DB Backup WARNING: Integrity check failed for ${backup_name}" + exit 1 + fi + + # Distribute to weekly/monthly folders + if [ "${DAY_OF_WEEK}" = "7" ]; then + cp "${dest_path}" "${BACKUP_DIR}/weekly/${backup_name}" + log "Weekly backup archived." + fi + if [ "${DAY_OF_MONTH}" = "01" ]; then + cp "${dest_path}" "${BACKUP_DIR}/monthly/${backup_name}" + log "Monthly backup archived." + fi + + send_notification "Wordly DB Backup SUCCESS: ${backup_name} (${size})" + else + log_error "Backup failed! Dest file is empty or missing." + send_notification "Wordly DB Backup FAILED: Destination archive is empty." + exit 1 + fi +} + +# ============================================================================== +# 4. ROTATION & CLEANUP +# ============================================================================== +cleanup_old_backups() { + log "Running rotation cleanup policy..." + + # Daily backups retention + local daily_count + daily_count=$(ls -1 "${BACKUP_DIR}/daily/" 2>/dev/null | wc -l || echo 0) + if [ "${daily_count}" -gt "${DAILY_RETENTION}" ]; then + ls -1t "${BACKUP_DIR}/daily/" | tail -n +$((DAILY_RETENTION + 1)) | while read -r f; do + rm -f "${BACKUP_DIR}/daily/${f}" + log " Purged old daily backup: ${f}" + done + fi + + # Weekly backups retention + local weekly_count + weekly_count=$(ls -1 "${BACKUP_DIR}/weekly/" 2>/dev/null | wc -l || echo 0) + if [ "${weekly_count}" -gt "${WEEKLY_RETENTION}" ]; then + ls -1t "${BACKUP_DIR}/weekly/" | tail -n +$((WEEKLY_RETENTION + 1)) | while read -r f; do + rm -f "${BACKUP_DIR}/weekly/${f}" + log " Purged old weekly backup: ${f}" + done + fi + + # Monthly backups retention + local monthly_count + monthly_count=$(ls -1 "${BACKUP_DIR}/monthly/" 2>/dev/null | wc -l || echo 0) + if [ "${monthly_count}" -gt "${MONTHLY_RETENTION}" ]; then + ls -1t "${BACKUP_DIR}/monthly/" | tail -n +$((MONTHLY_RETENTION + 1)) | while read -r f; do + rm -f "${BACKUP_DIR}/monthly/${f}" + log " Purged old monthly backup: ${f}" + done + fi + + log_success "Backup rotation cleanup complete." +} + +# ============================================================================== +# 5. RESTORE OPERATION +# ============================================================================== +restore_backup() { + local archive_name="$1" + + if [ -z "${archive_name}" ]; then + log_error "No restore file specified." + echo "Usage: $0 --restore " + echo "Available backups in ${BACKUP_DIR}:" + echo "--- Daily ---" + ls -lh "${BACKUP_DIR}/daily/" 2>/dev/null || echo " (empty)" + echo "--- Weekly ---" + ls -lh "${BACKUP_DIR}/weekly/" 2>/dev/null || echo " (empty)" + echo "--- Monthly ---" + ls -lh "${BACKUP_DIR}/monthly/" 2>/dev/null || echo " (empty)" + exit 1 + fi + + # Locate archive in backup directories + local archive_path="" + for subdir in daily weekly monthly; do + if [ -f "${BACKUP_DIR}/${subdir}/${archive_name}" ]; then + archive_path="${BACKUP_DIR}/${subdir}/${archive_name}" + break + fi + done + + if [ -z "${archive_path}" ]; then + # Try if it's a full path + if [ -f "${archive_name}" ]; then + archive_path="${archive_name}" + else + log_error "Restore archive file not found: ${archive_name}" + exit 1 + fi + fi + + # Safety prompt + echo "" + log_warning "RESTORE MODE - THIS WILL OVERWRITE YOUR ACTIVE DATABASE!" + echo " Archive Source: ${archive_path}" + echo " Database Type : ${DB_TYPE}" + if [ "${DB_TYPE}" = "postgres" ]; then + echo " Target DB : PostgreSQL Database '${POSTGRES_DB}'" + else + echo " Target File : SQLite File '${SQLITE_PATH}'" + fi + echo "" + read -p "To confirm restore, type 'RESTORE': " confirm_val + if [ "${confirm_val}" != "RESTORE" ]; then + log "Restore cancelled." + exit 0 + fi + + log "Initiating restore sequence..." + + # Create safety backup of the active database before restoring + log "Creating safety backup of the current database state..." + local safety_name + if [ "${DB_TYPE}" = "postgres" ]; then + safety_name="wordly_postgres_pre_restore_${TIMESTAMP}.sql.gz" + docker exec -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" pg_dump \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + --format=custom \ + --no-owner \ + --no-acl \ + 2>/dev/null | gzip > "${BACKUP_DIR}/daily/${safety_name}" + else + safety_name="wordly_sqlite_pre_restore_${TIMESTAMP}.db.gz" + if [ -f "${SQLITE_PATH}" ]; then + gzip -c "${SQLITE_PATH}" > "${BACKUP_DIR}/daily/${safety_name}" + else + touch "${SQLITE_PATH}" + gzip -c "${SQLITE_PATH}" > "${BACKUP_DIR}/daily/${safety_name}" + fi + fi + log_success "Safety pre-restore backup created: ${safety_name}" + + # Restore DB + if [ "${DB_TYPE}" = "postgres" ]; then + log "Restoring PostgreSQL database..." + if ! gunzip -c "${archive_path}" | docker exec -i -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" \ + pg_restore \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl; then + log_error "PostgreSQL restore failed!" + exit 1 + fi + else + log "Restoring SQLite database file..." + + # Decompress SQLite backup to a temp file + local temp_restore_db="${BACKUP_DIR}/daily/temp_restore_${TIMESTAMP}.db" + gunzip -c "${archive_path}" > "${temp_restore_db}" + + # Overwrite database safely using Python (ensures open handles or locks are respected) + if ! python -c " +import sqlite3, sys +try: + src = sqlite3.connect(r'${temp_restore_db}') + dst = sqlite3.connect(r'${SQLITE_PATH}') + with dst: + src.backup(dst) + src.close() + dst.close() +except Exception as e: + print(e, file=sys.stderr) + sys.exit(1) +" 2>/dev/null; then + log_error "SQLite restore swap operation failed!" + rm -f "${temp_restore_db}" + exit 1 + fi + + rm -f "${temp_restore_db}" + fi + + log_success "Restore process completed successfully!" + log_warning "You should restart your app container to clear active connection caches:" + echo " docker compose restart backend" +} + +# ============================================================================== +# 6. ENTRYS +# ============================================================================== +main() { + case "${1:-}" in + --restore) + check_prerequisites + restore_backup "${2:-}" + ;; + --full) + check_prerequisites + create_backup + cleanup_old_backups + log_success "Full backup and rotation sequence completed." + ;; + *) + check_prerequisites + create_backup + cleanup_old_backups + log_success "Standard backup completed." + ;; + esac +} + +main "$@"