Robustness: Add json-file log rotation limits to all docker services

docker-compose.local.yml

@@ -26,6 +26,11 @@ services:
       - "5432:5432"
     networks:
       - translate-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-translate} -d ${POSTGRES_DB:-translate_db}"]
       interval: 5s
@@ -47,6 +52,11 @@ services:
       - "6379:6379"
     networks:
       - translate-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -84,6 +94,11 @@ services:
       - "8000:8000"
     networks:
       - translate-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       postgres:
         condition: service_healthy
@@ -119,6 +134,11 @@ services:
       - "${FRONTEND_PORT:-3010}:3000"
     networks:
       - translate-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       backend:
         condition: service_healthy
@@ -144,6 +164,11 @@ services:
       - nginx_cache:/var/cache/nginx
     networks:
       - translate-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       backend:
         condition: service_healthy

docker-compose.yml

@@ -28,6 +28,11 @@ services:
       - postgres_data:/var/lib/postgresql/data
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-translate} -d ${POSTGRES_DB:-translate_db}"]
       interval: 10s
@@ -53,6 +58,11 @@ services:
       - redis_data:/data
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s
@@ -116,6 +126,11 @@ services:
       - logs_data:/app/logs
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       postgres:
         condition: service_healthy
@@ -155,6 +170,11 @@ services:
       - NEXT_PUBLIC_GOOGLE_CLIENT_ID=${NEXT_PUBLIC_GOOGLE_CLIENT_ID:-}
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       backend:
         condition: service_healthy
@@ -176,6 +196,11 @@ services:
       - ollama_data:/root/.ollama
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     deploy:
       resources:
         reservations:
@@ -205,6 +230,11 @@ services:
       - '--web.enable-lifecycle'
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
       interval: 30s
@@ -232,6 +262,11 @@ services:
       - "3001:3000"
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     depends_on:
       prometheus:
         condition: service_healthy
@@ -260,6 +295,11 @@ services:
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|etc|var/lib/docker)($$|/)'
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
 
   # ===========================================
   # cAdvisor - Container Metrics
@@ -279,6 +319,11 @@ services:
       - /dev/kmsg
     networks:
       - wordly-network
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
 
 # ===========================================
 # Networks

scripts/backup-database.sh

@@ -0,0 +1,401 @@
+#!/bin/bash
+# ==============================================================================
+# Wordly.art - Unified Database Backup & Restore Script
+# ==============================================================================
+# Sourced dynamically from .env. Supports PostgreSQL (Docker) and SQLite (Local).
+#
+# Usage:
+#   ./backup-database.sh                    # Perform scheduled backup (daily/weekly/monthly)
+#   ./backup-database.sh --full             # Run backup cycle with retention cleanup
+#   ./backup-database.sh --restore FILE.gz   # Restore database from a specific archive
+# ==============================================================================
+
+set -euo pipefail
+
+# SCRIPT DIRECTORY AND CONSTANTS
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+DAY_OF_WEEK=$(date +"%u")    # 1=Mon, 7=Sun
+DAY_OF_MONTH=$(date +"%d")
+
+# Colors for terminal output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+log() { echo -e "[Backup ${TIMESTAMP}] $1"; }
+log_success() { echo -e "[Backup ${TIMESTAMP}] ${GREEN}$1${NC}"; }
+log_warning() { echo -e "[Backup ${TIMESTAMP}] ${YELLOW}WARNING: $1${NC}"; }
+log_error() { echo -e "[Backup ${TIMESTAMP}] ${RED}ERROR: $1${NC}"; }
+
+# ==============================================================================
+# 1. LOAD CONFIGURATION
+# ==============================================================================
+ENV_FILE="${SCRIPT_DIR}/../.env"
+if [ -f "${ENV_FILE}" ]; then
+    set -a
+    source "${ENV_FILE}"
+    set +a
+    log "Loaded environment configuration from .env"
+else
+    log_warning ".env file not found at ${ENV_FILE}. Using system environment variables."
+fi
+
+# Configuration Fallbacks
+BACKUP_DIR="${BACKUP_DIR:-${SCRIPT_DIR}/../backups}"
+DAILY_RETENTION=${DAILY_RETENTION:-7}
+WEEKLY_RETENTION=${WEEKLY_RETENTION:-4}
+MONTHLY_RETENTION=${MONTHLY_RETENTION:-6}
+NOTIFICATION_WEBHOOK="${NOTIFICATION_WEBHOOK:-}"
+
+# Parse DB configuration
+DATABASE_URL="${DATABASE_URL:-}"
+DB_TYPE="sqlite"
+if [[ "${DATABASE_URL}" =~ ^postgres ]]; then
+    DB_TYPE="postgres"
+fi
+
+# SQLite-specific configs
+SQLITE_PATH="${SQLITE_PATH:-data/translate.db}"
+# Ensure SQLite path is absolute
+if [[ ! "$SQLITE_PATH" =~ ^/ ]] && [[ "$SQLITE_PATH" =~ ^[a-zA-Z]: ]]; then
+    # Already absolute (Windows path format)
+    true
+else
+    # Convert to absolute path relative to project root
+    SQLITE_PATH="${SCRIPT_DIR}/../${SQLITE_PATH}"
+fi
+
+# PostgreSQL-specific configs
+POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-wordly-postgres}"
+POSTGRES_USER="${POSTGRES_USER:-translate}"
+POSTGRES_DB="${POSTGRES_DB:-translate_db}"
+POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-}"
+
+# ==============================================================================
+# 2. VALIDATION AND PREPARATION
+# ==============================================================================
+check_prerequisites() {
+    mkdir -p "${BACKUP_DIR}/daily"
+    mkdir -p "${BACKUP_DIR}/weekly"
+    mkdir -p "${BACKUP_DIR}/monthly"
+
+    if [ "${DB_TYPE}" = "postgres" ]; then
+        # Check Docker PostgreSQL container
+        if ! command -v docker &>/dev/null; then
+            log_error "Docker is not installed or not in PATH."
+            exit 1
+        fi
+        if ! docker ps --format '{{.Names}}' | grep -q "^${POSTGRES_CONTAINER}$"; then
+            log_error "PostgreSQL container '${POSTGRES_CONTAINER}' is not running."
+            exit 1
+        fi
+        if [ -z "${POSTGRES_PASSWORD}" ]; then
+            log_error "POSTGRES_PASSWORD is not set in environment or .env file."
+            exit 1
+        fi
+    else
+        # Check SQLite db file
+        local db_dir
+        db_dir=$(dirname "${SQLITE_PATH}")
+        mkdir -p "${db_dir}"
+        if [ ! -f "${SQLITE_PATH}" ]; then
+            log_warning "SQLite database file not found at '${SQLITE_PATH}'. It will be created on first write."
+        fi
+    fi
+}
+
+send_notification() {
+    local message="$1"
+    if [ -n "${NOTIFICATION_WEBHOOK}" ]; then
+        curl -s -X POST "${NOTIFICATION_WEBHOOK}" \
+            -H "Content-Type: application/json" \
+            -d "{\"text\": \"${message}\"}" >/dev/null 2>&1 || true
+    fi
+}
+
+# ==============================================================================
+# 3. BACKUP OPERATION
+# ==============================================================================
+create_backup() {
+    local backup_name
+    local dest_path
+    
+    if [ "${DB_TYPE}" = "postgres" ]; then
+        backup_name="wordly_postgres_${TIMESTAMP}.sql.gz"
+        dest_path="${BACKUP_DIR}/daily/${backup_name}"
+        log "Starting PostgreSQL backup for database '${POSTGRES_DB}'..."
+        
+        # Dump using pg_dump inside Docker container
+        if ! docker exec -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" pg_dump \
+            -U "${POSTGRES_USER}" \
+            -d "${POSTGRES_DB}" \
+            --format=custom \
+            --no-owner \
+            --no-acl \
+            2>/dev/null | gzip > "${dest_path}"; then
+            log_error "PostgreSQL pg_dump failed!"
+            send_notification "Wordly DB Backup FAILED: PostgreSQL dump error."
+            exit 1
+        fi
+    else
+        backup_name="wordly_sqlite_${TIMESTAMP}.db.gz"
+        dest_path="${BACKUP_DIR}/daily/${backup_name}"
+        log "Starting SQLite backup for file '${SQLITE_PATH}'..."
+
+        # If the database file does not exist yet (fresh install), touch it to allow backup
+        if [ ! -f "${SQLITE_PATH}" ]; then
+            touch "${SQLITE_PATH}"
+        fi
+
+        # Safe online backup using Python
+        local temp_db="${BACKUP_DIR}/daily/temp_${TIMESTAMP}.db"
+        if ! python -c "
+import sqlite3, sys
+try:
+    src = sqlite3.connect(r'${SQLITE_PATH}')
+    dst = sqlite3.connect(r'${temp_db}')
+    with dst:
+        src.backup(dst)
+    src.close()
+    dst.close()
+except Exception as e:
+    print(e, file=sys.stderr)
+    sys.exit(1)
+" 2>/dev/null; then
+            log_error "SQLite online backup failed! Checking if DB is locked."
+            send_notification "Wordly DB Backup FAILED: SQLite online backup script error."
+            exit 1
+        fi
+
+        # Compress the temporary copy
+        gzip -c "${temp_db}" > "${dest_path}"
+        rm -f "${temp_db}"
+    fi
+
+    # Verify backup size and integrity
+    if [ -f "${dest_path}" ] && [ -s "${dest_path}" ]; then
+        local size
+        size=$(du -h "${dest_path}" | cut -f1)
+        log_success "Backup file created successfully: ${backup_name} (${size})"
+        
+        # Verify archive integrity
+        if gzip -t "${dest_path}" 2>/dev/null; then
+            log_success "Backup integrity verification passed."
+        else
+            log_error "Backup archive integrity check FAILED!"
+            send_notification "Wordly DB Backup WARNING: Integrity check failed for ${backup_name}"
+            exit 1
+        fi
+
+        # Distribute to weekly/monthly folders
+        if [ "${DAY_OF_WEEK}" = "7" ]; then
+            cp "${dest_path}" "${BACKUP_DIR}/weekly/${backup_name}"
+            log "Weekly backup archived."
+        fi
+        if [ "${DAY_OF_MONTH}" = "01" ]; then
+            cp "${dest_path}" "${BACKUP_DIR}/monthly/${backup_name}"
+            log "Monthly backup archived."
+        fi
+        
+        send_notification "Wordly DB Backup SUCCESS: ${backup_name} (${size})"
+    else
+        log_error "Backup failed! Dest file is empty or missing."
+        send_notification "Wordly DB Backup FAILED: Destination archive is empty."
+        exit 1
+    fi
+}
+
+# ==============================================================================
+# 4. ROTATION & CLEANUP
+# ==============================================================================
+cleanup_old_backups() {
+    log "Running rotation cleanup policy..."
+
+    # Daily backups retention
+    local daily_count
+    daily_count=$(ls -1 "${BACKUP_DIR}/daily/" 2>/dev/null | wc -l || echo 0)
+    if [ "${daily_count}" -gt "${DAILY_RETENTION}" ]; then
+        ls -1t "${BACKUP_DIR}/daily/" | tail -n +$((DAILY_RETENTION + 1)) | while read -r f; do
+            rm -f "${BACKUP_DIR}/daily/${f}"
+            log "  Purged old daily backup: ${f}"
+        done
+    fi
+
+    # Weekly backups retention
+    local weekly_count
+    weekly_count=$(ls -1 "${BACKUP_DIR}/weekly/" 2>/dev/null | wc -l || echo 0)
+    if [ "${weekly_count}" -gt "${WEEKLY_RETENTION}" ]; then
+        ls -1t "${BACKUP_DIR}/weekly/" | tail -n +$((WEEKLY_RETENTION + 1)) | while read -r f; do
+            rm -f "${BACKUP_DIR}/weekly/${f}"
+            log "  Purged old weekly backup: ${f}"
+        done
+    fi
+
+    # Monthly backups retention
+    local monthly_count
+    monthly_count=$(ls -1 "${BACKUP_DIR}/monthly/" 2>/dev/null | wc -l || echo 0)
+    if [ "${monthly_count}" -gt "${MONTHLY_RETENTION}" ]; then
+        ls -1t "${BACKUP_DIR}/monthly/" | tail -n +$((MONTHLY_RETENTION + 1)) | while read -r f; do
+            rm -f "${BACKUP_DIR}/monthly/${f}"
+            log "  Purged old monthly backup: ${f}"
+        done
+    fi
+
+    log_success "Backup rotation cleanup complete."
+}
+
+# ==============================================================================
+# 5. RESTORE OPERATION
+# ==============================================================================
+restore_backup() {
+    local archive_name="$1"
+
+    if [ -z "${archive_name}" ]; then
+        log_error "No restore file specified."
+        echo "Usage: $0 --restore <archive_filename_only>"
+        echo "Available backups in ${BACKUP_DIR}:"
+        echo "--- Daily ---"
+        ls -lh "${BACKUP_DIR}/daily/" 2>/dev/null || echo "  (empty)"
+        echo "--- Weekly ---"
+        ls -lh "${BACKUP_DIR}/weekly/" 2>/dev/null || echo "  (empty)"
+        echo "--- Monthly ---"
+        ls -lh "${BACKUP_DIR}/monthly/" 2>/dev/null || echo "  (empty)"
+        exit 1
+    fi
+
+    # Locate archive in backup directories
+    local archive_path=""
+    for subdir in daily weekly monthly; do
+        if [ -f "${BACKUP_DIR}/${subdir}/${archive_name}" ]; then
+            archive_path="${BACKUP_DIR}/${subdir}/${archive_name}"
+            break
+        fi
+    done
+
+    if [ -z "${archive_path}" ]; then
+        # Try if it's a full path
+        if [ -f "${archive_name}" ]; then
+            archive_path="${archive_name}"
+        else
+            log_error "Restore archive file not found: ${archive_name}"
+            exit 1
+        fi
+    fi
+
+    # Safety prompt
+    echo ""
+    log_warning "RESTORE MODE - THIS WILL OVERWRITE YOUR ACTIVE DATABASE!"
+    echo "  Archive Source: ${archive_path}"
+    echo "  Database Type : ${DB_TYPE}"
+    if [ "${DB_TYPE}" = "postgres" ]; then
+        echo "  Target DB     : PostgreSQL Database '${POSTGRES_DB}'"
+    else
+        echo "  Target File   : SQLite File '${SQLITE_PATH}'"
+    fi
+    echo ""
+    read -p "To confirm restore, type 'RESTORE': " confirm_val
+    if [ "${confirm_val}" != "RESTORE" ]; then
+        log "Restore cancelled."
+        exit 0
+    fi
+
+    log "Initiating restore sequence..."
+
+    # Create safety backup of the active database before restoring
+    log "Creating safety backup of the current database state..."
+    local safety_name
+    if [ "${DB_TYPE}" = "postgres" ]; then
+        safety_name="wordly_postgres_pre_restore_${TIMESTAMP}.sql.gz"
+        docker exec -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" pg_dump \
+            -U "${POSTGRES_USER}" \
+            -d "${POSTGRES_DB}" \
+            --format=custom \
+            --no-owner \
+            --no-acl \
+            2>/dev/null | gzip > "${BACKUP_DIR}/daily/${safety_name}"
+    else
+        safety_name="wordly_sqlite_pre_restore_${TIMESTAMP}.db.gz"
+        if [ -f "${SQLITE_PATH}" ]; then
+            gzip -c "${SQLITE_PATH}" > "${BACKUP_DIR}/daily/${safety_name}"
+        else
+            touch "${SQLITE_PATH}"
+            gzip -c "${SQLITE_PATH}" > "${BACKUP_DIR}/daily/${safety_name}"
+        fi
+    fi
+    log_success "Safety pre-restore backup created: ${safety_name}"
+
+    # Restore DB
+    if [ "${DB_TYPE}" = "postgres" ]; then
+        log "Restoring PostgreSQL database..."
+        if ! gunzip -c "${archive_path}" | docker exec -i -e PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_CONTAINER}" \
+            pg_restore \
+            -U "${POSTGRES_USER}" \
+            -d "${POSTGRES_DB}" \
+            --clean \
+            --if-exists \
+            --no-owner \
+            --no-acl; then
+            log_error "PostgreSQL restore failed!"
+            exit 1
+        fi
+    else
+        log "Restoring SQLite database file..."
+        
+        # Decompress SQLite backup to a temp file
+        local temp_restore_db="${BACKUP_DIR}/daily/temp_restore_${TIMESTAMP}.db"
+        gunzip -c "${archive_path}" > "${temp_restore_db}"
+        
+        # Overwrite database safely using Python (ensures open handles or locks are respected)
+        if ! python -c "
+import sqlite3, sys
+try:
+    src = sqlite3.connect(r'${temp_restore_db}')
+    dst = sqlite3.connect(r'${SQLITE_PATH}')
+    with dst:
+        src.backup(dst)
+    src.close()
+    dst.close()
+except Exception as e:
+    print(e, file=sys.stderr)
+    sys.exit(1)
+" 2>/dev/null; then
+            log_error "SQLite restore swap operation failed!"
+            rm -f "${temp_restore_db}"
+            exit 1
+        fi
+        
+        rm -f "${temp_restore_db}"
+    fi
+
+    log_success "Restore process completed successfully!"
+    log_warning "You should restart your app container to clear active connection caches:"
+    echo "  docker compose restart backend"
+}
+
+# ==============================================================================
+# 6. ENTRYS
+# ==============================================================================
+main() {
+    case "${1:-}" in
+        --restore)
+            check_prerequisites
+            restore_backup "${2:-}"
+            ;;
+        --full)
+            check_prerequisites
+            create_backup
+            cleanup_old_backups
+            log_success "Full backup and rotation sequence completed."
+            ;;
+        *)
+            check_prerequisites
+            create_backup
+            cleanup_old_backups
+            log_success "Standard backup completed."
+            ;;
+    esac
+}
+
+main "$@"