150 lines
4.2 KiB
Python
150 lines
4.2 KiB
Python
import os
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from typing import Optional, Any, Dict
|
|
from config import config
|
|
|
|
try:
|
|
import structlog
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
_HAS_STRUCTLOG = True
|
|
except ImportError:
|
|
logger = logging.getLogger(__name__)
|
|
_HAS_STRUCTLOG = False
|
|
|
|
|
|
def _log_info(event: str, **kwargs):
|
|
"""Log info with structlog or standard logging compatibility."""
|
|
if _HAS_STRUCTLOG:
|
|
logger.info(event, **kwargs)
|
|
else:
|
|
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
|
|
logger.info(msg)
|
|
|
|
|
|
def _log_error(event: str, **kwargs):
|
|
"""Log error with structlog or standard logging compatibility."""
|
|
if _HAS_STRUCTLOG:
|
|
logger.error(event, **kwargs)
|
|
else:
|
|
msg = f"{event} " + " ".join(f"{k}={v}" for k, v in kwargs.items())
|
|
logger.error(msg)
|
|
|
|
|
|
# Key pattern: translation:file:{job_id}
|
|
KEY_PREFIX = "translation:file"
|
|
|
|
|
|
def _get_default_ttl() -> int:
|
|
"""Get TTL from config or default to 60 minutes."""
|
|
try:
|
|
from config import config
|
|
|
|
return config.FILE_TTL_MINUTES * 60
|
|
except Exception:
|
|
return 3600 # 60 minutes default
|
|
|
|
|
|
_async_redis = None
|
|
|
|
|
|
def _get_async_redis():
|
|
"""Return async Redis client or None. Uses REDIS_URL from env."""
|
|
global _async_redis
|
|
if _async_redis is not None:
|
|
return _async_redis if _async_redis is not False else None
|
|
|
|
# Try to get from environment first
|
|
url = os.getenv("REDIS_URL", "").strip()
|
|
if not url:
|
|
_async_redis = False
|
|
return None
|
|
|
|
try:
|
|
import redis.asyncio as redis
|
|
|
|
_async_redis = redis.Redis.from_url(url, decode_responses=True)
|
|
_log_info("redis_connected", service="storage_tracker")
|
|
return _async_redis
|
|
except Exception as e:
|
|
_log_error("redis_connection_failed", service="storage_tracker", error=str(e))
|
|
_async_redis = False
|
|
return None
|
|
|
|
|
|
class StorageTracker:
|
|
"""
|
|
Tracks file locations and metadata in Redis.
|
|
Pattern: translation:file:{job_id} -> JSON metadata
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._redis = None
|
|
|
|
def _redis_client(self):
|
|
if self._redis is None:
|
|
self._redis = _get_async_redis()
|
|
return self._redis
|
|
|
|
async def track_file(
|
|
self, job_id: str, metadata: Dict[str, Any], ttl: Optional[int] = None
|
|
) -> bool:
|
|
"""
|
|
Store file metadata in Redis with TTL and log the upload.
|
|
"""
|
|
if ttl is None:
|
|
ttl = _get_default_ttl()
|
|
|
|
# Ensure timestamp is present
|
|
if "timestamp" not in metadata:
|
|
metadata["timestamp"] = datetime.now(timezone.utc).isoformat()
|
|
|
|
# Log metadata (no content)
|
|
_log_info(
|
|
"file_uploaded",
|
|
job_id=job_id,
|
|
original_filename=metadata.get("original_filename"),
|
|
file_size=metadata.get("file_size"),
|
|
file_hash=metadata.get("file_hash"),
|
|
user_id=metadata.get("user_id"),
|
|
timestamp=metadata.get("timestamp"),
|
|
)
|
|
|
|
redis_client = self._redis_client()
|
|
if not redis_client:
|
|
_log_error(
|
|
"redis_not_available", job_id=job_id, hint="File tracked in logs only"
|
|
)
|
|
return False
|
|
|
|
try:
|
|
key = f"{KEY_PREFIX}:{job_id}"
|
|
await redis_client.set(key, json.dumps(metadata), ex=ttl)
|
|
_log_info("file_tracked_in_redis", job_id=job_id, ttl_seconds=ttl)
|
|
return True
|
|
except Exception as e:
|
|
_log_error("redis_track_failed", job_id=job_id, error=str(e))
|
|
return False
|
|
|
|
async def get_file_metadata(self, job_id: str) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Retrieve file metadata from Redis.
|
|
"""
|
|
redis_client = self._redis_client()
|
|
if not redis_client:
|
|
return None
|
|
|
|
try:
|
|
key = f"{KEY_PREFIX}:{job_id}"
|
|
data = await redis_client.get(key)
|
|
return json.loads(data) if data else None
|
|
except Exception as e:
|
|
_log_error("redis_get_failed", job_id=job_id, error=str(e))
|
|
return None
|
|
|
|
|
|
# Singleton for app use
|
|
storage_tracker = StorageTracker()
|