office_translator/services/quality/config.py

"""
Language → script mapping for the L0 quality layer.

A "script" groups languages that share a Unicode block. The script detector
verifies that the *translation* is in the same script as the *target language*.

Languages are mapped by their ISO 639-1 (or 639-3) code. Anything not mapped
falls back to the "latin" script so we never crash on an unknown language.

Arabic-script languages (ar, fa, ur, ps, ku, sd, ug, yi, dv, ckb) all share
the same Unicode ranges, so we additionally use a set of *discriminating
characters* to tell them apart — e.g. Persian text contains چ/پ/ژ/گ which
Arabic text doesn't.
"""

from typing import Dict, FrozenSet, List, Optional, Tuple


# ---------- Unicode ranges per script ----------
# Format: list of (start, end) inclusive ranges.
# Latin is intentionally omitted (it's the fallback).

UNICODE_RANGES: Dict[str, List[Tuple[int, int]]] = {
    "cyrillic": [
        (0x0400, 0x04FF),   # Cyrillic
        (0x0500, 0x052F),   # Cyrillic Supplement
    ],
    "greek": [
        (0x0370, 0x03FF),   # Greek and Coptic
    ],
    "arabic": [
        (0x0600, 0x06FF),   # Arabic
        (0x0750, 0x077F),   # Arabic Supplement
        (0x08A0, 0x08FF),   # Arabic Extended-A
    ],
    "hebrew": [
        (0x0590, 0x05FF),   # Hebrew
    ],
    "devanagari": [
        (0x0900, 0x097F),   # Devanagari
    ],
    "bengali": [
        (0x0980, 0x09FF),   # Bengali
    ],
    "tamil": [
        (0x0B80, 0x0BFF),
    ],
    "telugu": [
        (0x0C00, 0x0C7F),
    ],
    "kannada": [
        (0x0C80, 0x0CFF),
    ],
    "malayalam": [
        (0x0D00, 0x0D7F),
    ],
    "sinhala": [
        (0x0D80, 0x0DFF),
    ],
    "gujarati": [
        (0x0A80, 0x0AFF),
    ],
    "gurmukhi": [
        (0x0A00, 0x0A7F),
    ],
    "thai": [
        (0x0E00, 0x0E7F),
    ],
    "lao": [
        (0x0E80, 0x0EFF),
    ],
    "burmese": [
        (0x1000, 0x109F),
    ],
    "khmer": [
        (0x1780, 0x17FF),
    ],
    "cjk": [
        (0x4E00, 0x9FFF),   # CJK Unified Ideographs
        (0x3400, 0x4DBF),   # CJK Extension A
        (0x20000, 0x2A6DF), # CJK Extension B (rare)
    ],
    "hiragana_katakana": [
        (0x3040, 0x309F),   # Hiragana
        (0x30A0, 0x30FF),   # Katakana
    ],
    "hangul": [
        (0xAC00, 0xD7AF),   # Hangul Syllables
        (0x1100, 0x11FF),   # Hangul Jamo
        (0xA960, 0xA97F),   # Hangul Jamo Extended-A
    ],
    "georgian": [
        (0x10A0, 0x10FF),
    ],
    "armenian": [
        (0x0530, 0x058F),
    ],
    "ethiopic": [
        (0x1200, 0x137F),   # Ethiopic
        (0x1380, 0x139F),   # Ethiopic Supplement
    ],
    "tibetan": [
        (0x0F00, 0x0FFF),
    ],
    "thaana": [
        (0x0780, 0x07BF),   # Thaana (Dhivehi)
    ],
    # Latin is the implicit fallback.
    "latin": [],
}


# ---------- Language → script mapping ----------
# Single source of truth. Keys are lower-case ISO codes.

LANG_TO_SCRIPT: Dict[str, str] = {
    # Latin-script languages (the long tail of European + colonial)
    "en": "latin", "fr": "latin", "de": "latin", "es": "latin", "it": "latin",
    "pt": "latin", "nl": "latin", "pl": "latin", "tr": "latin", "vi": "latin",
    "id": "latin", "ms": "latin", "ro": "latin", "cs": "latin", "sv": "latin",
    "da": "latin", "fi": "latin", "no": "latin", "nb": "latin", "nn": "latin",
    "hu": "latin", "sk": "latin", "sl": "latin", "lt": "latin", "lv": "latin",
    "et": "latin", "sq": "latin", "az": "latin", "uz": "latin", "kk": "latin",
    "ky": "latin", "tk": "latin", "sw": "latin", "eu": "latin", "gl": "latin",
    "is": "latin", "ga": "latin", "mt": "latin", "ca": "latin", "hr": "latin",
    "bs": "latin", "sr": "latin", "mk": "latin", "bg": "latin", "be": "latin",
    "uk": "latin",  # NOTE: uk/be/sr/mk/bg are actually Cyrillic — see fix below
    "af": "latin", "cy": "latin", "lb": "latin", "fo": "latin", "br": "latin",
    "co": "latin", "fy": "latin", "gd": "latin", "gn": "latin", "gu": "latin",
    "ht": "latin", "haw": "latin", "hmn": "latin", "jv": "latin", "ku": "latin",
    "mg": "latin", "mi": "latin", "mn": "latin", "nso": "latin", "ny": "latin",
    "oc": "latin", "os": "latin", "ps": "latin", "qu": "latin", "rw": "latin",
    "sc": "latin", "si": "latin", "sm": "latin", "sn": "latin", "so": "latin",
    "st": "latin", "su": "latin", "tg": "latin", "tt": "latin", "ty": "latin",
    "ug": "latin", "vo": "latin", "wa": "latin", "wo": "latin", "xh": "latin",
    "yi": "latin", "zu": "latin",
}

# Correct Cyrillic assignments (overrides above for the Cyrillic-script langs)
LANG_TO_SCRIPT.update({
    "ru": "cyrillic", "uk": "cyrillic", "be": "cyrillic", "sr": "cyrillic",
    "mk": "cyrillic", "bg": "cyrillic", "kk": "cyrillic", "ky": "cyrillic",
    "tg": "cyrillic", "tt": "cyrillic", "mn": "cyrillic", "ab": "cyrillic",
    "ba": "cyrillic", "ce": "cyrillic", "cv": "cyrillic", "kv": "cyrillic",
    "kv": "cyrillic", "l1": "cyrillic", "mhr": "cyrillic", "mrj": "cyrillic",
    "myv": "cyrillic", "os": "cyrillic", "rue": "cyrillic", "sah": "cyrillic",
    "udm": "cyrillic", "uk": "cyrillic",
})

# More corrections
LANG_TO_SCRIPT.update({
    "el": "greek",
    "ar": "arabic", "fa": "arabic", "ur": "arabic", "ps": "arabic",
    "ku": "arabic", "sd": "arabic", "ug": "arabic",
    "ckb": "arabic", "bal": "arabic", "bqi": "arabic",
    "glk": "arabic", "mzn": "arabic",
    "he": "hebrew",
    "yi": "hebrew",   # Yiddish uses Hebrew script
    "dv": "thaana",   # Maldivian (Dhivehi) uses Thaana script
    "hi": "devanagari", "ne": "devanagari", "mr": "devanagari", "sa": "devanagari",
    "mai": "devanagari", "bho": "devanagari", "awa": "devanagari",
    "bn": "bengali", "as": "bengali",
    "ta": "tamil",
    "te": "telugu",
    "kn": "kannada",
    "ml": "malayalam",
    "si": "sinhala",
    "gu": "gujarati",
    "pa": "gurmukhi",
    "th": "thai",
    "lo": "lao",
    "my": "burmese",
    "km": "khmer",
    "zh": "cjk", "zh-cn": "cjk", "zh-tw": "cjk", "zh-hk": "cjk",
    "ja": "hiragana_katakana",
    "ko": "hangul",
    "ka": "georgian",
    "hy": "armenian",
    "am": "ethiopic", "ti": "ethiopic", "gez": "ethiopic", "tig": "ethiopic",
    "bo": "tibetan", "dz": "tibetan",
})


# ---------- Discriminating characters for Arabic-script languages ----------
# These are characters that strongly suggest a SPECIFIC Arabic-script
# language as opposed to the generic "arabic" base.

DISCRIMINATING_CHARS: Dict[str, FrozenSet[str]] = {
    "fa": frozenset("پچژگ"),                       # Persian
    "ur": frozenset("ٹڈڑے"),                       # Urdu
    "ps": frozenset("ټډړږښ"),                       # Pashto
    "ku": frozenset("ڕێ"),                          # Kurdish
    "ckb": frozenset("ڕێ"),                         # Sorani Kurdish (same as ku)
    "sd": frozenset("ٿ"),                          # Sindhi
    "ug": frozenset("ۇۆې"),                         # Uyghur
    "yi": frozenset("ײ"),                          # Yiddish (uses Hebrew too)
    "bal": frozenset("ێ"),                         # Balochi
}


# ---------- Thresholds ----------
# These are tunable but with sensible defaults.

MIN_RATIO_IN_SCRIPT = 0.60        # at least 60% of letters must match the script
MIN_RATIO_FOR_ARABIC_VARIANT = 0.20  # at least 20% of letters in the discriminating set


def get_script(lang_code: Optional[str]) -> str:
    """
    Return the script id (e.g. 'cyrillic', 'cjk') for a given language code.
    Returns 'latin' for unknown codes (which is the safe default — Latin
    covers the largest number of languages).
    """
    if not lang_code:
        return "latin"
    return LANG_TO_SCRIPT.get(lang_code.lower(), "latin")


def is_arabic_script_lang(lang_code: Optional[str]) -> bool:
    """True if lang_code is one of the Arabic-script languages we discriminate."""
    if not lang_code:
        return False
    return get_script(lang_code) == "arabic"


def get_ranges(script_id: str) -> List[Tuple[int, int]]:
    """Return the Unicode ranges for a script id. Empty list for 'latin'."""
    return UNICODE_RANGES.get(script_id, [])


def get_discriminating_chars(lang_code: Optional[str]) -> FrozenSet[str]:
    """Return the discriminating characters for a specific Arabic-script language."""
    if not lang_code:
        return frozenset()
    return DISCRIMINATING_CHARS.get(lang_code.lower(), frozenset())