""" Language → script mapping for the L0 quality layer. A "script" groups languages that share a Unicode block. The script detector verifies that the *translation* is in the same script as the *target language*. Languages are mapped by their ISO 639-1 (or 639-3) code. Anything not mapped falls back to the "latin" script so we never crash on an unknown language. Arabic-script languages (ar, fa, ur, ps, ku, sd, ug, yi, dv, ckb) all share the same Unicode ranges, so we additionally use a set of *discriminating characters* to tell them apart — e.g. Persian text contains چ/پ/ژ/گ which Arabic text doesn't. """ from typing import Dict, FrozenSet, List, Optional, Tuple # ---------- Unicode ranges per script ---------- # Format: list of (start, end) inclusive ranges. # Latin is intentionally omitted (it's the fallback). UNICODE_RANGES: Dict[str, List[Tuple[int, int]]] = { "cyrillic": [ (0x0400, 0x04FF), # Cyrillic (0x0500, 0x052F), # Cyrillic Supplement ], "greek": [ (0x0370, 0x03FF), # Greek and Coptic ], "arabic": [ (0x0600, 0x06FF), # Arabic (0x0750, 0x077F), # Arabic Supplement (0x08A0, 0x08FF), # Arabic Extended-A ], "hebrew": [ (0x0590, 0x05FF), # Hebrew ], "devanagari": [ (0x0900, 0x097F), # Devanagari ], "bengali": [ (0x0980, 0x09FF), # Bengali ], "tamil": [ (0x0B80, 0x0BFF), ], "telugu": [ (0x0C00, 0x0C7F), ], "kannada": [ (0x0C80, 0x0CFF), ], "malayalam": [ (0x0D00, 0x0D7F), ], "sinhala": [ (0x0D80, 0x0DFF), ], "gujarati": [ (0x0A80, 0x0AFF), ], "gurmukhi": [ (0x0A00, 0x0A7F), ], "thai": [ (0x0E00, 0x0E7F), ], "lao": [ (0x0E80, 0x0EFF), ], "burmese": [ (0x1000, 0x109F), ], "khmer": [ (0x1780, 0x17FF), ], "cjk": [ (0x4E00, 0x9FFF), # CJK Unified Ideographs (0x3400, 0x4DBF), # CJK Extension A (0x20000, 0x2A6DF), # CJK Extension B (rare) ], "hiragana_katakana": [ (0x3040, 0x309F), # Hiragana (0x30A0, 0x30FF), # Katakana ], "hangul": [ (0xAC00, 0xD7AF), # Hangul Syllables (0x1100, 0x11FF), # Hangul Jamo (0xA960, 0xA97F), # Hangul Jamo Extended-A ], "georgian": [ (0x10A0, 0x10FF), ], "armenian": [ (0x0530, 0x058F), ], "ethiopic": [ (0x1200, 0x137F), # Ethiopic (0x1380, 0x139F), # Ethiopic Supplement ], "tibetan": [ (0x0F00, 0x0FFF), ], "thaana": [ (0x0780, 0x07BF), # Thaana (Dhivehi) ], # Latin is the implicit fallback. "latin": [], } # ---------- Language → script mapping ---------- # Single source of truth. Keys are lower-case ISO codes. LANG_TO_SCRIPT: Dict[str, str] = { # Latin-script languages (the long tail of European + colonial) "en": "latin", "fr": "latin", "de": "latin", "es": "latin", "it": "latin", "pt": "latin", "nl": "latin", "pl": "latin", "tr": "latin", "vi": "latin", "id": "latin", "ms": "latin", "ro": "latin", "cs": "latin", "sv": "latin", "da": "latin", "fi": "latin", "no": "latin", "nb": "latin", "nn": "latin", "hu": "latin", "sk": "latin", "sl": "latin", "lt": "latin", "lv": "latin", "et": "latin", "sq": "latin", "az": "latin", "uz": "latin", "kk": "latin", "ky": "latin", "tk": "latin", "sw": "latin", "eu": "latin", "gl": "latin", "is": "latin", "ga": "latin", "mt": "latin", "ca": "latin", "hr": "latin", "bs": "latin", "sr": "latin", "mk": "latin", "bg": "latin", "be": "latin", "uk": "latin", # NOTE: uk/be/sr/mk/bg are actually Cyrillic — see fix below "af": "latin", "cy": "latin", "lb": "latin", "fo": "latin", "br": "latin", "co": "latin", "fy": "latin", "gd": "latin", "gn": "latin", "gu": "latin", "ht": "latin", "haw": "latin", "hmn": "latin", "jv": "latin", "ku": "latin", "mg": "latin", "mi": "latin", "mn": "latin", "nso": "latin", "ny": "latin", "oc": "latin", "os": "latin", "ps": "latin", "qu": "latin", "rw": "latin", "sc": "latin", "si": "latin", "sm": "latin", "sn": "latin", "so": "latin", "st": "latin", "su": "latin", "tg": "latin", "tt": "latin", "ty": "latin", "ug": "latin", "vo": "latin", "wa": "latin", "wo": "latin", "xh": "latin", "yi": "latin", "zu": "latin", } # Correct Cyrillic assignments (overrides above for the Cyrillic-script langs) LANG_TO_SCRIPT.update({ "ru": "cyrillic", "uk": "cyrillic", "be": "cyrillic", "sr": "cyrillic", "mk": "cyrillic", "bg": "cyrillic", "kk": "cyrillic", "ky": "cyrillic", "tg": "cyrillic", "tt": "cyrillic", "mn": "cyrillic", "ab": "cyrillic", "ba": "cyrillic", "ce": "cyrillic", "cv": "cyrillic", "kv": "cyrillic", "kv": "cyrillic", "l1": "cyrillic", "mhr": "cyrillic", "mrj": "cyrillic", "myv": "cyrillic", "os": "cyrillic", "rue": "cyrillic", "sah": "cyrillic", "udm": "cyrillic", "uk": "cyrillic", }) # More corrections LANG_TO_SCRIPT.update({ "el": "greek", "ar": "arabic", "fa": "arabic", "ur": "arabic", "ps": "arabic", "ku": "arabic", "sd": "arabic", "ug": "arabic", "ckb": "arabic", "bal": "arabic", "bqi": "arabic", "glk": "arabic", "mzn": "arabic", "he": "hebrew", "yi": "hebrew", # Yiddish uses Hebrew script "dv": "thaana", # Maldivian (Dhivehi) uses Thaana script "hi": "devanagari", "ne": "devanagari", "mr": "devanagari", "sa": "devanagari", "mai": "devanagari", "bho": "devanagari", "awa": "devanagari", "bn": "bengali", "as": "bengali", "ta": "tamil", "te": "telugu", "kn": "kannada", "ml": "malayalam", "si": "sinhala", "gu": "gujarati", "pa": "gurmukhi", "th": "thai", "lo": "lao", "my": "burmese", "km": "khmer", "zh": "cjk", "zh-cn": "cjk", "zh-tw": "cjk", "zh-hk": "cjk", "ja": "hiragana_katakana", "ko": "hangul", "ka": "georgian", "hy": "armenian", "am": "ethiopic", "ti": "ethiopic", "gez": "ethiopic", "tig": "ethiopic", "bo": "tibetan", "dz": "tibetan", }) # ---------- Discriminating characters for Arabic-script languages ---------- # These are characters that strongly suggest a SPECIFIC Arabic-script # language as opposed to the generic "arabic" base. DISCRIMINATING_CHARS: Dict[str, FrozenSet[str]] = { "fa": frozenset("پچژگ"), # Persian "ur": frozenset("ٹڈڑے"), # Urdu "ps": frozenset("ټډړږښ"), # Pashto "ku": frozenset("ڕێ"), # Kurdish "ckb": frozenset("ڕێ"), # Sorani Kurdish (same as ku) "sd": frozenset("ٿ"), # Sindhi "ug": frozenset("ۇۆې"), # Uyghur "yi": frozenset("ײ"), # Yiddish (uses Hebrew too) "bal": frozenset("ێ"), # Balochi } # ---------- Thresholds ---------- # These are tunable but with sensible defaults. MIN_RATIO_IN_SCRIPT = 0.60 # at least 60% of letters must match the script MIN_RATIO_FOR_ARABIC_VARIANT = 0.20 # at least 20% of letters in the discriminating set def get_script(lang_code: Optional[str]) -> str: """ Return the script id (e.g. 'cyrillic', 'cjk') for a given language code. Returns 'latin' for unknown codes (which is the safe default — Latin covers the largest number of languages). """ if not lang_code: return "latin" return LANG_TO_SCRIPT.get(lang_code.lower(), "latin") def is_arabic_script_lang(lang_code: Optional[str]) -> bool: """True if lang_code is one of the Arabic-script languages we discriminate.""" if not lang_code: return False return get_script(lang_code) == "arabic" def get_ranges(script_id: str) -> List[Tuple[int, int]]: """Return the Unicode ranges for a script id. Empty list for 'latin'.""" return UNICODE_RANGES.get(script_id, []) def get_discriminating_chars(lang_code: Optional[str]) -> FrozenSet[str]: """Return the discriminating characters for a specific Arabic-script language.""" if not lang_code: return frozenset() return DISCRIMINATING_CHARS.get(lang_code.lower(), frozenset())