Fix get_tokenizer rejecting BCP-47 locale tags like en-US

tests/test_tokenizer.py

@@ -32,3 +32,34 @@ def test_split_on_unicode():
 
     assert words == [" elle", " est", " l", "'", "\ufffd", "é", "rit", "oire"]
     assert word_tokens == [[8404], [871], [287], [6], [246], [526], [3210], [20378]]
+
+
+@pytest.mark.parametrize(
+    "language, expected",
+    [
+        ("en", "en"),
+        ("EN", "en"),
+        ("english", "en"),
+        ("en-US", "en"),
+        ("en_GB", "en"),
+        ("zh-CN", "zh"),
+        ("zh_Hans", "zh"),
+        ("pt-BR", "pt"),
+        ("ko-KR", "ko"),
+        ("mandarin", "zh"),
+    ],
+)
+def test_language_locale_tags(language, expected):
+    tokenizer = get_tokenizer(multilingual=True, language=language)
+    assert tokenizer.language == expected
+
+
+def test_language_locale_tags_rejected_when_unknown():
+    with pytest.raises(ValueError, match="Unsupported language: xx-us"):
+        get_tokenizer(multilingual=True, language="xx-US")
+
+
+def test_english_only_tokenizer_accepts_locale_tags():
+    # English-only models ignore the language, but a locale tag should not raise.
+    tokenizer = get_tokenizer(multilingual=False, language="en-US")
+    assert tokenizer.language is None

whisper/tokenizer.py

@@ -363,6 +363,24 @@ def get_encoding(name: str = "gpt2", num_languages: int = 99):
     )
 
 
+def _normalize_language(language: str) -> str:
+    """Resolve a language name, ISO code, or locale tag to a Whisper language code."""
+    language = language.lower()
+    if language in LANGUAGES:
+        return language
+    if language in TO_LANGUAGE_CODE:
+        return TO_LANGUAGE_CODE[language]
+
+    # BCP-47 / locale tags such as "en-US" or "zh_CN"
+    primary = language.replace("_", "-").split("-")[0]
+    if primary in LANGUAGES:
+        return primary
+    if primary in TO_LANGUAGE_CODE:
+        return TO_LANGUAGE_CODE[primary]
+
+    raise ValueError(f"Unsupported language: {language}")
+
+
 @lru_cache(maxsize=None)
 def get_tokenizer(
     multilingual: bool,
@@ -372,12 +390,7 @@ def get_tokenizer(
     task: Optional[str] = None,  # Literal["transcribe", "translate", None]
 ) -> Tokenizer:
     if language is not None:
-        language = language.lower()
-        if language not in LANGUAGES:
-            if language in TO_LANGUAGE_CODE:
-                language = TO_LANGUAGE_CODE[language]
-            else:
-                raise ValueError(f"Unsupported language: {language}")
+        language = _normalize_language(language)
 
     if multilingual:
         encoding_name = "multilingual"