From ba1547c4a6a60a316bb22937b25dc2c7a220bf4f Mon Sep 17 00:00:00 2001
From: sclfcz <hepushuo@gmail.com>
Date: Wed, 7 Oct 2026 09:36:13 +0800
Subject: [PATCH] fix: stop suppressing CJK quotation marks as non-speech
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The default --suppress_tokens=-1 extends tokenizer.non_speech_tokens, which
included 「」『』. Those are ordinary Chinese and Japanese quotation marks, not
speaker tags or non-speech annotations, so every CLI run silently dropped the
quotes from the transcript. Keep the symbols that actually mark non-speech.
---
 tests/test_tokenizer.py | 24 ++++++++++++++++++++++++
 whisper/tokenizer.py    |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index be424e5fe..b342e9adb 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -32,3 +32,27 @@ def test_split_on_unicode():
 
     assert words == [" elle", " est", " l", "'", "\ufffd", "é", "rit", "oire"]
     assert word_tokens == [[8404], [871], [287], [6], [246], [526], [3210], [20378]]
+
+
+def test_cjk_quotation_marks_are_not_suppressed():
+    """「」『』 are ordinary CJK punctuation, not speaker tags.
+
+    They are in the default `--suppress_tokens=-1` set, so every CLI run silently
+    dropped the quotation marks from Chinese and Japanese output.
+    """
+    tokenizer = get_tokenizer(multilingual=True, language="zh", task="transcribe")
+    non_speech = set(tokenizer.non_speech_tokens)
+
+    for char in "「」『』":
+        tokens = tokenizer.encoding.encode(char)
+        assert not any(token in non_speech for token in tokens), char
+
+
+def test_non_speech_symbols_are_still_suppressed():
+    """Control: the suppression set keeps doing its actual job."""
+    tokenizer = get_tokenizer(multilingual=True, language="zh", task="transcribe")
+    non_speech = set(tokenizer.non_speech_tokens)
+
+    for char in "♪#()*":
+        tokens = tokenizer.encoding.encode(char)
+        assert any(token in non_speech for token in tokens), char
diff --git a/whisper/tokenizer.py b/whisper/tokenizer.py
index 2af837570..ee6d55f0f 100644
--- a/whisper/tokenizer.py
+++ b/whisper/tokenizer.py
@@ -250,7 +250,7 @@ def non_speech_tokens(self) -> Tuple[int]:
 
         keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
         """
-        symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』')
+        symbols = list('"#()*+/:;<=>@[\\]^_`{|}~')
         symbols += (
             "<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split()
         )