From 36ee9688f88023ae2b7daf68b94c899804b8bcbd Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 00:32:11 +0700 Subject: [PATCH 01/12] Add PhayaThaiBERT POS tagger engine Add the nlp-chula/phayathaibert-thai-pos-tagger model, fine-tuned on the TUD treebank, as a Universal POS tagger: - pos_tag(engine="phayathaibert") runs an ONNX export of the model with ONNX Runtime and the tokenizers library, without PyTorch. The model is downloaded from the Hugging Face Hub, pinned to a commit. Whitespace-only words are tagged PUNCT, and input longer than the model limit is tagged in chunks of whole words. - pos_tag_transformers(engine="phayathaibert", corpus="tud") runs the original model with Transformers. The per-corpus engine tables are merged into one lookup. Add the phayathaibert_onnx optional dependency group, tests, and API documentation. Refs #1536 --- CHANGELOG.md | 3 + docs/api/tag.rst | 22 +++ pyproject.toml | 11 +- pythainlp/tag/phayathaibert_onnx.py | 215 ++++++++++++++++++++++++++ pythainlp/tag/pos_tag.py | 64 +++++--- tests/README.md | 1 + tests/core/test_tag.py | 66 ++++++++ tests/noauto_onnx/testn_tag_onnx.py | 58 +++++++ tests/noauto_torch/testn_tag_torch.py | 34 ++++ 9 files changed, 448 insertions(+), 26 deletions(-) create mode 100644 pythainlp/tag/phayathaibert_onnx.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d04339fb7..1c837caff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ and this project adheres to ([#1509]) - `evaluate_word_tokenization`, `word_eval_function`, `char_eval_function`, and `TokenizationScore` in `pythainlp.benchmarks` ([#1512]) +- `pos_tag()` engine `phayathaibert` (ONNX) and `pos_tag_transformers()` + corpus `tud`: PhayaThaiBERT Universal POS tagger trained on TUD ([#1536]) ### Changed @@ -86,6 +88,7 @@ and this project adheres to [#1512]: https://github.com/PyThaiNLP/pythainlp/pull/1512 [#1526]: https://github.com/PyThaiNLP/pythainlp/pull/1526 [#1529]: https://github.com/PyThaiNLP/pythainlp/pull/1529 +[#1536]: https://github.com/PyThaiNLP/pythainlp/issues/1536 [#1541]: https://github.com/PyThaiNLP/pythainlp/pull/1541 [#1542]: https://github.com/PyThaiNLP/pythainlp/pull/1542 [#1546]: https://github.com/PyThaiNLP/pythainlp/pull/1546 diff --git a/docs/api/tag.rst b/docs/api/tag.rst index 46203fa4e..5f1415752 100644 --- a/docs/api/tag.rst +++ b/docs/api/tag.rst @@ -228,6 +228,7 @@ Modules .. autofunction:: pos_tag .. autofunction:: pos_tag_sents +.. autofunction:: pos_tag_transformers .. autofunction:: tag_provinces .. autofunction:: chunk_parse .. autoclass:: NER @@ -251,6 +252,23 @@ unigram Unigram tagger doesn't take the ordering of words in the list into account. +phayathaibert ++++++++++++++ + +PhayaThaiBERT tagger [#Sriwirote_2025]_ uses +`nlp-chula/phayathaibert-thai-pos-tagger `_, +fine-tuned on the TUD treebank, and tags words with Universal POS tags. + +- :func:`pos_tag` with ``engine="phayathaibert"`` runs an ONNX export of the + model with ONNX Runtime and does not need PyTorch. + Install the dependencies with ``pip install "pythainlp[phayathaibert_onnx]"``. + The model (about 530 MB) is downloaded from the Hugging Face Hub on first use. +- :func:`pos_tag_transformers` with ``engine="phayathaibert"`` and + ``corpus="tud"`` runs the original model with Transformers. + +.. autoclass:: pythainlp.tag.phayathaibert_onnx.PhayaThaiBERTTagger + :members: tag + References ---------- @@ -261,3 +279,7 @@ References .. [#Prachya_2020] Prachya Boonkwan and Vorapon Luantangsrisuk and Sitthaa Phaholphinyo and Kanyanat Kriengket and Dhanon Leenoi and Charun Phrombut and Monthika Boriboon and Krit Kosawat and Thepchai Supnithi. (2020). The Annotation Guideline of LST20 Corpus. arXiv:2008.05055 +.. [#Sriwirote_2025] Panyut Sriwirote, Attapol T. Rutherford, Jalinee Thapiang and Vasan Timtong. (2025). + PhayaThaiBERT: Enhancing a Pretrained Thai Language Model with Unassimilated Loanwords. + ACM Transactions on Asian and Low-Resource Language Information Processing, Vol. 24, No. 11, pp 1-17. + doi:10.1145/3765962 diff --git a/pyproject.toml b/pyproject.toml index 7c9141a52..24995cd53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,7 +142,14 @@ onnx = ["numpy>=1.22", "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"] oskut = ["lekcut>=1.0.0"] -qwen3 = ["torch>=1.9.0", "transformers>=4.22.1"] +phayathaibert_onnx = [ + "huggingface-hub>=0.16.0", + "numpy>=1.22", + "onnxruntime>=1.10.0", + "tokenizers>=0.13.0", +] + +qwen3 =["torch>=1.9.0", "transformers>=4.22.1"] sefr_cut = ["lekcut>=1.0.0"] @@ -238,6 +245,7 @@ noauto-onnx = [ "huggingface-hub>=0.16.0", "numpy>=1.26.0", "onnxruntime>=1.10.0", + "tokenizers>=0.13.0", ] # Cython-based dependencies - for tests.noauto_cython @@ -628,6 +636,7 @@ module = [ "symspellpy.*", "thai_nner.*", "tltk.*", + "tokenizers.*", "torch.*", "tqdm.*", "transformers.*", diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py new file mode 100644 index 000000000..63229c899 --- /dev/null +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""PhayaThaiBERT part-of-speech tagger with ONNX Runtime backend. + +The model is an ONNX export of +`nlp-chula/phayathaibert-thai-pos-tagger +`_, +fine-tuned on the UD Thai-TUD treebank. It tags words with Universal POS tags. +""" + +from __future__ import annotations + +import json +from collections import Counter +from typing import TYPE_CHECKING, Optional + +from pythainlp.corpus import get_hf_hub + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from onnxruntime import InferenceSession + from tokenizers import Encoding, Tokenizer + +_REPO_ID = "wiriyabot/phayathaibert-thai-pos-tagger-onnx" +# Pinned for reproducible and secure downloads. +_REVISION = "ff55e2c66ef22deee18c423cee272f6e207171ae" + +# Maximum number of subword tokens the model accepts, including and . +MAX_SEQUENCE_LENGTH = 510 + +# Tag for words the model never sees (whitespace-only words), matching +# the perceptron tagger with Universal POS corpora. +_WHITESPACE_TAG = "PUNCT" + +# Universal POS tag for words that produce no subword token. +_UNKNOWN_TAG = "X" + + +def _first_subword_labels( + word_ids: Sequence[Optional[int]], + label_ids: Sequence[int], + n_words: int, + id2label: Mapping[int, str], +) -> list[str]: + """Pick each word's tag from the prediction for its first subword. + + This is the alignment the model was trained with. + + :param word_ids: word index of each subword token; ``None`` for special + tokens + :param label_ids: predicted label id of each subword token + :param n_words: number of input words + :param id2label: mapping from label id to tag name + :return: one tag per word; ``"X"`` for a word that produced no subword + :rtype: list[str] + """ + tags = [_UNKNOWN_TAG] * n_words + seen: set[int] = set() + for word_id, label_id in zip(word_ids, label_ids): + if word_id is None or word_id in seen: + continue + seen.add(word_id) + tags[word_id] = id2label[label_id] + return tags + + +def _chunk_spans( + subword_counts: Sequence[int], max_subwords: int +) -> list[tuple[int, int]]: + """Group consecutive words into spans that fit the model's input length. + + A word longer than ``max_subwords`` gets a span of its own; its subwords + are truncated at encoding time, which keeps its first subword. + + :param subword_counts: number of subword tokens of each word + :param max_subwords: maximum number of subword tokens per span + :return: ``(start, end)`` word index pairs, end exclusive + :rtype: list[tuple[int, int]] + """ + spans: list[tuple[int, int]] = [] + start = 0 + total = 0 + for i, count in enumerate(subword_counts): + if i > start and total + count > max_subwords: + spans.append((start, i)) + start = i + total = 0 + total += count + if subword_counts: + spans.append((start, len(subword_counts))) + return spans + + +class PhayaThaiBERTTagger: + """Universal POS tagger using PhayaThaiBERT with ONNX Runtime. + + Requires ``numpy``, ``onnxruntime``, ``tokenizers`` and + ``huggingface-hub``. The model (about 530 MB) is downloaded from + the Hugging Face Hub on first use. + + :param str repo_id: Hugging Face Hub repository of the ONNX model + :param Optional[str] revision: git revision of the repository + """ + + session: InferenceSession + tokenizer: Tokenizer + id2label: dict[int, str] + + def __init__( + self, repo_id: str = _REPO_ID, revision: Optional[str] = _REVISION + ) -> None: + try: + import numpy # noqa: F401 + from onnxruntime import InferenceSession + from tokenizers import Tokenizer + except ImportError as e: + raise ImportError( + "PhayaThaiBERT POS tagger requires numpy, onnxruntime," + " tokenizers and huggingface-hub." + ' Install them with: pip install "pythainlp[phayathaibert_onnx]"' + ) from e + + model_path = get_hf_hub(repo_id, "model.onnx", revision=revision) + tokenizer_path = get_hf_hub( + repo_id, "tokenizer.json", revision=revision + ) + config_path = get_hf_hub(repo_id, "config.json", revision=revision) + + self.session = InferenceSession( + model_path, providers=["CPUExecutionProvider"] + ) + self.tokenizer = Tokenizer.from_file(tokenizer_path) + with open(config_path, encoding="utf-8") as f: + self.id2label = { + int(k): v for k, v in json.load(f)["id2label"].items() + } + + def tag(self, words: list[str]) -> list[tuple[str, str]]: + """Tag a list of words with Universal POS tags. + + Whitespace-only words are tagged ``PUNCT`` without being passed + to the model. Input longer than the model limit is tagged in + consecutive chunks of whole words. + + :param list[str] words: a list of tokenized words + :return: a list of tuples (word, POS tag) + :rtype: list[tuple[str, str]] + """ + tags = [_WHITESPACE_TAG] * len(words) + positions = [i for i, word in enumerate(words) if word.strip()] + content = [words[i] for i in positions] + if content: + for position, tag in zip(positions, self._tag_content(content)): + tags[position] = tag + return list(zip(words, tags)) + + def _tag_content(self, words: list[str]) -> list[str]: + encoding = self.tokenizer.encode(words, is_pretokenized=True) + if len(encoding.ids) <= MAX_SEQUENCE_LENGTH: + return self._predict(encoding, len(words)) + + counts = Counter(i for i in encoding.word_ids if i is not None) + subword_counts = [counts.get(i, 0) for i in range(len(words))] + labels: list[str] = [] + # Two positions are taken by and . + for start, end in _chunk_spans( + subword_counts, MAX_SEQUENCE_LENGTH - 2 + ): + chunk = words[start:end] + chunk_encoding = self.tokenizer.encode(chunk, is_pretokenized=True) + labels.extend(self._predict(chunk_encoding, len(chunk))) + return labels + + def _predict(self, encoding: Encoding, n_words: int) -> list[str]: + import numpy as np + + ids = encoding.ids + mask = encoding.attention_mask + word_ids = encoding.word_ids + if len(ids) > MAX_SEQUENCE_LENGTH: + # Only a single over-long word gets here; its first subword stays. + ids = ids[: MAX_SEQUENCE_LENGTH - 1] + ids[-1:] + mask = mask[:MAX_SEQUENCE_LENGTH] + word_ids = word_ids[: MAX_SEQUENCE_LENGTH - 1] + [None] + logits = self.session.run( + None, + { + "input_ids": np.array([ids], dtype=np.int64), + "attention_mask": np.array([mask], dtype=np.int64), + }, + )[0][0] + return _first_subword_labels( + word_ids, logits.argmax(axis=-1).tolist(), n_words, self.id2label + ) + + +_TAGGER: Optional[PhayaThaiBERTTagger] = None + + +def tag(words: list[str], corpus: str = "tud") -> list[tuple[str, str]]: + """Tag words with Universal POS tags using PhayaThaiBERT (ONNX). + + :param list[str] words: a list of tokenized words + :param str corpus: ignored; the model is trained on UD Thai-TUD only + :return: a list of tuples (word, POS tag) + :rtype: list[tuple[str, str]] + """ + global _TAGGER + if not words: + return [] + if _TAGGER is None: + _TAGGER = PhayaThaiBERTTagger() + return _TAGGER.tag(words) diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index 6dd69843e..81ba68f3f 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -21,6 +21,11 @@ def pos_tag( * *unigram* - unigram tagger * *tltk* - TLTK: Thai Language Toolkit (supports the TNC corpus only; other corpora are converted to the TNC corpus) + * *phayathaibert* - `PhayaThaiBERT POS tagger + `_ + fine-tuned on the TUD corpus, run with ONNX Runtime (supports + the TUD corpus only; other corpora are ignored). Requires + ``pip install "pythainlp[phayathaibert_onnx]"`` :param str corpus: corpus used to train the tagger model. Options are: * *orchid* - `ORCHID @@ -125,6 +130,10 @@ def pos_tag( corpus = "tnc" elif engine == "unigram" and corpus in _support_corpus: # default from pythainlp.tag.unigram import tag as tag_ + elif engine == "phayathaibert": + from pythainlp.tag.phayathaibert_onnx import tag as tag_ + + corpus = "tud" else: raise ValueError( f"pos_tag not support {engine} engine or {corpus} corpus." @@ -150,6 +159,11 @@ def pos_tag_sents( * *unigram* - unigram tagger * *tltk* - TLTK: Thai Language Toolkit (supports the TNC corpus only; other corpora are converted to the TNC corpus) + * *phayathaibert* - `PhayaThaiBERT POS tagger + `_ + fine-tuned on the TUD corpus, run with ONNX Runtime (supports + the TUD corpus only; other corpora are ignored). Requires + ``pip install "pythainlp[phayathaibert_onnx]"`` :param str corpus: corpus used to train the tagger model. Options are: * *orchid* - `ORCHID @@ -206,6 +220,10 @@ def pos_tag_transformers( (supports the PUD corpus only) * *phayathai* - fine-tuned version of clicknext/phayathaibert on the blackboard corpus (supports the blackboard corpus only) + * *phayathaibert* - `nlp-chula/phayathaibert-thai-pos-tagger + `_, + fine-tuned version of clicknext/phayathaibert on the TUD corpus + (supports the TUD corpus only) * *mdeberta* - mDeBERTa: Multilingual Decoding-enhanced BERT with disentangled attention (supports the PUD corpus only) :param str corpus: corpus used to train the tagger model. Options are: @@ -217,6 +235,9 @@ def pos_tag_transformers( `_ treebank, natively uses Universal POS tags (supports the wangchanberta and mdeberta engines) + * *tud* - `Thai Universal Dependency Treebank (TUD) + `_, natively uses Universal + POS tags (supports the phayathaibert engine) :param Optional[str] revision: git revision (branch, tag, or commit hash) of the model. Pin to a full commit hash for secure downloads :return: list of lists of tuples (word, POS tag) @@ -248,36 +269,29 @@ def pos_tag_transformers( if not sentence: return [] - _blackboard_support_engine = { - "bert": "lunarlist/pos_thai", - "phayathai": "lunarlist/pos_thai_phayathai", + _support_engine: dict[str, dict[str, str]] = { + "blackboard": { + "bert": "lunarlist/pos_thai", + "phayathai": "lunarlist/pos_thai_phayathai", + }, + "pud": { + "wangchanberta": "Pavarissy/wangchanberta-ud-thai-pud-upos", + "mdeberta": "Pavarissy/mdeberta-v3-ud-thai-pud-upos", + }, + "tud": { + "phayathaibert": "nlp-chula/phayathaibert-thai-pos-tagger", + }, } - _pud_support_engine = { - "wangchanberta": "Pavarissy/wangchanberta-ud-thai-pud-upos", - "mdeberta": "Pavarissy/mdeberta-v3-ud-thai-pud-upos", - } - - if corpus == "blackboard" and engine in _blackboard_support_engine: - base_model = _blackboard_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained( - base_model, revision=revision - ) - tokenizer = AutoTokenizer.from_pretrained( - base_model, revision=revision - ) - elif corpus == "pud" and engine in _pud_support_engine: - base_model = _pud_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained( - base_model, revision=revision - ) - tokenizer = AutoTokenizer.from_pretrained( - base_model, revision=revision - ) - else: + base_model = _support_engine.get(corpus, {}).get(engine) + if base_model is None: raise ValueError( f"pos_tag_transformers not support {engine} engine or {corpus} corpus." ) + model = AutoModelForTokenClassification.from_pretrained( + base_model, revision=revision + ) + tokenizer = AutoTokenizer.from_pretrained(base_model, revision=revision) pipeline = TokenClassificationPipeline( model=model, diff --git a/tests/README.md b/tests/README.md index 557134ec5..bef26aab1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -142,6 +142,7 @@ By separating tests by dependency group, we can: - Need dependencies from `pip install "pythainlp[noauto-onnx]"` - Tests requiring ONNX Runtime: - attacut, deepcut, oskut, sefr_cut tokenizers (via LEKCut / ONNX) + - phayathaibert POS tagger (downloads a ~530 MB model on first run) - Dependencies: ~200-500 MB - Test case class suffix: `TestCaseN` diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index f5ba10801..d0f0feeb1 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -459,3 +459,69 @@ def test_iob_to_markup(self): def test_iob_to_markup_long_input(self): tagged = [("ก", "B-A")] + [("ก", "I-A")] * 10000 self.assertEqual(_iob_to_markup(tagged), "" + "ก" * 10001 + "") + + +class PhayaThaiBERTHelperTestCase(unittest.TestCase): + """Test the pure-Python helpers of pythainlp.tag.phayathaibert_onnx""" + + def test_first_subword_labels(self): + from pythainlp.tag.phayathaibert_onnx import _first_subword_labels + + id2label = {0: "NOUN", 1: "VERB", 2: "PRON"} + # word0-a word0-b word1 + word_ids = [None, 0, 0, 1, None] + label_ids = [1, 2, 1, 0, 1] + self.assertEqual( + _first_subword_labels(word_ids, label_ids, 2, id2label), + ["PRON", "NOUN"], + ) + + def test_first_subword_labels_word_without_subword(self): + from pythainlp.tag.phayathaibert_onnx import _first_subword_labels + + id2label = {0: "NOUN", 1: "VERB"} + # word 1 produced no subword at all + word_ids = [None, 0, 2, None] + label_ids = [0, 0, 1, 0] + self.assertEqual( + _first_subword_labels(word_ids, label_ids, 3, id2label), + ["NOUN", "X", "VERB"], + ) + + def test_chunk_spans_fits_in_one(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([3, 4, 5], 20), [(0, 3)]) + + def test_chunk_spans_splits_at_word_boundaries(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([3, 4, 5], 7), [(0, 2), (2, 3)]) + + def test_chunk_spans_oversized_word_gets_own_span(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([3, 10, 2], 5), [(0, 1), (1, 2), (2, 3)]) + + def test_chunk_spans_empty(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([], 5), []) + + def test_pos_tag_phayathaibert_empty_list(self): + self.assertEqual(pos_tag([], engine="phayathaibert"), []) + + def test_missing_dependency_error_before_download(self): + import sys + from unittest import mock + + from pythainlp.tag import phayathaibert_onnx + + with mock.patch.dict(sys.modules, {"onnxruntime": None}): + with mock.patch( + "pythainlp.tag.phayathaibert_onnx.get_hf_hub" + ) as get_hf_hub: + with self.assertRaises(ImportError) as ctx: + phayathaibert_onnx.PhayaThaiBERTTagger() + self.assertIn("pip install", str(ctx.exception)) + get_hf_hub.assert_not_called() diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index 6e6fced1c..3ddf005a7 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -51,3 +51,61 @@ def test_pos_tag_wangchanberta_onnx_empty_list(self): result = pos_tag([], engine="wangchanberta_onnx") self.assertIsInstance(result, list) self.assertEqual(len(result), 0) + + +class TagPhayaThaiBERTONNXTestCaseN(unittest.TestCase): + """Tests for the PhayaThaiBERT ONNX POS tagger + (requires onnxruntime, tokenizers, huggingface-hub)""" + + WORDS = ["ฉัน", "กิน", "ข้าว", "ที่", "ร้านอาหาร"] + + def test_pos_tag_phayathaibert(self): + from pythainlp.tag import pos_tag + + self.assertEqual( + pos_tag(self.WORDS, engine="phayathaibert"), + [ + ("ฉัน", "PRON"), + ("กิน", "VERB"), + ("ข้าว", "NOUN"), + ("ที่", "ADP"), + ("ร้านอาหาร", "NOUN"), + ], + ) + + def test_pos_tag_phayathaibert_ignores_corpus(self): + from pythainlp.tag import pos_tag + + self.assertEqual( + pos_tag(self.WORDS, engine="phayathaibert", corpus="orchid"), + pos_tag(self.WORDS, engine="phayathaibert", corpus="tud"), + ) + + def test_pos_tag_phayathaibert_whitespace_words(self): + from pythainlp.tag import pos_tag + + result = pos_tag(["แมว", " ", "กิน", ""], engine="phayathaibert") + self.assertEqual(len(result), 4) + self.assertEqual(result[1], (" ", "PUNCT")) + self.assertEqual(result[3], ("", "PUNCT")) + self.assertEqual( + [result[0], result[2]], + pos_tag(["แมว", "กิน"], engine="phayathaibert"), + ) + + def test_pos_tag_phayathaibert_long_input(self): + from pythainlp.tag import pos_tag + + words = self.WORDS * 300 # well over the 510-token model limit + result = pos_tag(words, engine="phayathaibert") + self.assertEqual([w for w, _ in result], words) + self.assertEqual([t for _, t in result[:3]], ["PRON", "VERB", "NOUN"]) + + def test_pos_tag_sents_phayathaibert(self): + from pythainlp.tag import pos_tag_sents + + result = pos_tag_sents( + [self.WORDS, ["แมว", "กิน", "ปลา"]], engine="phayathaibert" + ) + self.assertEqual(len(result), 2) + self.assertEqual(len(result[1]), 3) diff --git a/tests/noauto_torch/testn_tag_torch.py b/tests/noauto_torch/testn_tag_torch.py index 8e2c05251..651139e87 100644 --- a/tests/noauto_torch/testn_tag_torch.py +++ b/tests/noauto_torch/testn_tag_torch.py @@ -99,3 +99,37 @@ def test_pos_tag_transformers(self): engine="bert", corpus="non-existing corpus", ) + + def test_pos_tag_transformers_phayathaibert_tud(self): + upos = { + "ADJ", + "ADP", + "ADV", + "AUX", + "CCONJ", + "DET", + "NOUN", + "NUM", + "PART", + "PRON", + "PROPN", + "PUNCT", + "SCONJ", + "SYM", + "VERB", + } + result = pos_tag_transformers( + sentence="ฉันกินข้าวที่ร้านอาหาร", + engine="phayathaibert", + corpus="tud", + ) + self.assertEqual(len(result), 1) + self.assertGreater(len(result[0]), 0) + tags = {tag for _, tag in result[0]} + self.assertTrue(tags <= upos, tags - upos) + self.assertIn("VERB", tags) + + with self.assertRaises(ValueError): + pos_tag_transformers( + sentence="แมวทำอะไร", engine="phayathaibert", corpus="pud" + ) From 5fd17ab2dc3e2e6215d00b0bdb420192e0bfceaa Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 00:46:51 +0700 Subject: [PATCH 02/12] Fix PhayaThaiBERT tagger on long input and tighten requirements The published tokenizer.json truncates at 510 tokens, so long input was cut before chunking and every word past the cut was tagged X. Turn off the tokenizer's built-in truncation and padding so chunking by whole words takes effect. - Tag words made only of whitespace or format characters (such as the zero-width space) as PUNCT without passing them to the model. - Require onnxruntime>=1.12.0 (opset 17) and tokenizers>=0.19.0 (the Metaspace format in tokenizer.json). - Pin the default revision only for the default repository. - Guard the lazily created tagger with a lock. - Name huggingface-hub in the import check. - List the phayathaibert_onnx extra in the installation notes. - Test tags past the first 510 tokens, an over-long word, and format characters. --- CHANGELOG.md | 4 +- docs/notes/installation.rst | 1 + pyproject.toml | 10 ++-- pythainlp/tag/phayathaibert_onnx.py | 55 ++++++++++++++----- tests/core/test_tag.py | 82 ++++++++++++++++++++++++++--- tests/noauto_onnx/__init__.py | 1 + tests/noauto_onnx/testn_tag_onnx.py | 27 +++++++++- 7 files changed, 149 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c837caff..9c1a2a077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,8 @@ and this project adheres to ([#1509]) - `evaluate_word_tokenization`, `word_eval_function`, `char_eval_function`, and `TokenizationScore` in `pythainlp.benchmarks` ([#1512]) -- `pos_tag()` engine `phayathaibert` (ONNX) and `pos_tag_transformers()` - corpus `tud`: PhayaThaiBERT Universal POS tagger trained on TUD ([#1536]) +- PhayaThaiBERT POS tagger trained on TUD: `pos_tag()` engine `phayathaibert` + (ONNX, extra `phayathaibert_onnx`); `pos_tag_transformers()` with corpus `tud` ([#1536]) ### Changed diff --git a/docs/notes/installation.rst b/docs/notes/installation.rst index f7700e3a6..e186a67e5 100644 --- a/docs/notes/installation.rst +++ b/docs/notes/installation.rst @@ -32,6 +32,7 @@ The extras can include: - ``nlpo3`` — nlpo3 Thai word tokenization support - ``onnx`` - ONNX model support - ``oskut`` — OSKUT support (via LEKCut ONNX) + - ``phayathaibert_onnx`` — PhayaThaiBERT part-of-speech tagger with ONNX Runtime - ``sefr_cut`` — SEFR CUT Thai word tokenization support (via LEKCut ONNX) - ``spacy_thai`` — spaCy Thai language support - ``spell`` — support for more spell-checkers (phunspell & symspellpy) diff --git a/pyproject.toml b/pyproject.toml index 24995cd53..118151a65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,11 +145,11 @@ oskut = ["lekcut>=1.0.0"] phayathaibert_onnx = [ "huggingface-hub>=0.16.0", "numpy>=1.22", - "onnxruntime>=1.10.0", - "tokenizers>=0.13.0", + "onnxruntime>=1.12.0", + "tokenizers>=0.19.0", ] -qwen3 =["torch>=1.9.0", "transformers>=4.22.1"] +qwen3 = ["torch>=1.9.0", "transformers>=4.22.1"] sefr_cut = ["lekcut>=1.0.0"] @@ -244,8 +244,8 @@ noauto-onnx = [ "lekcut>=1.0.0", "huggingface-hub>=0.16.0", "numpy>=1.26.0", - "onnxruntime>=1.10.0", - "tokenizers>=0.13.0", + "onnxruntime>=1.12.0", + "tokenizers>=0.19.0", ] # Cython-based dependencies - for tests.noauto_cython diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 63229c899..49d7e0cc5 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -12,6 +12,8 @@ from __future__ import annotations import json +import threading +import unicodedata from collections import Counter from typing import TYPE_CHECKING, Optional @@ -28,7 +30,7 @@ _REVISION = "ff55e2c66ef22deee18c423cee272f6e207171ae" # Maximum number of subword tokens the model accepts, including and . -MAX_SEQUENCE_LENGTH = 510 +_MAX_SEQUENCE_LENGTH = 510 # Tag for words the model never sees (whitespace-only words), matching # the perceptron tagger with Universal POS corpora. @@ -38,6 +40,19 @@ _UNKNOWN_TAG = "X" +def _is_blank(word: str) -> bool: + """Return True if a word has only whitespace or format characters. + + Format characters (Unicode category Cf) include the zero-width space, + zero-width non-joiner and byte order mark, which carry no text. + + :param str word: a word + :return: whether the word is empty, whitespace or format characters + :rtype: bool + """ + return all(ch.isspace() or unicodedata.category(ch) == "Cf" for ch in word) + + def _first_subword_labels( word_ids: Sequence[Optional[int]], label_ids: Sequence[int], @@ -101,7 +116,8 @@ class PhayaThaiBERTTagger: the Hugging Face Hub on first use. :param str repo_id: Hugging Face Hub repository of the ONNX model - :param Optional[str] revision: git revision of the repository + :param Optional[str] revision: git revision of the repository. + The default repository is pinned to a commit when this is ``None``. """ session: InferenceSession @@ -109,9 +125,10 @@ class PhayaThaiBERTTagger: id2label: dict[int, str] def __init__( - self, repo_id: str = _REPO_ID, revision: Optional[str] = _REVISION + self, repo_id: str = _REPO_ID, revision: Optional[str] = None ) -> None: try: + import huggingface_hub # noqa: F401 import numpy # noqa: F401 from onnxruntime import InferenceSession from tokenizers import Tokenizer @@ -122,6 +139,8 @@ def __init__( ' Install them with: pip install "pythainlp[phayathaibert_onnx]"' ) from e + if revision is None and repo_id == _REPO_ID: + revision = _REVISION model_path = get_hf_hub(repo_id, "model.onnx", revision=revision) tokenizer_path = get_hf_hub( repo_id, "tokenizer.json", revision=revision @@ -132,6 +151,10 @@ def __init__( model_path, providers=["CPUExecutionProvider"] ) self.tokenizer = Tokenizer.from_file(tokenizer_path) + # tokenizer.json ships with truncation at 510 tokens, which would + # silently drop words; chunking is done here instead. + self.tokenizer.no_truncation() + self.tokenizer.no_padding() with open(config_path, encoding="utf-8") as f: self.id2label = { int(k): v for k, v in json.load(f)["id2label"].items() @@ -140,16 +163,17 @@ def __init__( def tag(self, words: list[str]) -> list[tuple[str, str]]: """Tag a list of words with Universal POS tags. - Whitespace-only words are tagged ``PUNCT`` without being passed - to the model. Input longer than the model limit is tagged in - consecutive chunks of whole words. + Words with only whitespace or format characters (such as the + zero-width space) are tagged ``PUNCT`` without being passed to the + model. Input longer than the model limit is tagged in consecutive + chunks of whole words. :param list[str] words: a list of tokenized words :return: a list of tuples (word, POS tag) :rtype: list[tuple[str, str]] """ tags = [_WHITESPACE_TAG] * len(words) - positions = [i for i, word in enumerate(words) if word.strip()] + positions = [i for i, word in enumerate(words) if not _is_blank(word)] content = [words[i] for i in positions] if content: for position, tag in zip(positions, self._tag_content(content)): @@ -158,7 +182,7 @@ def tag(self, words: list[str]) -> list[tuple[str, str]]: def _tag_content(self, words: list[str]) -> list[str]: encoding = self.tokenizer.encode(words, is_pretokenized=True) - if len(encoding.ids) <= MAX_SEQUENCE_LENGTH: + if len(encoding.ids) <= _MAX_SEQUENCE_LENGTH: return self._predict(encoding, len(words)) counts = Counter(i for i in encoding.word_ids if i is not None) @@ -166,7 +190,7 @@ def _tag_content(self, words: list[str]) -> list[str]: labels: list[str] = [] # Two positions are taken by and . for start, end in _chunk_spans( - subword_counts, MAX_SEQUENCE_LENGTH - 2 + subword_counts, _MAX_SEQUENCE_LENGTH - 2 ): chunk = words[start:end] chunk_encoding = self.tokenizer.encode(chunk, is_pretokenized=True) @@ -179,11 +203,11 @@ def _predict(self, encoding: Encoding, n_words: int) -> list[str]: ids = encoding.ids mask = encoding.attention_mask word_ids = encoding.word_ids - if len(ids) > MAX_SEQUENCE_LENGTH: + if len(ids) > _MAX_SEQUENCE_LENGTH: # Only a single over-long word gets here; its first subword stays. - ids = ids[: MAX_SEQUENCE_LENGTH - 1] + ids[-1:] - mask = mask[:MAX_SEQUENCE_LENGTH] - word_ids = word_ids[: MAX_SEQUENCE_LENGTH - 1] + [None] + ids = ids[: _MAX_SEQUENCE_LENGTH - 1] + ids[-1:] + mask = mask[:_MAX_SEQUENCE_LENGTH] + word_ids = word_ids[: _MAX_SEQUENCE_LENGTH - 1] + [None] logits = self.session.run( None, { @@ -197,6 +221,7 @@ def _predict(self, encoding: Encoding, n_words: int) -> list[str]: _TAGGER: Optional[PhayaThaiBERTTagger] = None +_TAGGER_LOCK = threading.Lock() def tag(words: list[str], corpus: str = "tud") -> list[tuple[str, str]]: @@ -211,5 +236,7 @@ def tag(words: list[str], corpus: str = "tud") -> list[tuple[str, str]]: if not words: return [] if _TAGGER is None: - _TAGGER = PhayaThaiBERTTagger() + with _TAGGER_LOCK: + if _TAGGER is None: + _TAGGER = PhayaThaiBERTTagger() return _TAGGER.tag(words) diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index d0f0feeb1..b89c77551 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -508,8 +508,28 @@ def test_chunk_spans_empty(self): self.assertEqual(_chunk_spans([], 5), []) - def test_pos_tag_phayathaibert_empty_list(self): - self.assertEqual(pos_tag([], engine="phayathaibert"), []) + def test_chunk_spans_exact_fit(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([3, 4], 7), [(0, 2)]) + + def test_chunk_spans_zero_count_words(self): + from pythainlp.tag.phayathaibert_onnx import _chunk_spans + + self.assertEqual(_chunk_spans([0, 3, 0, 4], 7), [(0, 4)]) + + def test_is_blank(self): + from pythainlp.tag.phayathaibert_onnx import _is_blank + + for word in ("", " ", "\n", " ", "　", "​", ""): + self.assertTrue(_is_blank(word), repr(word)) + for word in ("แมว", " แมว ", "a", "1", "​แมว"): + self.assertFalse(_is_blank(word), repr(word)) + + def test_tag_empty_list(self): + from pythainlp.tag.phayathaibert_onnx import tag + + self.assertEqual(tag([]), []) def test_missing_dependency_error_before_download(self): import sys @@ -517,11 +537,57 @@ def test_missing_dependency_error_before_download(self): from pythainlp.tag import phayathaibert_onnx - with mock.patch.dict(sys.modules, {"onnxruntime": None}): + for module in ("onnxruntime", "tokenizers", "huggingface_hub"): + with mock.patch.dict(sys.modules, {module: None}): + with mock.patch( + "pythainlp.tag.phayathaibert_onnx.get_hf_hub" + ) as get_hf_hub: + with self.assertRaises(ImportError) as ctx: + phayathaibert_onnx.PhayaThaiBERTTagger() + self.assertIn( + "pythainlp[phayathaibert_onnx]", str(ctx.exception) + ) + get_hf_hub.assert_not_called() + + def _first_download_revision(self, **kwargs: str) -> object: + """Return the revision PhayaThaiBERTTagger passes to get_hf_hub.""" + import sys + from unittest import mock + + from pythainlp.tag import phayathaibert_onnx + + fake_modules = { + name: mock.MagicMock() + for name in ( + "numpy", + "onnxruntime", + "tokenizers", + "huggingface_hub", + ) + } + with mock.patch.dict(sys.modules, fake_modules): with mock.patch( - "pythainlp.tag.phayathaibert_onnx.get_hf_hub" + "pythainlp.tag.phayathaibert_onnx.get_hf_hub", + side_effect=RuntimeError("stop"), ) as get_hf_hub: - with self.assertRaises(ImportError) as ctx: - phayathaibert_onnx.PhayaThaiBERTTagger() - self.assertIn("pip install", str(ctx.exception)) - get_hf_hub.assert_not_called() + with self.assertRaises(RuntimeError): + phayathaibert_onnx.PhayaThaiBERTTagger(**kwargs) + return get_hf_hub.call_args.kwargs["revision"] + + def test_default_repo_is_pinned(self): + from pythainlp.tag import phayathaibert_onnx + + self.assertEqual( + self._first_download_revision(), phayathaibert_onnx._REVISION + ) + + def test_custom_repo_is_not_pinned_to_default_revision(self): + self.assertIsNone( + self._first_download_revision(repo_id="someone/other-model") + ) + self.assertEqual( + self._first_download_revision( + repo_id="someone/other-model", revision="abc123" + ), + "abc123", + ) diff --git a/tests/noauto_onnx/__init__.py b/tests/noauto_onnx/__init__.py index 798f02261..b092491e0 100644 --- a/tests/noauto_onnx/__init__.py +++ b/tests/noauto_onnx/__init__.py @@ -11,6 +11,7 @@ - onnxruntime - oskut - sefr_cut +- tokenizers and huggingface-hub (PhayaThaiBERT POS tagger) These tests are NOT run in automated CI workflows due to: - Large dependencies diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index 3ddf005a7..47dce547c 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -4,7 +4,7 @@ # Tests for POS tagging functions that require ONNX Runtime # These tests are NOT run in automated CI workflows due to: -# - Large dependencies (onnxruntime) +# - Large dependencies (onnxruntime, tokenizers, huggingface-hub) # - Platform-specific compatibility issues # - Version constraints @@ -98,8 +98,31 @@ def test_pos_tag_phayathaibert_long_input(self): words = self.WORDS * 300 # well over the 510-token model limit result = pos_tag(words, engine="phayathaibert") + tags = [t for _, t in result] self.assertEqual([w for w, _ in result], words) - self.assertEqual([t for _, t in result[:3]], ["PRON", "VERB", "NOUN"]) + # Every word, including those past the first 510 tokens, is tagged. + self.assertNotIn("X", tags) + self.assertEqual(tags[-5:], ["PRON", "VERB", "NOUN", "ADP", "NOUN"]) + + def test_pos_tag_phayathaibert_overlong_word(self): + from pythainlp.tag import pos_tag + + # A single word of far more than 510 subwords, between normal words. + words = ["ฉัน", "กิน", "ก" * 5000, "ที่", "ร้านอาหาร"] + result = pos_tag(words, engine="phayathaibert") + tags = [t for _, t in result] + self.assertEqual(len(result), 5) + self.assertNotIn("X", tags) + self.assertEqual(tags[:2], ["PRON", "VERB"]) + self.assertEqual(tags[3:], ["ADP", "NOUN"]) + + def test_pos_tag_phayathaibert_format_characters(self): + from pythainlp.tag import pos_tag + + # Zero-width space, zero-width non-joiner and BOM carry no text. + for char in ("​", "‌", ""): + result = pos_tag(["แมว", char, "กิน"], engine="phayathaibert") + self.assertEqual(result[1], (char, "PUNCT")) def test_pos_tag_sents_phayathaibert(self): from pythainlp.tag import pos_tag_sents From bebfe74f01fd957996574ed905b97f0ad1c8e4c0 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 10:01:52 +0700 Subject: [PATCH 03/12] Follow new docstring rules and cover the tagger in core tests - Use D213 docstrings and typed fields in phayathaibert_onnx, and add an __init__ docstring (D107). - Test PhayaThaiBERTTagger, tag() and the new pos_tag and pos_tag_transformers branches with a fake model, so CI covers the new code without onnxruntime, tokenizers or transformers. - Fake every dependency inside patch.dict(sys.modules), so no real module (such as numpy) is imported and then unloaded by the test. - Merge the _chunk_spans tests into one table-driven test. --- pythainlp/tag/phayathaibert_onnx.py | 55 ++++-- tests/core/test_tag.py | 283 ++++++++++++++++++++++++---- tests/noauto_onnx/testn_tag_onnx.py | 3 +- 3 files changed, 285 insertions(+), 56 deletions(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 49d7e0cc5..3d6f296c4 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""PhayaThaiBERT part-of-speech tagger with ONNX Runtime backend. +""" +PhayaThaiBERT part-of-speech tagger with ONNX Runtime backend. The model is an ONNX export of `nlp-chula/phayathaibert-thai-pos-tagger @@ -41,12 +42,13 @@ def _is_blank(word: str) -> bool: - """Return True if a word has only whitespace or format characters. + """ + Return True if a word has only whitespace or format characters. Format characters (Unicode category Cf) include the zero-width space, zero-width non-joiner and byte order mark, which carry no text. - :param str word: a word + :param str word: word to check :return: whether the word is empty, whitespace or format characters :rtype: bool """ @@ -59,15 +61,16 @@ def _first_subword_labels( n_words: int, id2label: Mapping[int, str], ) -> list[str]: - """Pick each word's tag from the prediction for its first subword. + """ + Pick each word's tag from the prediction for its first subword. This is the alignment the model was trained with. - :param word_ids: word index of each subword token; ``None`` for special - tokens - :param label_ids: predicted label id of each subword token - :param n_words: number of input words - :param id2label: mapping from label id to tag name + :param Sequence[Optional[int]] word_ids: word index of each subword + token; ``None`` for special tokens + :param Sequence[int] label_ids: predicted label id of each subword token + :param int n_words: number of input words + :param Mapping[int, str] id2label: mapping from label id to tag name :return: one tag per word; ``"X"`` for a word that produced no subword :rtype: list[str] """ @@ -84,13 +87,15 @@ def _first_subword_labels( def _chunk_spans( subword_counts: Sequence[int], max_subwords: int ) -> list[tuple[int, int]]: - """Group consecutive words into spans that fit the model's input length. + """ + Group consecutive words into spans that fit the model's input length. A word longer than ``max_subwords`` gets a span of its own; its subwords are truncated at encoding time, which keeps its first subword. - :param subword_counts: number of subword tokens of each word - :param max_subwords: maximum number of subword tokens per span + :param Sequence[int] subword_counts: number of subword tokens of each + word + :param int max_subwords: maximum number of subword tokens per span :return: ``(start, end)`` word index pairs, end exclusive :rtype: list[tuple[int, int]] """ @@ -109,7 +114,8 @@ def _chunk_spans( class PhayaThaiBERTTagger: - """Universal POS tagger using PhayaThaiBERT with ONNX Runtime. + """ + Universal POS tagger using PhayaThaiBERT with ONNX Runtime. Requires ``numpy``, ``onnxruntime``, ``tokenizers`` and ``huggingface-hub``. The model (about 530 MB) is downloaded from @@ -127,6 +133,15 @@ class PhayaThaiBERTTagger: def __init__( self, repo_id: str = _REPO_ID, revision: Optional[str] = None ) -> None: + """ + Download the model and load it with ONNX Runtime. + + :param str repo_id: Hugging Face Hub repository of the ONNX model + :param Optional[str] revision: git revision of the repository. + The default repository is pinned to a commit when this is + ``None``. + :raises ImportError: if a required dependency is not installed + """ try: import huggingface_hub # noqa: F401 import numpy # noqa: F401 @@ -161,15 +176,16 @@ def __init__( } def tag(self, words: list[str]) -> list[tuple[str, str]]: - """Tag a list of words with Universal POS tags. + """ + Tag a list of words with Universal POS tags. Words with only whitespace or format characters (such as the zero-width space) are tagged ``PUNCT`` without being passed to the model. Input longer than the model limit is tagged in consecutive chunks of whole words. - :param list[str] words: a list of tokenized words - :return: a list of tuples (word, POS tag) + :param list[str] words: list of words to be tagged + :return: list of tuples (word, POS tag) :rtype: list[tuple[str, str]] """ tags = [_WHITESPACE_TAG] * len(words) @@ -225,11 +241,12 @@ def _predict(self, encoding: Encoding, n_words: int) -> list[str]: def tag(words: list[str], corpus: str = "tud") -> list[tuple[str, str]]: - """Tag words with Universal POS tags using PhayaThaiBERT (ONNX). + """ + Tag words with Universal POS tags using PhayaThaiBERT (ONNX). - :param list[str] words: a list of tokenized words + :param list[str] words: list of words to be tagged :param str corpus: ignored; the model is trained on UD Thai-TUD only - :return: a list of tuples (word, POS tag) + :return: list of tuples (word, POS tag) :rtype: list[tuple[str, str]] """ global _TAGGER diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index b89c77551..7331e25e9 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import unittest +from importlib.util import find_spec from os import path from pythainlp.corpus import download @@ -20,6 +21,13 @@ TEST_TOKENS = ["ผม", "รัก", "คุณ"] +_PHAYATHAIBERT_DEPENDENCIES = ( + "huggingface_hub", + "numpy", + "onnxruntime", + "tokenizers", +) + class TagTestCase(unittest.TestCase): """Test pythainlp.tag.pos_tag.""" @@ -488,25 +496,26 @@ def test_first_subword_labels_word_without_subword(self): ["NOUN", "X", "VERB"], ) - def test_chunk_spans_fits_in_one(self): - from pythainlp.tag.phayathaibert_onnx import _chunk_spans - - self.assertEqual(_chunk_spans([3, 4, 5], 20), [(0, 3)]) - - def test_chunk_spans_splits_at_word_boundaries(self): - from pythainlp.tag.phayathaibert_onnx import _chunk_spans - - self.assertEqual(_chunk_spans([3, 4, 5], 7), [(0, 2), (2, 3)]) - - def test_chunk_spans_oversized_word_gets_own_span(self): - from pythainlp.tag.phayathaibert_onnx import _chunk_spans - - self.assertEqual(_chunk_spans([3, 10, 2], 5), [(0, 1), (1, 2), (2, 3)]) - - def test_chunk_spans_empty(self): + def test_chunk_spans(self): from pythainlp.tag.phayathaibert_onnx import _chunk_spans - self.assertEqual(_chunk_spans([], 5), []) + cases = [ + ("empty", [], 5, []), + ("fits in one", [3, 4, 5], 20, [(0, 3)]), + ("exact fit", [3, 4], 7, [(0, 2)]), + ("split at word boundaries", [3, 4, 5], 7, [(0, 2), (2, 3)]), + ( + "oversized word gets own span", + [3, 10, 2], + 5, + [(0, 1), (1, 2), (2, 3)], + ), + ("oversized first word", [10, 1], 5, [(0, 1), (1, 2)]), + ("word with no subword", [3, 0, 4], 7, [(0, 3)]), + ] + for name, counts, limit, expected in cases: + with self.subTest(name): + self.assertEqual(_chunk_spans(counts, limit), expected) def test_chunk_spans_exact_fit(self): from pythainlp.tag.phayathaibert_onnx import _chunk_spans @@ -537,17 +546,24 @@ def test_missing_dependency_error_before_download(self): from pythainlp.tag import phayathaibert_onnx - for module in ("onnxruntime", "tokenizers", "huggingface_hub"): - with mock.patch.dict(sys.modules, {module: None}): - with mock.patch( - "pythainlp.tag.phayathaibert_onnx.get_hf_hub" - ) as get_hf_hub: - with self.assertRaises(ImportError) as ctx: - phayathaibert_onnx.PhayaThaiBERTTagger() - self.assertIn( - "pythainlp[phayathaibert_onnx]", str(ctx.exception) - ) - get_hf_hub.assert_not_called() + for module in _PHAYATHAIBERT_DEPENDENCIES: + with self.subTest(module): + # Fake every dependency, so none is really imported (and + # then unloaded) inside patch.dict. + modules = { + m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES + } + modules[module] = None + with mock.patch.dict(sys.modules, modules): + with mock.patch( + "pythainlp.tag.phayathaibert_onnx.get_hf_hub" + ) as get_hf_hub: + with self.assertRaises(ImportError) as ctx: + phayathaibert_onnx.PhayaThaiBERTTagger() + self.assertIn( + "pythainlp[phayathaibert_onnx]", str(ctx.exception) + ) + get_hf_hub.assert_not_called() def _first_download_revision(self, **kwargs: str) -> object: """Return the revision PhayaThaiBERTTagger passes to get_hf_hub.""" @@ -557,13 +573,7 @@ def _first_download_revision(self, **kwargs: str) -> object: from pythainlp.tag import phayathaibert_onnx fake_modules = { - name: mock.MagicMock() - for name in ( - "numpy", - "onnxruntime", - "tokenizers", - "huggingface_hub", - ) + m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES } with mock.patch.dict(sys.modules, fake_modules): with mock.patch( @@ -591,3 +601,206 @@ def test_custom_repo_is_not_pinned_to_default_revision(self): ), "abc123", ) + + +class _FakeEncoding: + """ + Encode each word as one subword per character. + + Only the first subword carries the word's label id (1 for words that + start with "ก", 2 otherwise); the others carry 0, so tagging with any + subword but the first gives the wrong tag. + """ + + def __init__(self, words): + self.ids = [0] + self.word_ids = [None] + for i, word in enumerate(words): + label_id = 1 if word.startswith("ก") else 2 + self.ids += [label_id] + [0] * (len(word) - 1) + self.word_ids += [i] * len(word) + self.ids.append(0) + self.word_ids.append(None) + self.attention_mask = [1] * len(self.ids) + + +class _FakeTokenizer: + def encode(self, words, is_pretokenized=False): + return _FakeEncoding(words) + + +class _FakeSession: + """Predict the label id equal to each input id (one-hot logits).""" + + def __init__(self): + self.lengths = [] + + def run(self, output_names, feeds): + import numpy as np + + ids = feeds["input_ids"] + self.lengths.append(ids.shape[1]) + return [np.eye(3)[ids]] + + +class PhayaThaiBERTTaggerTestCase(unittest.TestCase): + """Test pythainlp.tag.phayathaibert_onnx with a fake model.""" + + def _tagger(self): + from pythainlp.tag.phayathaibert_onnx import PhayaThaiBERTTagger + + tagger = PhayaThaiBERTTagger.__new__(PhayaThaiBERTTagger) + tagger.session = _FakeSession() + tagger.tokenizer = _FakeTokenizer() + tagger.id2label = {0: "SCONJ", 1: "NOUN", 2: "VERB"} + return tagger + + @unittest.skipUnless(find_spec("numpy"), "numpy is not installed") + def test_tag(self): + from unittest import mock + + from pythainlp.tag import phayathaibert_onnx + + cases = [ + ("short", ["กา", "ขาว"], 20, ["NOUN", "VERB"]), + ( + "blank words skipped", + ["กา", " ", "ขา", "", "​"], + 20, + ["NOUN", "PUNCT", "VERB", "PUNCT", "PUNCT"], + ), + ("all blank", [" ", ""], 20, ["PUNCT", "PUNCT"]), + ( + "chunked", + ["กา", "ขา", "ก", "ขาว", "กก", "ขข"], + 6, + ["NOUN", "VERB", "NOUN", "VERB", "NOUN", "VERB"], + ), + ("overlong word", ["ก" * 20, "ขา"], 6, ["NOUN", "VERB"]), + ("overlong last word", ["ขา", "ก" * 20], 6, ["VERB", "NOUN"]), + ] + for name, words, limit, expected in cases: + with self.subTest(name): + tagger = self._tagger() + with mock.patch.object( + phayathaibert_onnx, "_MAX_SEQUENCE_LENGTH", limit + ): + result = tagger.tag(words) + self.assertEqual(result, list(zip(words, expected))) + self.assertTrue( + all(n <= limit for n in tagger.session.lengths) + ) + + def test_init(self): + import json + import sys + import tempfile + from unittest import mock + + from pythainlp.tag import phayathaibert_onnx + + fake_modules = { + m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES + } + with tempfile.TemporaryDirectory() as tmp: + config_path = path.join(tmp, "config.json") + with open(config_path, "w", encoding="utf-8") as f: + json.dump({"id2label": {"0": "NOUN", "1": "VERB"}}, f) + + def fake_get_hf_hub(repo_id, filename, revision=None): + return config_path if filename == "config.json" else filename + + with mock.patch.dict(sys.modules, fake_modules): + with mock.patch.object( + phayathaibert_onnx, + "get_hf_hub", + side_effect=fake_get_hf_hub, + ) as get_hf_hub: + tagger = phayathaibert_onnx.PhayaThaiBERTTagger() + self.assertEqual( + [c.args[1] for c in get_hf_hub.call_args_list], + ["model.onnx", "tokenizer.json", "config.json"], + ) + self.assertEqual(tagger.id2label, {0: "NOUN", 1: "VERB"}) + # tokenizer.json truncates at 510 tokens; the tagger must undo it. + tagger.tokenizer.no_truncation.assert_called_once_with() + tagger.tokenizer.no_padding.assert_called_once_with() + + def test_tag_function_loads_tagger_once(self): + from unittest import mock + + from pythainlp.tag import phayathaibert_onnx + + with mock.patch.object(phayathaibert_onnx, "_TAGGER", None): + with mock.patch.object( + phayathaibert_onnx, "PhayaThaiBERTTagger" + ) as tagger_class: + tagger_class.return_value.tag.return_value = [("ก", "NOUN")] + for _ in range(2): + self.assertEqual( + phayathaibert_onnx.tag(["ก"], corpus="orchid"), + [("ก", "NOUN")], + ) + tagger_class.assert_called_once_with() + + def test_pos_tag_phayathaibert_uses_tud(self): + from unittest import mock + + with mock.patch( + "pythainlp.tag.phayathaibert_onnx.tag", + return_value=[("ก", "NOUN")], + ) as tag: + self.assertEqual( + pos_tag(["ก"], engine="phayathaibert", corpus="orchid"), + [("ก", "NOUN")], + ) + tag.assert_called_once_with(["ก"], corpus="tud") + + def test_pos_tag_transformers_engine_lookup(self): + import sys + from unittest import mock + + from pythainlp.tag import pos_tag_transformers + + transformers = mock.MagicMock() + pipeline = transformers.TokenClassificationPipeline.return_value + pipeline.return_value = [{"word": "กิน", "entity_group": "VERB"}] + supported = [ + ("bert", "blackboard", "lunarlist/pos_thai"), + ("phayathai", "blackboard", "lunarlist/pos_thai_phayathai"), + ( + "wangchanberta", + "pud", + "Pavarissy/wangchanberta-ud-thai-pud-upos", + ), + ("mdeberta", "pud", "Pavarissy/mdeberta-v3-ud-thai-pud-upos"), + ( + "phayathaibert", + "tud", + "nlp-chula/phayathaibert-thai-pos-tagger", + ), + ] + unsupported = [ + ("phayathaibert", "pud"), + ("bert", "tud"), + ("bert", "non-existing corpus"), + ] + auto_tokenizer = transformers.AutoTokenizer.from_pretrained + with mock.patch.dict(sys.modules, {"transformers": transformers}): + for engine, corpus, model_name in supported: + with self.subTest(engine=engine, corpus=corpus): + self.assertEqual( + pos_tag_transformers( + "กิน", engine=engine, corpus=corpus + ), + [[("กิน", "VERB")]], + ) + auto_tokenizer.assert_called_with( + model_name, revision=None + ) + for engine, corpus in unsupported: + with self.subTest(engine=engine, corpus=corpus): + with self.assertRaises(ValueError): + pos_tag_transformers( + "กิน", engine=engine, corpus=corpus + ) diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index 47dce547c..51a11638f 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -54,8 +54,7 @@ def test_pos_tag_wangchanberta_onnx_empty_list(self): class TagPhayaThaiBERTONNXTestCaseN(unittest.TestCase): - """Tests for the PhayaThaiBERT ONNX POS tagger - (requires onnxruntime, tokenizers, huggingface-hub)""" + """Test the PhayaThaiBERT ONNX POS tagger.""" WORDS = ["ฉัน", "กิน", "ข้าว", "ที่", "ร้านอาหาร"] From f97fda16ee11d00af0503c15ef44a113b535b130 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 10:26:35 +0700 Subject: [PATCH 04/12] Limit PhayaThaiBERT input chunks to 415 tokens The model accepts 512 positions, but its accuracy drops sharply from position 415 on, in the original PyTorch model as well as the ONNX export. With chunks of up to 510 tokens, tagging the whole UD Thai-TUD test set as one input gave 80.26% accuracy; with 415 it gives 90.08% (91.21% sentence by sentence). The long-input test now checks that every repeat of a sentence gets the same tags as the sentence alone, which fails with the old limit. --- pythainlp/tag/phayathaibert_onnx.py | 6 ++++-- tests/noauto_onnx/testn_tag_onnx.py | 15 +++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 3d6f296c4..5a3f35b43 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -30,8 +30,10 @@ # Pinned for reproducible and secure downloads. _REVISION = "ff55e2c66ef22deee18c423cee272f6e207171ae" -# Maximum number of subword tokens the model accepts, including and . -_MAX_SEQUENCE_LENGTH = 510 +# Maximum number of subword tokens per model call, including and . +# The model accepts 512, but its accuracy drops sharply from position 415 +# on, which suggests it was pretrained on shorter sequences. +_MAX_SEQUENCE_LENGTH = 415 # Tag for words the model never sees (whitespace-only words), matching # the perceptron tagger with Universal POS corpora. diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index 51a11638f..4aa234cc2 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -95,18 +95,17 @@ def test_pos_tag_phayathaibert_whitespace_words(self): def test_pos_tag_phayathaibert_long_input(self): from pythainlp.tag import pos_tag - words = self.WORDS * 300 # well over the 510-token model limit - result = pos_tag(words, engine="phayathaibert") - tags = [t for _, t in result] - self.assertEqual([w for w, _ in result], words) - # Every word, including those past the first 510 tokens, is tagged. - self.assertNotIn("X", tags) - self.assertEqual(tags[-5:], ["PRON", "VERB", "NOUN", "ADP", "NOUN"]) + sentence = pos_tag(self.WORDS, engine="phayathaibert") + # Far over the usable length of 415 tokens, so the input is chunked. + result = pos_tag(self.WORDS * 300, engine="phayathaibert") + # Words late in a chunk must be tagged as well as early ones; the + # model is unreliable past position 414 even though it accepts 512. + self.assertEqual(result, sentence * 300) def test_pos_tag_phayathaibert_overlong_word(self): from pythainlp.tag import pos_tag - # A single word of far more than 510 subwords, between normal words. + # A single word of far more than 415 subwords, between normal words. words = ["ฉัน", "กิน", "ก" * 5000, "ที่", "ร้านอาหาร"] result = pos_tag(words, engine="phayathaibert") tags = [t for _, t in result] From 065e0c27fca9d792c5847fd6b8d0ebd27c8bb90e Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 10:31:07 +0700 Subject: [PATCH 05/12] Pin PhayaThaiBERT ONNX model to the revision with the updated model card Only README.md changed on the Hugging Face repository; model.onnx, tokenizer.json and config.json are the same files. --- pythainlp/tag/phayathaibert_onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 5a3f35b43..3d43a988d 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -28,7 +28,7 @@ _REPO_ID = "wiriyabot/phayathaibert-thai-pos-tagger-onnx" # Pinned for reproducible and secure downloads. -_REVISION = "ff55e2c66ef22deee18c423cee272f6e207171ae" +_REVISION = "59e3502bd06e2222d32f11b51b4fb9569235a020" # Maximum number of subword tokens per model call, including and . # The model accepts 512, but its accuracy drops sharply from position 415 From 6ac2d761f6ca4192b093bb036d3752028eb9ba0e Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 10:42:07 +0700 Subject: [PATCH 06/12] Sort the pos_tag_transformers engine lookup keys --- pythainlp/tag/pos_tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index 81ba68f3f..cbdb5525d 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -275,8 +275,8 @@ def pos_tag_transformers( "phayathai": "lunarlist/pos_thai_phayathai", }, "pud": { - "wangchanberta": "Pavarissy/wangchanberta-ud-thai-pud-upos", "mdeberta": "Pavarissy/mdeberta-v3-ud-thai-pud-upos", + "wangchanberta": "Pavarissy/wangchanberta-ud-thai-pud-upos", }, "tud": { "phayathaibert": "nlp-chula/phayathaibert-thai-pos-tagger", From dc1658c5b4f8d780ffe7613f512d149047225447 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 12:09:51 +0700 Subject: [PATCH 07/12] Link the PhayaThaiBERT change log entry to its PR --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c1a2a077..cc6a11c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,8 @@ and this project adheres to ([#1509]) - `evaluate_word_tokenization`, `word_eval_function`, `char_eval_function`, and `TokenizationScore` in `pythainlp.benchmarks` ([#1512]) -- PhayaThaiBERT POS tagger trained on TUD: `pos_tag()` engine `phayathaibert` - (ONNX, extra `phayathaibert_onnx`); `pos_tag_transformers()` with corpus `tud` ([#1536]) +- PhayaThaiBERT POS tagger: `pos_tag()` engine `phayathaibert` (ONNX) and + `pos_tag_transformers()` corpus `tud` ([#1550]) ### Changed @@ -88,12 +88,12 @@ and this project adheres to [#1512]: https://github.com/PyThaiNLP/pythainlp/pull/1512 [#1526]: https://github.com/PyThaiNLP/pythainlp/pull/1526 [#1529]: https://github.com/PyThaiNLP/pythainlp/pull/1529 -[#1536]: https://github.com/PyThaiNLP/pythainlp/issues/1536 [#1541]: https://github.com/PyThaiNLP/pythainlp/pull/1541 [#1542]: https://github.com/PyThaiNLP/pythainlp/pull/1542 [#1546]: https://github.com/PyThaiNLP/pythainlp/pull/1546 [#1547]: https://github.com/PyThaiNLP/pythainlp/pull/1547 [#1548]: https://github.com/PyThaiNLP/pythainlp/pull/1548 +[#1550]: https://github.com/PyThaiNLP/pythainlp/pull/1550 [#1551]: https://github.com/PyThaiNLP/pythainlp/pull/1551 [#1552]: https://github.com/PyThaiNLP/pythainlp/pull/1552 From 6bcec8de88b48de1291b4451695409b934c037e9 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 12:30:30 +0700 Subject: [PATCH 08/12] Fix Codacy and SonarCloud issues in the PhayaThaiBERT tagger - Check dependencies with importlib.util.find_spec instead of unused imports (pylint W0611). - Use a one-line module docstring (pydocstyle D212); the model description moves to the class docstring. - Mark the unused corpus parameter of tag() for SonarCloud (S1172). - Tests: end a docstring with a period (D415), keep one assertion argument order (S3415), and use assertLessEqual (S5906). --- pythainlp/tag/phayathaibert_onnx.py | 35 +++++++++++++++------------ tests/core/test_tag.py | 19 ++++++++++++--- tests/noauto_onnx/testn_tag_onnx.py | 2 +- tests/noauto_torch/testn_tag_torch.py | 2 +- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 3d43a988d..71d0a5d2e 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -1,14 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -""" -PhayaThaiBERT part-of-speech tagger with ONNX Runtime backend. - -The model is an ONNX export of -`nlp-chula/phayathaibert-thai-pos-tagger -`_, -fine-tuned on the UD Thai-TUD treebank. It tags words with Universal POS tags. -""" +"""PhayaThaiBERT part-of-speech tagger with ONNX Runtime backend.""" from __future__ import annotations @@ -16,6 +9,7 @@ import threading import unicodedata from collections import Counter +from importlib.util import find_spec from typing import TYPE_CHECKING, Optional from pythainlp.corpus import get_hf_hub @@ -42,6 +36,9 @@ # Universal POS tag for words that produce no subword token. _UNKNOWN_TAG = "X" +# Modules the tagger needs at run time. +_DEPENDENCIES = ("huggingface_hub", "numpy", "onnxruntime", "tokenizers") + def _is_blank(word: str) -> bool: """ @@ -119,6 +116,11 @@ class PhayaThaiBERTTagger: """ Universal POS tagger using PhayaThaiBERT with ONNX Runtime. + The model is an ONNX export of + `nlp-chula/phayathaibert-thai-pos-tagger + `_, + fine-tuned on the UD Thai-TUD treebank. + Requires ``numpy``, ``onnxruntime``, ``tokenizers`` and ``huggingface-hub``. The model (about 530 MB) is downloaded from the Hugging Face Hub on first use. @@ -144,17 +146,15 @@ def __init__( ``None``. :raises ImportError: if a required dependency is not installed """ - try: - import huggingface_hub # noqa: F401 - import numpy # noqa: F401 - from onnxruntime import InferenceSession - from tokenizers import Tokenizer - except ImportError as e: + # Check every dependency before the download starts. + if any(find_spec(name) is None for name in _DEPENDENCIES): raise ImportError( "PhayaThaiBERT POS tagger requires numpy, onnxruntime," " tokenizers and huggingface-hub." ' Install them with: pip install "pythainlp[phayathaibert_onnx]"' - ) from e + ) + from onnxruntime import InferenceSession + from tokenizers import Tokenizer if revision is None and repo_id == _REPO_ID: revision = _REVISION @@ -242,7 +242,10 @@ def _predict(self, encoding: Encoding, n_words: int) -> list[str]: _TAGGER_LOCK = threading.Lock() -def tag(words: list[str], corpus: str = "tud") -> list[tuple[str, str]]: +def tag( + words: list[str], + corpus: str = "tud", # NOSONAR: unused, kept for the pos_tag interface +) -> list[tuple[str, str]]: """ Tag words with Universal POS tags using PhayaThaiBERT (ONNX). diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 7331e25e9..b4c4e62f7 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -29,6 +29,17 @@ ) +def _fake_module(name): + """Return a stand-in for an installed module, for patching sys.modules.""" + from importlib.machinery import ModuleSpec + from unittest import mock + + module = mock.MagicMock() + # importlib.util.find_spec() reads __spec__ of modules in sys.modules. + module.__spec__ = ModuleSpec(name, None) + return module + + class TagTestCase(unittest.TestCase): """Test pythainlp.tag.pos_tag.""" @@ -470,7 +481,7 @@ def test_iob_to_markup_long_input(self): class PhayaThaiBERTHelperTestCase(unittest.TestCase): - """Test the pure-Python helpers of pythainlp.tag.phayathaibert_onnx""" + """Test the pure-Python helpers of pythainlp.tag.phayathaibert_onnx.""" def test_first_subword_labels(self): from pythainlp.tag.phayathaibert_onnx import _first_subword_labels @@ -551,7 +562,7 @@ def test_missing_dependency_error_before_download(self): # Fake every dependency, so none is really imported (and # then unloaded) inside patch.dict. modules = { - m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES + m: _fake_module(m) for m in _PHAYATHAIBERT_DEPENDENCIES } modules[module] = None with mock.patch.dict(sys.modules, modules): @@ -573,7 +584,7 @@ def _first_download_revision(self, **kwargs: str) -> object: from pythainlp.tag import phayathaibert_onnx fake_modules = { - m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES + m: _fake_module(m) for m in _PHAYATHAIBERT_DEPENDENCIES } with mock.patch.dict(sys.modules, fake_modules): with mock.patch( @@ -700,7 +711,7 @@ def test_init(self): from pythainlp.tag import phayathaibert_onnx fake_modules = { - m: mock.MagicMock() for m in _PHAYATHAIBERT_DEPENDENCIES + m: _fake_module(m) for m in _PHAYATHAIBERT_DEPENDENCIES } with tempfile.TemporaryDirectory() as tmp: config_path = path.join(tmp, "config.json") diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index 4aa234cc2..cb2f7c789 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -88,8 +88,8 @@ def test_pos_tag_phayathaibert_whitespace_words(self): self.assertEqual(result[1], (" ", "PUNCT")) self.assertEqual(result[3], ("", "PUNCT")) self.assertEqual( - [result[0], result[2]], pos_tag(["แมว", "กิน"], engine="phayathaibert"), + [result[0], result[2]], ) def test_pos_tag_phayathaibert_long_input(self): diff --git a/tests/noauto_torch/testn_tag_torch.py b/tests/noauto_torch/testn_tag_torch.py index 651139e87..97f4ddfb6 100644 --- a/tests/noauto_torch/testn_tag_torch.py +++ b/tests/noauto_torch/testn_tag_torch.py @@ -126,7 +126,7 @@ def test_pos_tag_transformers_phayathaibert_tud(self): self.assertEqual(len(result), 1) self.assertGreater(len(result[0]), 0) tags = {tag for _, tag in result[0]} - self.assertTrue(tags <= upos, tags - upos) + self.assertLessEqual(tags, upos, tags - upos) self.assertIn("VERB", tags) with self.assertRaises(ValueError): From 68bd287070ee87e14d5d22c5526573815cd9563d Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 13:06:26 +0700 Subject: [PATCH 09/12] Add type hints to the PhayaThaiBERT test helpers CI runs mypy on tests too (mypy 1.19.1, Python 3.9), which rejects calls to the untyped fake model helpers. --- tests/core/test_tag.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index b4c4e62f7..102c88bdf 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -2,9 +2,13 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import unittest from importlib.util import find_spec from os import path +from typing import TYPE_CHECKING, Any, Optional +from unittest import mock from pythainlp.corpus import download from pythainlp.tag import ( @@ -19,6 +23,9 @@ ) from pythainlp.tag._utils import _iob_to_markup +if TYPE_CHECKING: + from pythainlp.tag.phayathaibert_onnx import PhayaThaiBERTTagger + TEST_TOKENS = ["ผม", "รัก", "คุณ"] _PHAYATHAIBERT_DEPENDENCIES = ( @@ -29,10 +36,9 @@ ) -def _fake_module(name): +def _fake_module(name: str) -> mock.MagicMock: """Return a stand-in for an installed module, for patching sys.modules.""" from importlib.machinery import ModuleSpec - from unittest import mock module = mock.MagicMock() # importlib.util.find_spec() reads __spec__ of modules in sys.modules. @@ -561,7 +567,7 @@ def test_missing_dependency_error_before_download(self): with self.subTest(module): # Fake every dependency, so none is really imported (and # then unloaded) inside patch.dict. - modules = { + modules: dict[str, Optional[mock.MagicMock]] = { m: _fake_module(m) for m in _PHAYATHAIBERT_DEPENDENCIES } modules[module] = None @@ -623,9 +629,9 @@ class _FakeEncoding: subword but the first gives the wrong tag. """ - def __init__(self, words): - self.ids = [0] - self.word_ids = [None] + def __init__(self, words: list[str]) -> None: + self.ids: list[int] = [0] + self.word_ids: list[Optional[int]] = [None] for i, word in enumerate(words): label_id = 1 if word.startswith("ก") else 2 self.ids += [label_id] + [0] * (len(word) - 1) @@ -636,17 +642,21 @@ def __init__(self, words): class _FakeTokenizer: - def encode(self, words, is_pretokenized=False): + def encode( + self, words: list[str], is_pretokenized: bool = False + ) -> _FakeEncoding: return _FakeEncoding(words) class _FakeSession: """Predict the label id equal to each input id (one-hot logits).""" - def __init__(self): - self.lengths = [] + def __init__(self) -> None: + self.lengths: list[int] = [] - def run(self, output_names, feeds): + def run( + self, output_names: Optional[list[str]], feeds: dict[str, Any] + ) -> list[Any]: import numpy as np ids = feeds["input_ids"] @@ -657,7 +667,7 @@ def run(self, output_names, feeds): class PhayaThaiBERTTaggerTestCase(unittest.TestCase): """Test pythainlp.tag.phayathaibert_onnx with a fake model.""" - def _tagger(self): + def _tagger(self) -> PhayaThaiBERTTagger: from pythainlp.tag.phayathaibert_onnx import PhayaThaiBERTTagger tagger = PhayaThaiBERTTagger.__new__(PhayaThaiBERTTagger) From 875bc91703b548cb8161e0907f867bcf5f76e83c Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 13:10:33 +0700 Subject: [PATCH 10/12] Drop local mock imports now imported at module level Fixes pylint W0404 (reimport) reported by Codacy. --- tests/core/test_tag.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 102c88bdf..359346485 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -559,7 +559,6 @@ def test_tag_empty_list(self): def test_missing_dependency_error_before_download(self): import sys - from unittest import mock from pythainlp.tag import phayathaibert_onnx @@ -585,7 +584,6 @@ def test_missing_dependency_error_before_download(self): def _first_download_revision(self, **kwargs: str) -> object: """Return the revision PhayaThaiBERTTagger passes to get_hf_hub.""" import sys - from unittest import mock from pythainlp.tag import phayathaibert_onnx @@ -678,7 +676,6 @@ def _tagger(self) -> PhayaThaiBERTTagger: @unittest.skipUnless(find_spec("numpy"), "numpy is not installed") def test_tag(self): - from unittest import mock from pythainlp.tag import phayathaibert_onnx @@ -716,7 +713,6 @@ def test_init(self): import json import sys import tempfile - from unittest import mock from pythainlp.tag import phayathaibert_onnx @@ -748,7 +744,6 @@ def fake_get_hf_hub(repo_id, filename, revision=None): tagger.tokenizer.no_padding.assert_called_once_with() def test_tag_function_loads_tagger_once(self): - from unittest import mock from pythainlp.tag import phayathaibert_onnx @@ -765,7 +760,6 @@ def test_tag_function_loads_tagger_once(self): tagger_class.assert_called_once_with() def test_pos_tag_phayathaibert_uses_tud(self): - from unittest import mock with mock.patch( "pythainlp.tag.phayathaibert_onnx.tag", @@ -779,7 +773,6 @@ def test_pos_tag_phayathaibert_uses_tud(self): def test_pos_tag_transformers_engine_lookup(self): import sys - from unittest import mock from pythainlp.tag import pos_tag_transformers From e50f2c37c9e5366a20ca427286f3526aeaa11b79 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 13:16:21 +0700 Subject: [PATCH 11/12] Escape invisible characters in tests and tidy the tagger - Write zero-width and special space characters in tests as \u escapes, so they are visible and editors cannot strip them (pylint E2515). - Rename a loop variable that shadowed the module-level tag() (W0621). - Correct the length comment: the model runs on up to 510 tokens. --- pythainlp/tag/phayathaibert_onnx.py | 10 ++++++---- tests/core/test_tag.py | 8 ++++---- tests/noauto_onnx/testn_tag_onnx.py | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 71d0a5d2e..1a3114a19 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -25,8 +25,8 @@ _REVISION = "59e3502bd06e2222d32f11b51b4fb9569235a020" # Maximum number of subword tokens per model call, including and . -# The model accepts 512, but its accuracy drops sharply from position 415 -# on, which suggests it was pretrained on shorter sequences. +# The model runs on up to 510 tokens, but its accuracy drops sharply from +# position 415 on, which suggests it was pretrained on shorter sequences. _MAX_SEQUENCE_LENGTH = 415 # Tag for words the model never sees (whitespace-only words), matching @@ -194,8 +194,10 @@ def tag(self, words: list[str]) -> list[tuple[str, str]]: positions = [i for i, word in enumerate(words) if not _is_blank(word)] content = [words[i] for i in positions] if content: - for position, tag in zip(positions, self._tag_content(content)): - tags[position] = tag + for position, word_tag in zip( + positions, self._tag_content(content) + ): + tags[position] = word_tag return list(zip(words, tags)) def _tag_content(self, words: list[str]) -> list[str]: diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 359346485..f6f42f02c 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -547,9 +547,9 @@ def test_chunk_spans_zero_count_words(self): def test_is_blank(self): from pythainlp.tag.phayathaibert_onnx import _is_blank - for word in ("", " ", "\n", " ", "　", "​", ""): + for word in ("", " ", "\n", "\u00a0", "\u3000", "\u200b", "\ufeff"): self.assertTrue(_is_blank(word), repr(word)) - for word in ("แมว", " แมว ", "a", "1", "​แมว"): + for word in ("แมว", " แมว ", "a", "1", "\u200bแมว"): self.assertFalse(_is_blank(word), repr(word)) def test_tag_empty_list(self): @@ -683,11 +683,11 @@ def test_tag(self): ("short", ["กา", "ขาว"], 20, ["NOUN", "VERB"]), ( "blank words skipped", - ["กา", " ", "ขา", "", "​"], + ["กา", " ", "ขา", "", "\u200b"], 20, ["NOUN", "PUNCT", "VERB", "PUNCT", "PUNCT"], ), - ("all blank", [" ", ""], 20, ["PUNCT", "PUNCT"]), + ("all blank", [" ", "\ufeff"], 20, ["PUNCT", "PUNCT"]), ( "chunked", ["กา", "ขา", "ก", "ขาว", "กก", "ขข"], diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index cb2f7c789..ec5c9dbef 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -99,7 +99,7 @@ def test_pos_tag_phayathaibert_long_input(self): # Far over the usable length of 415 tokens, so the input is chunked. result = pos_tag(self.WORDS * 300, engine="phayathaibert") # Words late in a chunk must be tagged as well as early ones; the - # model is unreliable past position 414 even though it accepts 512. + # model is unreliable past position 414 even though it runs on 510. self.assertEqual(result, sentence * 300) def test_pos_tag_phayathaibert_overlong_word(self): @@ -118,7 +118,7 @@ def test_pos_tag_phayathaibert_format_characters(self): from pythainlp.tag import pos_tag # Zero-width space, zero-width non-joiner and BOM carry no text. - for char in ("​", "‌", ""): + for char in ("\u200b", "\u200c", "\ufeff"): result = pos_tag(["แมว", char, "กิน"], engine="phayathaibert") self.assertEqual(result[1], (char, "PUNCT")) From 9a5da2183444e4c934b46afffec013ea929db293 Mon Sep 17 00:00:00 2001 From: wiriyabot Date: Wed, 7 Oct 2026 21:08:04 +0700 Subject: [PATCH 12/12] Fix RUF005 and RUF012 in the PhayaThaiBERT tagger - Build the truncated word_ids list by unpacking, not concatenation. - Annotate the shared WORDS test attribute as ClassVar. --- pythainlp/tag/phayathaibert_onnx.py | 2 +- tests/noauto_onnx/testn_tag_onnx.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pythainlp/tag/phayathaibert_onnx.py b/pythainlp/tag/phayathaibert_onnx.py index 1a3114a19..9d88ae4e4 100644 --- a/pythainlp/tag/phayathaibert_onnx.py +++ b/pythainlp/tag/phayathaibert_onnx.py @@ -227,7 +227,7 @@ def _predict(self, encoding: Encoding, n_words: int) -> list[str]: # Only a single over-long word gets here; its first subword stays. ids = ids[: _MAX_SEQUENCE_LENGTH - 1] + ids[-1:] mask = mask[:_MAX_SEQUENCE_LENGTH] - word_ids = word_ids[: _MAX_SEQUENCE_LENGTH - 1] + [None] + word_ids = [*word_ids[: _MAX_SEQUENCE_LENGTH - 1], None] logits = self.session.run( None, { diff --git a/tests/noauto_onnx/testn_tag_onnx.py b/tests/noauto_onnx/testn_tag_onnx.py index ec5c9dbef..f2174f25c 100644 --- a/tests/noauto_onnx/testn_tag_onnx.py +++ b/tests/noauto_onnx/testn_tag_onnx.py @@ -9,6 +9,7 @@ # - Version constraints import unittest +from typing import ClassVar class TagONNXTestCaseN(unittest.TestCase): @@ -56,7 +57,7 @@ def test_pos_tag_wangchanberta_onnx_empty_list(self): class TagPhayaThaiBERTONNXTestCaseN(unittest.TestCase): """Test the PhayaThaiBERT ONNX POS tagger.""" - WORDS = ["ฉัน", "กิน", "ข้าว", "ที่", "ร้านอาหาร"] + WORDS: ClassVar[list[str]] = ["ฉัน", "กิน", "ข้าว", "ที่", "ร้านอาหาร"] def test_pos_tag_phayathaibert(self): from pythainlp.tag import pos_tag