From 7d234fdc0f6b9951aead1b7d9649017a75c5575b Mon Sep 17 00:00:00 2001 From: Adrien Barbaresi Date: Thu, 10 Sep 2026 16:50:55 +0200 Subject: [PATCH] feat: add py3langid to the benchmark --- README.md | 5 ++- pyproject.toml | 2 + src/commonlid/models/__init__.py | 1 + src/commonlid/models/py3langid.py | 45 ++++++++++++++++++++ tests/models/test_model_registration.py | 1 + tests/models/test_py3langid.py | 56 +++++++++++++++++++++++++ uv.lock | 19 ++++++++- 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 src/commonlid/models/py3langid.py create mode 100644 tests/models/test_py3langid.py diff --git a/README.md b/README.md index 1731d00..2fb697d 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,8 @@ from commonlid import list_models, list_datasets assert list_models() == [ "AfroLID", "GlotLID", "OpenLID-v2", "cld2", "cld3", - "commonlingua", "fasttext", "funlangid", "pyfranc", + "commonlingua", "fasttext", "funlangid", "py3langid", + "pyfranc", ] assert list_datasets() == [ "bibles_300", "bibles_300_nano", @@ -323,6 +324,7 @@ for line in preds_path.read_text().splitlines(): | `AfroLID` | [UBC-NLP/afrolid_1.5](https://huggingface.co/UBC-NLP/afrolid_1.5) | Requires `[afrolid]` extra | | `commonlingua` | [PleIAs/CommonLingua](https://huggingface.co/PleIAs/CommonLingua) | 2.35M-param byte-level model, 334 languages; requires `[commonlingua]` extra | | `funlangid` | Vendored in `src/commonlid/vendor/fun_langid.py` | Simple char-4gram baseline | +| `py3langid` | [py3langid](https://pypi.org/project/py3langid/) | Pure Python + numpy, 139 languages; abstains below 0.25 confidence. Requires `[py3langid]` extra | LLM models are instantiated dynamically (`DSPyLLMModel`) and not auto-registered — they need per-instance configuration (endpoint + key). @@ -466,6 +468,7 @@ happen to share a range. Only compare a model's scores with its own. | `cld3` | yes | The neural net's softmax probability, 0-1 | | `AfroLID` | yes | The text-classification pipeline's softmax probability, 0-1 | | `commonlingua` | yes | Softmax over the model's class logits, 0-1 | +| `py3langid` | yes | Normalized naive Bayes posterior of the top label, 0-1; predictions below 0.25 abstain but keep their score | | `funlangid` | yes | Fraction of the text's character 4-grams covered by the winning language's lexicon. Does not sum to 1 across languages | | `GlotLID`, `OpenLID-v2`, `fasttext` | no | The batched `multilinePredict` these wrappers use returns bare labels under the pinned `fasttext-predict` dependency; see the note in `src/commonlid/models/_fasttext_base.py` | | `cld2` | no | CLD2 returns `percent` (share of the input in that language) and an unbounded internal `score` in the hundreds. Neither is a confidence | diff --git a/pyproject.toml b/pyproject.toml index af844b9..cc71879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ commonlingua = [ # transformers stack that [afrolid] pulls in. "torch>=2.4", ] +py3langid = ["py3langid>=0.4"] leaderboard = [ # gradio 4.x imports HfFolder from huggingface_hub, which was removed in # huggingface-hub 1.0; gradio 5 dropped that import. @@ -175,6 +176,7 @@ module = [ "fasttext.*", "pycld2.*", "pyfranc.*", + "py3langid.*", "iso639.*", "datasets.*", "huggingface_hub.*", diff --git a/src/commonlid/models/__init__.py b/src/commonlid/models/__init__.py index cb74c2f..ec9b5d4 100644 --- a/src/commonlid/models/__init__.py +++ b/src/commonlid/models/__init__.py @@ -16,6 +16,7 @@ from commonlid.models import funlangid as _funlangid # noqa: F401 from commonlid.models import glotlid as _glotlid # noqa: F401 from commonlid.models import openlidv2 as _openlidv2 # noqa: F401 +from commonlid.models import py3langid as _py3langid # noqa: F401 from commonlid.models import pyfranc as _pyfranc # noqa: F401 from commonlid.models.dspy_llm import DSPyLLMModel diff --git a/src/commonlid/models/py3langid.py b/src/commonlid/models/py3langid.py new file mode 100644 index 0000000..108b4de --- /dev/null +++ b/src/commonlid/models/py3langid.py @@ -0,0 +1,45 @@ +"""py3langid wrapper: does its own normalization; zxx and sub-threshold predictions abstain.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from commonlid.core.lid_model import LIDModel, LIDPrediction +from commonlid.core.registry import register_model + +# codes the generic 639-1 -> 639-3 conformation would not reach +_OUT_MAP = {"ar": "arb", "sw": "swh", "az": "azj", "or": "ory", "bcl": "bik"} +_MIN_CONFIDENCE = 0.25 # best micro+macro F1 on commonlid + + +@register_model +class Py3LangIDModel(LIDModel): + model_id = "py3langid" + requires_preprocessing = False + + def load(self) -> None: + from py3langid.langid import MODEL_FILE, LanguageIdentifier + + self._identifier = LanguageIdentifier.from_model_file( + MODEL_FILE, norm_probs=True, min_confidence=_MIN_CONFIDENCE + ) + super().load() + + def _predict_batch(self, texts: Sequence[str]) -> list[LIDPrediction]: + out: list[LIDPrediction] = [] + for text in texts: + code, score = self._identifier.classify(text) + label = None if code in ("zxx", "und") else _OUT_MAP.get(code, code) + out.append(LIDPrediction(label, float(score))) + return out + + def discover_supported_languages(self) -> frozenset[str]: + self.load() + codes: set[str] = set() + for code in self._identifier.labels: + if code == "zxx": + continue + conformed = self._conform(_OUT_MAP.get(code, code)) + if conformed is not None: + codes.add(conformed) + return frozenset(codes) diff --git a/tests/models/test_model_registration.py b/tests/models/test_model_registration.py index c46b92e..b11b2b5 100644 --- a/tests/models/test_model_registration.py +++ b/tests/models/test_model_registration.py @@ -23,6 +23,7 @@ def _import_models() -> None: "commonlingua", "fasttext", "funlangid", + "py3langid", "pyfranc", } diff --git a/tests/models/test_py3langid.py b/tests/models/test_py3langid.py new file mode 100644 index 0000000..1576b40 --- /dev/null +++ b/tests/models/test_py3langid.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import pytest + +pytest.importorskip("py3langid") + +from commonlid.models.py3langid import _MIN_CONFIDENCE, _OUT_MAP, Py3LangIDModel + + +def test_returns_conformed_codes() -> None: + m = Py3LangIDModel() + preds = m.predict([ + "The quick brown fox jumps over the lazy dog.", + "Le renard brun saute par-dessus le chien paresseux.", + "Der schnelle braune Fuchs springt über den faulen Hund.", + ]) + assert preds == ["eng", "fra", "deu"] + + +def test_predict_scored_returns_the_posterior() -> None: + m = Py3LangIDModel() + scored = m.predict_scored(["The quick brown fox jumps over the lazy dog.", "123 456"]) + assert scored[0].iso639_3 == "eng" + assert scored[0].score is not None + assert _MIN_CONFIDENCE < scored[0].score <= 1.0 + assert scored[1].iso639_3 is None + assert scored[1].score is not None + assert 0.0 <= scored[1].score <= 1.0 + + +def test_abstains_on_empty_and_gibberish() -> None: + m = Py3LangIDModel() + assert m.predict(["", " ", "123 456", "!!!"]) == [None, None, None, None] + + +def test_out_map_overrides_default_conformation() -> None: + m = Py3LangIDModel() + assert _OUT_MAP["ar"] == "arb" + assert m.predict(["هذا نص قصير مكتوب باللغة العربية الفصحى الحديثة."]) == ["arb"] + bikol = "An mga tawo dapat magkaminootan asin magtinabangan sa lambang aldaw." + assert m.predict([bikol]) == ["bik"] + + +def test_discover_supported_languages() -> None: + m = Py3LangIDModel() + langs = m.discover_supported_languages() + assert {"eng", "fra", "deu", "arz", "vec"} <= langs + assert "zxx" not in langs + assert "bik" in langs + assert not {"bcl", "ar", "und"} & langs + + +def test_min_confidence_is_in_effect() -> None: + m = Py3LangIDModel() + m.load() + assert m._identifier.min_confidence == _MIN_CONFIDENCE diff --git a/uv.lock b/uv.lock index 9ef226b..834caed 100644 --- a/uv.lock +++ b/uv.lock @@ -900,6 +900,9 @@ notebooks = [ { name = "matplotlib" }, { name = "nbclient" }, ] +py3langid = [ + { name = "py3langid" }, +] [package.metadata] requires-dist = [ @@ -928,6 +931,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.26,<3" }, { name = "pandas", specifier = ">=2.1" }, { name = "protobuf", marker = "extra == 'afrolid'", specifier = ">=4.25" }, + { name = "py3langid", marker = "extra == 'py3langid'", specifier = ">=0.4" }, { name = "pycld2", specifier = ">=0.42" }, { name = "pyfranc", specifier = ">=0.2.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, @@ -944,7 +948,7 @@ requires-dist = [ { name = "types-regex", marker = "extra == 'dev'" }, { name = "types-tqdm", marker = "extra == 'dev'" }, ] -provides-extras = ["afrolid", "llm", "cld3", "commonlingua", "leaderboard", "dev", "notebooks", "all"] +provides-extras = ["afrolid", "llm", "cld3", "commonlingua", "py3langid", "leaderboard", "dev", "notebooks", "all"] [[package]] name = "contourpy" @@ -4439,6 +4443,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py3langid" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/fc/51c88e5ef8346878ae6c09e1f7c86c6e6308c57794723bf423a72ddba4dd/py3langid-0.4.0.tar.gz", hash = "sha256:5924159039b5cc282c2edc10a7ea9363b2099834a0e99ed8d5bea3cc435d7871", size = 4604185, upload-time = "2026-09-02T09:10:53.983Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/2d/0eeff2727c970b1d553be70d15ba2d84e3830696c09a79d954c5ae3d5212/py3langid-0.4.0-py3-none-any.whl", hash = "sha256:c25bd038147951ba3a89efe9ec12cc7d2116cb6f311ff8af84ade398a8bc0616", size = 4600605, upload-time = "2026-09-02T09:10:49.54Z" }, +] + [[package]] name = "pyarrow" version = "23.0.1"