feat: add py3langid to the benchmark

README.md

@@ -219,7 +219,7 @@ from commonlid import list_models, list_datasets
 assert list_models() == [
     "AfroLID", "GlotLID", "GoogleTranslate-v2", "GoogleTranslate-v3",
     "OpenLID-v2", "cld2", "cld3", "commonlingua", "fasttext",
-    "funlangid", "pyfranc",
+    "funlangid", "py3langid", "pyfranc",
 ]
 assert list_datasets() == [
     "bibles_300", "bibles_300_nano",
@@ -329,6 +329,7 @@ for line in preds_path.read_text().splitlines():
 | `funlangid` | Vendored in `src/commonlid/vendor/fun_langid.py` | Simple char-4gram baseline |
 | `GoogleTranslate-v2` | [Cloud Translation Basic (v2)](https://docs.cloud.google.com/translate/docs/basic/detecting-language) | Requires `[google-translate]` extra + the `GOOGLE_TRANSLATE_API_KEY` env var |
 | `GoogleTranslate-v3` | [Cloud Translation Advanced (v3)](https://docs.cloud.google.com/translate/docs/advanced/detecting-language-v3) | Requires `[google-translate]` extra + Application Default Credentials. v3 does not accept API keys |
+| `py3langid` | [py3langid](https://pypi.org/project/py3langid/) | Pure Python + numpy, 139 languages; abstains below 0.25 confidence. Requires `[py3langid]` extra |
 
 LLM models are instantiated dynamically (`DSPyLLMModel`) and not
 auto-registered — they need per-instance configuration (endpoint + key).
@@ -478,6 +479,7 @@ happen to share a range. Only compare a model's scores with its own.
 | `cld3` | yes | The neural net's softmax probability, 0-1 |
 | `AfroLID` | yes | The text-classification pipeline's softmax probability, 0-1 |
 | `commonlingua` | yes | Softmax over the model's class logits, 0-1 |
+| `py3langid` | yes | Normalized naive Bayes posterior of the top label, 0-1; predictions below 0.25 abstain but keep their score |
 | `funlangid` | yes | Fraction of the text's character 4-grams covered by the winning language's lexicon. Does not sum to 1 across languages |
 | `GlotLID`, `OpenLID-v2`, `fasttext` | no | The batched `multilinePredict` these wrappers use returns bare labels under the pinned `fasttext-predict` dependency; see the note in `src/commonlid/models/_fasttext_base.py` |
 | `cld2` | no | CLD2 returns `percent` (share of the input in that language) and an unbounded internal `score` in the hundreds. Neither is a confidence |

pyproject.toml

@@ -69,6 +69,7 @@ commonlingua = [
     # transformers stack that [afrolid] pulls in.
     "torch>=2.4",
 ]
+py3langid = ["py3langid>=0.4"]
 leaderboard = [
     # gradio 4.x imports HfFolder from huggingface_hub, which was removed in
     # huggingface-hub 1.0; gradio 5 dropped that import.
@@ -82,7 +83,7 @@ notebooks = [
 ]
 all = [
     # Every optional extra: models, the leaderboard app and the notebooks.
-    "commonlid[afrolid,cld3,commonlingua,google-translate,leaderboard,llm,notebooks]",
+    "commonlid[afrolid,cld3,commonlingua,google-translate,leaderboard,llm,notebooks,py3langid]",
 ]
 
 # PEP 735 dependency groups: dev-only, never published in the wheel metadata.
@@ -93,7 +94,7 @@ dev = [
     # in the published extras instead of re-listing their pins. afrolid and
     # commonlingua (torch) stay out: they are covered by mocked unit tests
     # and excluded from coverage, see [tool.coverage.run].
-    "commonlid[llm,cld3,google-translate,leaderboard]",
+    "commonlid[llm,cld3,google-translate,leaderboard,py3langid]",
     "ruff>=0.6",
     "mypy>=1.11",
     "pytest>=8",
@@ -182,6 +183,7 @@ module = [
     "fasttext.*",
     "pycld2.*",
     "pyfranc.*",
+    "py3langid.*",
     "iso639.*",
     "datasets.*",
     "huggingface_hub.*",

src/commonlid/models/__init__.py

@@ -18,6 +18,7 @@
 from commonlid.models import google_translate_v2 as _google_translate_v2  # noqa: F401
 from commonlid.models import google_translate_v3 as _google_translate_v3  # noqa: F401
 from commonlid.models import openlidv2 as _openlidv2  # noqa: F401
+from commonlid.models import py3langid as _py3langid  # noqa: F401
 from commonlid.models import pyfranc as _pyfranc  # noqa: F401
 from commonlid.models.dspy_llm import DSPyLLMModel
 

src/commonlid/models/py3langid.py

@@ -0,0 +1,51 @@
+"""py3langid wrapper: does its own normalization; zxx and sub-threshold predictions abstain."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from commonlid.core.lid_model import LIDModel, LIDPrediction
+from commonlid.core.registry import register_model
+
+# codes the generic 639-1 -> 639-3 conformation would not reach
+_OUT_MAP = {"ar": "arb", "sw": "swh", "az": "azj", "or": "ory", "bcl": "bik"}
+# Tuned on commonlid, so treat the scores as optimistic. Measured effect:
+# no threshold 60.55 macro / 86.46 micro F1, 0.25 gives 64.81 / 86.85.
+_MIN_CONFIDENCE = 0.25
+
+
+@register_model
+class Py3LangIDModel(LIDModel):
+    model_id = "py3langid"
+    requires_preprocessing = False
+
+    def load(self) -> None:
+        try:
+            from py3langid.langid import MODEL_FILE, LanguageIdentifier
+        except ImportError as exc:
+            msg = "py3langid is not installed. Install with: pip install 'commonlid[py3langid]'"
+            raise ImportError(msg) from exc
+
+        self._identifier = LanguageIdentifier.from_model_file(
+            MODEL_FILE, norm_probs=True, min_confidence=_MIN_CONFIDENCE
+        )
+        super().load()
+
+    def _predict_batch(self, texts: Sequence[str]) -> list[LIDPrediction]:
+        out: list[LIDPrediction] = []
+        for text in texts:
+            code, score = self._identifier.classify(text)
+            label = None if code in ("zxx", "und") else _OUT_MAP.get(code, code)
+            out.append(LIDPrediction(label, float(score)))
+        return out
+
+    def discover_supported_languages(self) -> frozenset[str]:
+        self.load()
+        codes: set[str] = set()
+        for code in self._identifier.labels:
+            if code == "zxx":
+                continue
+            conformed = self._conform(_OUT_MAP.get(code, code))
+            if conformed is not None:
+                codes.add(conformed)
+        return frozenset(codes)

tests/models/test_model_registration.py

@@ -25,6 +25,7 @@ def _import_models() -> None:
     "commonlingua",
     "fasttext",
     "funlangid",
+    "py3langid",
     "pyfranc",
 }
 

tests/models/test_py3langid.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("py3langid")
+
+from commonlid.models.py3langid import _MIN_CONFIDENCE, _OUT_MAP, Py3LangIDModel
+
+
+def test_returns_conformed_codes() -> None:
+    m = Py3LangIDModel()
+    preds = m.predict([
+        "The quick brown fox jumps over the lazy dog.",
+        "Le renard brun saute par-dessus le chien paresseux.",
+        "Der schnelle braune Fuchs springt über den faulen Hund.",
+    ])
+    assert preds == ["eng", "fra", "deu"]
+
+
+def test_predict_scored_returns_the_posterior() -> None:
+    m = Py3LangIDModel()
+    scored = m.predict_scored(["The quick brown fox jumps over the lazy dog.", "123 456"])
+    assert scored[0].iso639_3 == "eng"
+    assert scored[0].score is not None
+    assert _MIN_CONFIDENCE < scored[0].score <= 1.0
+    assert scored[1].iso639_3 is None
+    assert scored[1].score is not None
+    assert 0.0 <= scored[1].score <= 1.0
+
+
+def test_abstains_on_empty_and_gibberish() -> None:
+    m = Py3LangIDModel()
+    assert m.predict(["", "   ", "123 456", "!!!"]) == [None, None, None, None]
+
+
+def test_out_map_overrides_default_conformation() -> None:
+    m = Py3LangIDModel()
+    assert _OUT_MAP["ar"] == "arb"
+    assert m.predict(["هذا نص قصير مكتوب باللغة العربية الفصحى الحديثة."]) == ["arb"]
+    bikol = "An mga tawo dapat magkaminootan asin magtinabangan sa lambang aldaw."
+    assert m.predict([bikol]) == ["bik"]
+
+
+def test_discover_supported_languages() -> None:
+    m = Py3LangIDModel()
+    langs = m.discover_supported_languages()
+    assert {"eng", "fra", "deu", "arz", "vec"} <= langs
+    assert "zxx" not in langs
+    assert "bik" in langs
+    assert not {"bcl", "ar", "und"} & langs
+
+
+def test_min_confidence_is_in_effect() -> None:
+    m = Py3LangIDModel()
+    m.load()
+    assert m._identifier.min_confidence == _MIN_CONFIDENCE