From 38ab40562c8ca23978afe922e168dfe152ec1fa8 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Tue, 11 Aug 2026 21:43:40 +0900 Subject: [PATCH 1/5] Use daachorse v5.0.0 and support abi3-py310 --- .github/workflows/CI.yml | 56 +++++++++++++++++++++-------------- Cargo.toml | 6 ++-- pyproject.toml | 3 +- src/lib.rs | 64 +++++++++++++++++++++++++++++++++------- tests/test_bytewise.py | 1 + 5 files changed, 94 insertions(+), 36 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b4fd0a8..bd29266 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -14,27 +14,48 @@ permissions: contents: read jobs: - test: + build-test: strategy: matrix: runs-on: [ ubuntu-latest, windows-latest, macos-latest ] - python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: '3.13' - name: Build package uses: PyO3/maturin-action@v1 with: manylinux: auto - args: -i ${{ matrix.python-version }} --release + args: --release + - name: Upload wheels + uses: actions/upload-artifact@v7 + with: + name: test-wheels-${{ matrix.runs-on }} + path: target/wheels + + test: + needs: [ build-test ] + strategy: + matrix: + runs-on: [ ubuntu-latest, windows-latest, macos-latest ] + python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v8 + with: + name: test-wheels-${{ matrix.runs-on }} + path: test-wheels - name: Test package run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - pip install daachorse --no-index --find-links target/wheels --force-reinstall + pip install daachorse --no-index --find-links test-wheels --force-reinstall mypy --strict tests pytest @@ -63,23 +84,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] target: [ x86_64, i686 ] steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - name: Build package uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} manylinux: auto - args: -i ${{ matrix.python-version }} --release + args: --release - name: Upload wheels uses: actions/upload-artifact@v7 with: - name: wheels-linux-${{ matrix.python-version }}-${{ matrix.target }} + name: wheels-linux-${{ matrix.target }} path: target/wheels pack-windows: @@ -88,46 +105,41 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] target: [ x64, x86 ] steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: '3.13' architecture: ${{ matrix.target }} - name: Build package uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: -i ${{ matrix.python-version }} --release + args: --release - name: Upload wheels uses: actions/upload-artifact@v7 with: - name: wheels-windows-${{ matrix.python-version }}-${{ matrix.target }} + name: wheels-windows-${{ matrix.target }} path: target/wheels pack-macos: if: "startsWith(github.ref, 'refs/tags/')" needs: [ test ] runs-on: macos-latest - strategy: - matrix: - python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: '3.13' - name: Build package uses: PyO3/maturin-action@v1 with: - target: ${{ matrix.target }} - args: -i ${{ matrix.python-version }} --release --target universal2-apple-darwin + args: --release --target universal2-apple-darwin - name: Upload wheels uses: actions/upload-artifact@v7 with: - name: wheels-macos-${{ matrix.python-version }} + name: wheels-macos path: target/wheels release: diff --git a/Cargo.toml b/Cargo.toml index e3909d6..b149093 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "python-daachorse" -version = "0.3.0" +version = "0.5.0" edition = "2021" authors = [ "Koichi Akabe ", @@ -16,8 +16,8 @@ name = "daachorse" crate-type = ["cdylib"] [dependencies] -daachorse = "3.0.0" # MIT or Apache-2.0 -pyo3 = { version = "0.28.3", features = ["extension-module"] } # Apache-2.0 +daachorse = "5.0.0" # MIT or Apache-2.0 +pyo3 = { version = "0.28.3", features = ["extension-module", "abi3-py310"] } # Apache-2.0 [profile.release] lto = true diff --git a/pyproject.toml b/pyproject.toml index 8cefed2..60d0bb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,10 @@ build-backend = "maturin" [project] name = "daachorse" -version = "0.3.0" +version = "0.5.0" description = "🐎 A fast implementation of the Aho-Corasick algorithm using the compact double-array data structure" readme = "README.md" +requires-python = ">=3.10" classifiers = [ "License :: OSI Approved :: Apache Software License", "License :: OSI Approved :: MIT License", diff --git a/src/lib.rs b/src/lib.rs index 781cb61..228849a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,8 +15,12 @@ use pyo3::{ /// /// :param patterns: List of bytes patterns. /// :param match_kind: A search option of the Aho-Corasick automaton. +/// :param corpus: List of bytes to be used as a corpus for the profile-guided optimization. +/// :param use_prefilter: Whether to use a prefilter. /// :type patterns: list[bytes] /// :type match_kind: int +/// :type corpus: list[bytes] | None +/// :type use_prefilter: bool /// :rtype: daachorse.DoubleArrayAhoCorasick #[pyclass(module = "daachorse")] struct DoubleArrayAhoCorasick { @@ -26,17 +30,35 @@ struct DoubleArrayAhoCorasick { #[pymethods] impl DoubleArrayAhoCorasick { #[new] - #[pyo3(signature = (patterns, /, match_kind = 0))] - fn new(py: Python, patterns: Vec>, match_kind: u8) -> PyResult { - let raw_patterns: PyResult>> = - patterns.iter().map(|pat| pat.extract(py)).collect(); - let raw_patterns = raw_patterns?; + #[pyo3(signature = (patterns, /, match_kind = 0, corpus = None, use_prefilter = true))] + fn new( + py: Python, + patterns: Vec>, + match_kind: u8, + corpus: Option>>, + use_prefilter: bool, + ) -> PyResult { + let raw_patterns = patterns + .iter() + .map(|pat| pat.extract(py)) + .collect::>>>()?; let match_kind = ::daachorse::MatchKind::from(match_kind); + let raw_corpus = corpus.map_or_else( + || Ok(vec![]), + |corpus| { + corpus + .iter() + .map(|c| c.extract(py)) + .collect::>>>() + }, + )?; Ok(Self { pma: py .detach(|| { ::daachorse::DoubleArrayAhoCorasickBuilder::new() .match_kind(match_kind) + .corpus(raw_corpus) + .use_prefilter(use_prefilter) .build(raw_patterns) }) .map_err(|e| PyValueError::new_err(e.to_string()))?, @@ -249,8 +271,12 @@ impl DoubleArrayAhoCorasick { /// /// :param patterns: List of string patterns. /// :param match_kind: A search option of the Aho-Corasick automaton. +/// :param corpus: List of strings to be used as a corpus for the profile-guided optimization. +/// :param use_prefilter: Whether to use a prefilter. /// :type patterns: list[str] /// :type match_kind: int +/// :type corpus: list[str] | None +/// :type use_prefilter: bool /// :rtype: daachorse.CharwiseDoubleArrayAhoCorasick #[pyclass(module = "daachorse")] struct CharwiseDoubleArrayAhoCorasick { @@ -260,17 +286,35 @@ struct CharwiseDoubleArrayAhoCorasick { #[pymethods] impl CharwiseDoubleArrayAhoCorasick { #[new] - #[pyo3(signature = (patterns, /, match_kind = 0))] - fn new(py: Python, patterns: Vec>, match_kind: u8) -> PyResult { - let raw_patterns: PyResult> = - patterns.iter().map(|pat| pat.extract(py)).collect(); - let raw_patterns = raw_patterns?; + #[pyo3(signature = (patterns, /, match_kind = 0, corpus = None, use_prefilter = true))] + fn new( + py: Python, + patterns: Vec>, + match_kind: u8, + corpus: Option>>, + use_prefilter: bool, + ) -> PyResult { + let raw_patterns = patterns + .iter() + .map(|pat| pat.extract(py)) + .collect::>>()?; let match_kind = ::daachorse::MatchKind::from(match_kind); + let raw_corpus = corpus.map_or_else( + || Ok(vec![]), + |corpus| { + corpus + .iter() + .map(|c| c.extract(py)) + .collect::>>() + }, + )?; Ok(Self { pma: py .detach(|| { ::daachorse::CharwiseDoubleArrayAhoCorasickBuilder::new() .match_kind(match_kind) + .corpus(raw_corpus) + .use_prefilter(use_prefilter) .build(raw_patterns) }) .map_err(|e| PyValueError::new_err(e.to_string()))?, diff --git a/tests/test_bytewise.py b/tests/test_bytewise.py index abdaeb1..c259bbc 100644 --- a/tests/test_bytewise.py +++ b/tests/test_bytewise.py @@ -77,6 +77,7 @@ def test_find_overlapping_invalid_option() -> None: with pytest.raises(ValueError): pma.find_overlapping(haystack) + def test_serialization() -> None: haystack = b'this is a pen' patterns = [b't', b'hi', b'h', b'this', b'pe'] From e51845320baad837db1084189fa812b102f29831 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Tue, 11 Aug 2026 21:50:43 +0900 Subject: [PATCH 2/5] Update deps --- .github/workflows/CI.yml | 24 ++++++++++++------------ Cargo.toml | 2 +- requirements-dev.txt | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bd29266..daa65dd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,8 +20,8 @@ jobs: runs-on: [ ubuntu-latest, windows-latest, macos-latest ] runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.13' - name: Build package @@ -43,8 +43,8 @@ jobs: python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - uses: actions/download-artifact@v8 @@ -64,8 +64,8 @@ jobs: needs: [ test ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.13' - name: Build package @@ -86,7 +86,7 @@ jobs: matrix: target: [ x86_64, i686 ] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Build package uses: PyO3/maturin-action@v1 with: @@ -107,8 +107,8 @@ jobs: matrix: target: [ x64, x86 ] steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.13' architecture: ${{ matrix.target }} @@ -128,8 +128,8 @@ jobs: needs: [ test ] runs-on: macos-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.13' - name: Build package @@ -155,6 +155,6 @@ jobs: with: pattern: wheels-* - name: Install uv - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Publish to PyPI run: uv publish wheels-*/* diff --git a/Cargo.toml b/Cargo.toml index b149093..8346a51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ crate-type = ["cdylib"] [dependencies] daachorse = "5.0.0" # MIT or Apache-2.0 -pyo3 = { version = "0.28.3", features = ["extension-module", "abi3-py310"] } # Apache-2.0 +pyo3 = { version = "0.29.2", features = ["extension-module", "abi3-py310"] } # Apache-2.0 [profile.release] lto = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 5c7bef4..d062de7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -pytest>=7.2.1 -mypy>=0.991 +pytest>=9.1.1 +mypy>=2.3.0 From 2b4b0c9e912e3147cd34fafba19ccd11df7e2858 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Tue, 11 Aug 2026 22:04:14 +0900 Subject: [PATCH 3/5] add example --- docs/source/examples.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 7b38536..76f8db0 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -78,3 +78,19 @@ To build an automaton for strings, use ``CharwiseDoubleArrayAhoCorasick`` instea >>> pma = daachorse.CharwiseDoubleArrayAhoCorasick(patterns) >>> pma.find('ε…¨δΈ–η•ŒδΈ­γ«') [(0, 3, 0), (4, 5, 2)] + +Profile-Guided Optimization (PGO) +--------------------------------- + +The double array constructed by daachorse is designed to improve CPU cache efficiency. However, it +is not always the optimal layout for every document being scanned. + +daachorse addresses this issue by supporting Profile-Guided Optimization (PGO). By providing a +corpus when constructing the automaton, the memory layout is optimized, which may improve +performance on documents similar to the given corpus. + +.. code-block:: python + + >>> import daachorse + >>> patterns = [b'ab', b'a', b'abcd'] + >>> pma = daachorse.DoubleArrayAhoCorasick(patterns, corpus=[b'abcd']) From b5a178a8ad0f5a4e0974edbd4207c7ed8bb262e9 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Tue, 11 Aug 2026 22:10:36 +0900 Subject: [PATCH 4/5] add tests --- daachorse.pyi | 4 ++-- tests/test_bytewise.py | 25 +++++++++++++++++++++++++ tests/test_charwise.py | 25 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/daachorse.pyi b/daachorse.pyi index c4a1f82..03095c6 100644 --- a/daachorse.pyi +++ b/daachorse.pyi @@ -3,7 +3,7 @@ MATCH_KIND_LEFTMOST_LONGEST: int MATCH_KIND_STANDARD: int class DoubleArrayAhoCorasick: - def __init__(self, patterns: list[bytes], match_kind: int = 0) -> None: ... + def __init__(self, patterns: list[bytes], /, match_kind: int = 0, corpus: list[bytes] | None = None, use_prefilter: bool = True) -> None: ... def find(self, haystack: bytes) -> list[tuple[int, int, int]]: ... def find_overlapping(self, haystack: bytes) -> list[tuple[int, int, int]]: ... def find_overlapping_no_suffix(self, haystack: bytes) -> list[tuple[int, int, int]]: ... @@ -12,7 +12,7 @@ class DoubleArrayAhoCorasick: def deserialize(data: bytes) -> "DoubleArrayAhoCorasick": ... class CharwiseDoubleArrayAhoCorasick: - def __init__(self, patterns: list[str], match_kind: int = 0) -> None: ... + def __init__(self, patterns: list[str], /, match_kind: int = 0, corpus: list[str] | None = None, use_prefilter: bool = True) -> None: ... def find(self, haystack: str) -> list[tuple[int, int, int]]: ... def find_overlapping(self, haystack: str) -> list[tuple[int, int, int]]: ... def find_overlapping_no_suffix(self, haystack: str) -> list[tuple[int, int, int]]: ... diff --git a/tests/test_bytewise.py b/tests/test_bytewise.py index c259bbc..fdac8bb 100644 --- a/tests/test_bytewise.py +++ b/tests/test_bytewise.py @@ -78,6 +78,31 @@ def test_find_overlapping_invalid_option() -> None: pma.find_overlapping(haystack) +def test_corpus() -> None: + haystack = b'this is a pen' + patterns = [b't', b'hi', b'h', b'this', b'pe'] + corpus = [b'this is a pen', b'that is a pencil'] + pma = daachorse.DoubleArrayAhoCorasick(patterns, corpus=corpus) + + assert [ + (0, 1, 0), + (1, 2, 2), + (10, 12, 4), + ] == pma.find(haystack) + + +def test_use_prefilter_disabled() -> None: + haystack = b'this is a pen' + patterns = [b't', b'hi', b'h', b'this', b'pe'] + pma = daachorse.DoubleArrayAhoCorasick(patterns, use_prefilter=False) + + assert [ + (0, 1, 0), + (1, 2, 2), + (10, 12, 4), + ] == pma.find(haystack) + + def test_serialization() -> None: haystack = b'this is a pen' patterns = [b't', b'hi', b'h', b'this', b'pe'] diff --git a/tests/test_charwise.py b/tests/test_charwise.py index 83dde94..49081c7 100644 --- a/tests/test_charwise.py +++ b/tests/test_charwise.py @@ -78,6 +78,31 @@ def test_find_overlapping_invalid_option() -> None: pma.find_overlapping(haystack) +def test_corpus() -> None: + haystack = 'this is a γƒ†γ‚Ήγƒˆ' + patterns = ['t', 'hi', 'h', 'this', 'テス'] + corpus = ['this is a γƒ†γ‚Ήγƒˆ', 'that is a γƒ†γ‚­γ‚Ήγƒˆ'] + pma = daachorse.CharwiseDoubleArrayAhoCorasick(patterns, corpus=corpus) + + assert [ + (0, 1, 0), + (1, 2, 2), + (10, 12, 4), + ] == pma.find(haystack) + + +def test_use_prefilter_disabled() -> None: + haystack = 'this is a γƒ†γ‚Ήγƒˆ' + patterns = ['t', 'hi', 'h', 'this', 'テス'] + pma = daachorse.CharwiseDoubleArrayAhoCorasick(patterns, use_prefilter=False) + + assert [ + (0, 1, 0), + (1, 2, 2), + (10, 12, 4), + ] == pma.find(haystack) + + def test_serialization() -> None: haystack = 'this is a γƒ†γ‚Ήγƒˆ' patterns = ['t', 'hi', 'h', 'this', 'テス'] From ab753b57e805b61d38e06d2afa744d35821d5f55 Mon Sep 17 00:00:00 2001 From: Koichi Akabe Date: Tue, 11 Aug 2026 22:19:21 +0900 Subject: [PATCH 5/5] update README --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 42bff99..eefdafd 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,21 @@ To build an automaton for strings, use `CharwiseDoubleArrayAhoCorasick` instead. [(0, 3, 0), (4, 5, 2)] ``` +### Profile-Guided Optimization (PGO) + +The double array constructed by daachorse is designed to improve CPU cache efficiency. However, it +is not always the optimal layout for every document being scanned. + +daachorse addresses this issue by supporting Profile-Guided Optimization (PGO). By providing a +corpus when constructing the automaton, the memory layout is optimized, which may improve +performance on documents similar to the given corpus. + +```python +>>> import daachorse +>>> patterns = [b'ab', b'a', b'abcd'] +>>> pma = daachorse.DoubleArrayAhoCorasick(patterns, corpus=[b'abcd']) +``` + ## License Licensed under either of