From 918dcf071abed9bc2ef9b84c2a55efa40d98d371 Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:59:42 +0200 Subject: [PATCH 1/6] Made wheels contain platform and python ABI tags --- .vscode/settings.json | 1 + jecq/python/setup.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 45c09f6..30798ca 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,7 @@ "addlicense", "atleast", "BLAS", + "cmdclass", "CMPLR", "DBLA", "DBUILD", diff --git a/jecq/python/setup.py b/jecq/python/setup.py index f813d3a..ddadee2 100644 --- a/jecq/python/setup.py +++ b/jecq/python/setup.py @@ -96,13 +96,26 @@ long_description = """ Jecq is a library for efficient similarity search based on the Faiss library. """ + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + + """Custom bdist_wheel to ensure that the root is not pure Python.""" + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False +except ImportError: + bdist_wheel = None + setup( name="jecq", version="0.0.1", description="A Faiss-based library for efficient similarity search " "and clustering of dense vectors", long_description=long_description, - license="TODO", + url="https://github.com/JaneaSystems/jecq", + license="MIT", keywords="search nearest neighbors", install_requires=["numpy", "packaging", "faiss-cpu"], packages=["jecq"], @@ -110,4 +123,5 @@ "jecq": ["*.so", "*.pyd"], }, zip_safe=False, + cmdclass={'bdist_wheel': bdist_wheel}, ) From 4fb283e2c35f4d161003a39b746ec94d4c01aca1 Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Tue, 1 Jul 2025 18:13:03 +0200 Subject: [PATCH 2/6] Fixed mypy issue --- jecq/python/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jecq/python/setup.py b/jecq/python/setup.py index ddadee2..c66c269 100644 --- a/jecq/python/setup.py +++ b/jecq/python/setup.py @@ -101,12 +101,14 @@ from wheel.bdist_wheel import bdist_wheel as _bdist_wheel """Custom bdist_wheel to ensure that the root is not pure Python.""" + class bdist_wheel(_bdist_wheel): def finalize_options(self): _bdist_wheel.finalize_options(self) self.root_is_pure = False + except ImportError: - bdist_wheel = None + print("Not using custom bdist_wheel; wheel package not installed") setup( name="jecq", @@ -123,5 +125,5 @@ def finalize_options(self): "jecq": ["*.so", "*.pyd"], }, zip_safe=False, - cmdclass={'bdist_wheel': bdist_wheel}, + cmdclass={"bdist_wheel": bdist_wheel}, ) From fc69c94802390d5eb0515a30651d7fd45e4d66bc Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:53:16 +0200 Subject: [PATCH 3/6] Changed setuptools requirement; ensured bdist_wheel used on Windows; improved Windows build script --- .vscode/settings.json | 1 + build.ps1 | 9 ++++++--- jecq/python/requirements.txt | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 30798ca..76e47d9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "cSpell.words": [ "addlicense", "atleast", + "bdist", "BLAS", "cmdclass", "CMPLR", diff --git a/build.ps1 b/build.ps1 index fbf24c5..52dd738 100644 --- a/build.ps1 +++ b/build.ps1 @@ -65,6 +65,7 @@ Write-Output "Setting up virtual environment..." Push-Location . python -m venv .venv; Test-Exit-Code ./.venv/scripts/activate.ps1; Test-Exit-Code +python -m pip install --upgrade pip; Test-Exit-Code pip install -r ../jecq/python/requirements.txt; Test-Exit-Code Pop-Location @@ -149,12 +150,14 @@ Test-Exit-Code # Install Python package Write-Output "Installing Python package..." Push-Location jecq/python -python setup.py install; Test-Exit-Code pip install wheel; Test-Exit-Code -pip wheel --no-binary jecq .; Test-Exit-Code +python setup.py bdist_wheel; Test-Exit-Code +Get-ChildItem dist/jecq*.whl | ForEach-Object { pip install $_.FullName } +Pop-Location + +# Verify installation python -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()"; Test-Exit-Code python -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'"; Test-Exit-Code -Pop-Location # Update wheels Write-Output "Updating wheels..." diff --git a/jecq/python/requirements.txt b/jecq/python/requirements.txt index 4ea25b7..06e7d03 100644 --- a/jecq/python/requirements.txt +++ b/jecq/python/requirements.txt @@ -1,5 +1,5 @@ faiss-cpu numpy>=2 -setuptools +setuptools>=80 packaging swig \ No newline at end of file From bc02ae231b9e300304e95d6f7b5968355e9ad024 Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:53:59 +0200 Subject: [PATCH 4/6] Used 3.13-slim python image for DockerFile.linux --- Dockerfile.linux | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.linux b/Dockerfile.linux index b3e35e9..21c0eb8 100644 --- a/Dockerfile.linux +++ b/Dockerfile.linux @@ -1,4 +1,5 @@ FROM ubuntu:22.04 +FROM python:3.13-slim ARG BUILD_TYPE=Debug From 374745ebe704e44fcb9dab2adc528a9cc96186af Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Tue, 1 Jul 2025 23:38:14 +0200 Subject: [PATCH 5/6] Ensured Python 3.13 before running demo --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b72fe36..5b697b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,6 +123,11 @@ jobs: mkdir -p wheels/linux docker cp jecq-wheel-builder:/app/wheels/linux/. wheels/linux/ docker rm jecq-wheel-builder + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' - name: Run sample demo in clean environment run: | From ced2092807413bf4364a35a666430ab796d0cc92 Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Wed, 2 Jul 2025 01:22:38 +0200 Subject: [PATCH 6/6] Ensured linux wheels are manylinux wheels --- .vscode/settings.json | 1 + build.sh | 18 +++++++++++++---- jecq/python/setup.py | 47 ++++++++++++++++++++++++++++++++++--------- requirements.linux | 3 ++- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 76e47d9..36ce8dd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "cSpell.words": [ "addlicense", "atleast", + "auditwheel", "bdist", "BLAS", "cmdclass", diff --git a/build.sh b/build.sh index 18c9095..81cd727 100644 --- a/build.sh +++ b/build.sh @@ -46,6 +46,7 @@ pushd . echo "Setting up virtual environment..." python3 -m venv .venv source .venv/bin/activate +python -m pip install --upgrade pip pip install -r ../jecq/python/requirements.txt popd @@ -87,13 +88,22 @@ fi echo "Installing Python package..." pushd . cd ./jecq/python -# Make sure wheel is installed pip install wheel -python3 setup.py install -pip wheel --no-binary jecq . +python3 setup.py bdist_wheel +# Repair wheel for Linux compatibility +pip install auditwheel +pushd ./dist +for f in jecq*.whl; do + auditwheel repair "$f" -w . && rm "$f" +done +popd +# Install wheels +pip install dist/jecq*.whl +popd + +# Verify installation python3 -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()" python3 -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'" -popd # Update wheels echo "Updating wheels..." diff --git a/jecq/python/setup.py b/jecq/python/setup.py index c66c269..9e36b55 100644 --- a/jecq/python/setup.py +++ b/jecq/python/setup.py @@ -23,7 +23,8 @@ import platform import shutil -from setuptools import setup +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext # make the jecq python package dir shutil.rmtree("jecq", ignore_errors=True) @@ -32,23 +33,22 @@ shutil.copyfile("loader.py", "jecq/loader.py") shutil.copyfile("class_wrappers.py", "jecq/class_wrappers.py") -ext = ".pyd" if platform.system() == "Windows" else ".so" +is_windows = platform.system() == "Windows" +ext = ".pyd" if is_windows else ".so" build_type = os.environ.get("JECQ_BUILD_TYPE", "RelWithDebInfo") -prefix = f"{build_type}/" * (platform.system() == "Windows") +prefix = f"{build_type}/" * is_windows swigjecq_generic_lib = f"{prefix}_swigjecq{ext}" swigjecq_avx2_lib = f"{prefix}_swigjecq_avx2{ext}" swigjecq_avx512_lib = f"{prefix}_swigjecq_avx512{ext}" swigjecq_avx512_spr_lib = f"{prefix}_swigjecq_avx512_spr{ext}" -callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}" swigjecq_sve_lib = f"{prefix}_swigjecq_sve{ext}" found_swigjecq_generic = os.path.exists(swigjecq_generic_lib) found_swigjecq_avx2 = os.path.exists(swigjecq_avx2_lib) found_swigjecq_avx512 = os.path.exists(swigjecq_avx512_lib) found_swigjecq_avx512_spr = os.path.exists(swigjecq_avx512_spr_lib) -found_callbacks = os.path.exists(callbacks_lib) found_swigjecq_sve = os.path.exists(swigjecq_sve_lib) assert ( @@ -64,34 +64,60 @@ f"Jecq may not be compiled yet." ) +libs = [] + if found_swigjecq_generic: print(f"Copying {swigjecq_generic_lib}") shutil.copyfile("swigjecq.py", "jecq/swigjecq.py") shutil.copyfile(swigjecq_generic_lib, f"jecq/_swigjecq{ext}") + libs.append("_swigjecq") if found_swigjecq_avx2: print(f"Copying {swigjecq_avx2_lib}") shutil.copyfile("swigjecq_avx2.py", "jecq/swigjecq_avx2.py") shutil.copyfile(swigjecq_avx2_lib, f"jecq/_swigjecq_avx2{ext}") + libs.append("_swigjecq_avx2") if found_swigjecq_avx512: print(f"Copying {swigjecq_avx512_lib}") shutil.copyfile("swigjecq_avx512.py", "jecq/swigjecq_avx512.py") shutil.copyfile(swigjecq_avx512_lib, f"jecq/_swigjecq_avx512{ext}") + libs.append("_swigjecq_avx512") + if found_swigjecq_avx512_spr: print(f"Copying {swigjecq_avx512_spr_lib}") shutil.copyfile("swigjecq_avx512_spr.py", "jecq/swigjecq_avx512_spr.py") shutil.copyfile(swigjecq_avx512_spr_lib, f"jecq/_swigjecq_avx512_spr{ext}") + libs.append("_swigjecq_avx512_spr") -if found_callbacks: - print(f"Copying {callbacks_lib}") - shutil.copyfile(callbacks_lib, f"jecq/{callbacks_lib}") if found_swigjecq_sve: print(f"Copying {swigjecq_sve_lib}") shutil.copyfile("swigjecq_sve.py", "jecq/swigjecq_sve.py") shutil.copyfile(swigjecq_sve_lib, f"jecq/_swigjecq_sve{ext}") + libs.append("_swigjecq_sve") + +ext_modules = [Extension(f"jecq.{mod}", sources=[]) for mod in libs] + + +# 2) CopyBuildExt just copies the .so into the build directory without compiling +class CopyBuildExt(_build_ext): + def run(self): + # skip the normal compiler run + for extension in self.extensions: + self.build_extension(extension) + + def build_extension(self, extension): + name = extension.name.split(".", 1)[1] # e.g. "_swigjecq" + src = f"{prefix}{name}{ext}" + if not os.path.exists(src): + raise FileNotFoundError(f"Binary output '{src}' not found") + dst = self.get_ext_fullpath(extension.name) + self.mkpath(os.path.dirname(dst)) + shutil.copyfile(src, dst) + print(f"Copied {src} to {dst}") + long_description = """ Jecq is a library for efficient similarity search based on the Faiss library. @@ -122,8 +148,9 @@ def finalize_options(self): install_requires=["numpy", "packaging", "faiss-cpu"], packages=["jecq"], package_data={ - "jecq": ["*.so", "*.pyd"], + "jecq": ["*.pyd"] if is_windows else [], }, zip_safe=False, - cmdclass={"bdist_wheel": bdist_wheel}, + ext_modules=[] if is_windows else ext_modules, + cmdclass={"bdist_wheel": bdist_wheel, "build_ext": CopyBuildExt}, ) diff --git a/requirements.linux b/requirements.linux index 7dbcf24..a35f7f0 100644 --- a/requirements.linux +++ b/requirements.linux @@ -7,4 +7,5 @@ cmake build-essential autoconf automake -libtool \ No newline at end of file +libtool +patchelf \ No newline at end of file