diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b72fe36..5b697b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,6 +123,11 @@ jobs: mkdir -p wheels/linux docker cp jecq-wheel-builder:/app/wheels/linux/. wheels/linux/ docker rm jecq-wheel-builder + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' - name: Run sample demo in clean environment run: | diff --git a/.vscode/settings.json b/.vscode/settings.json index 45c09f6..36ce8dd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,10 @@ "cSpell.words": [ "addlicense", "atleast", + "auditwheel", + "bdist", "BLAS", + "cmdclass", "CMPLR", "DBLA", "DBUILD", diff --git a/Dockerfile.linux b/Dockerfile.linux index b3e35e9..21c0eb8 100644 --- a/Dockerfile.linux +++ b/Dockerfile.linux @@ -1,4 +1,5 @@ FROM ubuntu:22.04 +FROM python:3.13-slim ARG BUILD_TYPE=Debug diff --git a/build.ps1 b/build.ps1 index fbf24c5..52dd738 100644 --- a/build.ps1 +++ b/build.ps1 @@ -65,6 +65,7 @@ Write-Output "Setting up virtual environment..." Push-Location . python -m venv .venv; Test-Exit-Code ./.venv/scripts/activate.ps1; Test-Exit-Code +python -m pip install --upgrade pip; Test-Exit-Code pip install -r ../jecq/python/requirements.txt; Test-Exit-Code Pop-Location @@ -149,12 +150,14 @@ Test-Exit-Code # Install Python package Write-Output "Installing Python package..." Push-Location jecq/python -python setup.py install; Test-Exit-Code pip install wheel; Test-Exit-Code -pip wheel --no-binary jecq .; Test-Exit-Code +python setup.py bdist_wheel; Test-Exit-Code +Get-ChildItem dist/jecq*.whl | ForEach-Object { pip install $_.FullName } +Pop-Location + +# Verify installation python -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()"; Test-Exit-Code python -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'"; Test-Exit-Code -Pop-Location # Update wheels Write-Output "Updating wheels..." diff --git a/build.sh b/build.sh index 18c9095..81cd727 100644 --- a/build.sh +++ b/build.sh @@ -46,6 +46,7 @@ pushd . echo "Setting up virtual environment..." python3 -m venv .venv source .venv/bin/activate +python -m pip install --upgrade pip pip install -r ../jecq/python/requirements.txt popd @@ -87,13 +88,22 @@ fi echo "Installing Python package..." pushd . cd ./jecq/python -# Make sure wheel is installed pip install wheel -python3 setup.py install -pip wheel --no-binary jecq . +python3 setup.py bdist_wheel +# Repair wheel for Linux compatibility +pip install auditwheel +pushd ./dist +for f in jecq*.whl; do + auditwheel repair "$f" -w . && rm "$f" +done +popd +# Install wheels +pip install dist/jecq*.whl +popd + +# Verify installation python3 -c "import jecq;jecq.IndexJecq();jecq.IndexIVFJecq()" python3 -c "import jecq;assert jecq.IndexJecq.__module__ == 'jecq.swigjecq_avx2'" -popd # Update wheels echo "Updating wheels..." diff --git a/jecq/python/requirements.txt b/jecq/python/requirements.txt index 4ea25b7..06e7d03 100644 --- a/jecq/python/requirements.txt +++ b/jecq/python/requirements.txt @@ -1,5 +1,5 @@ faiss-cpu numpy>=2 -setuptools +setuptools>=80 packaging swig \ No newline at end of file diff --git a/jecq/python/setup.py b/jecq/python/setup.py index f813d3a..9e36b55 100644 --- a/jecq/python/setup.py +++ b/jecq/python/setup.py @@ -23,7 +23,8 @@ import platform import shutil -from setuptools import setup +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext # make the jecq python package dir shutil.rmtree("jecq", ignore_errors=True) @@ -32,23 +33,22 @@ shutil.copyfile("loader.py", "jecq/loader.py") shutil.copyfile("class_wrappers.py", "jecq/class_wrappers.py") -ext = ".pyd" if platform.system() == "Windows" else ".so" +is_windows = platform.system() == "Windows" +ext = ".pyd" if is_windows else ".so" build_type = os.environ.get("JECQ_BUILD_TYPE", "RelWithDebInfo") -prefix = f"{build_type}/" * (platform.system() == "Windows") +prefix = f"{build_type}/" * is_windows swigjecq_generic_lib = f"{prefix}_swigjecq{ext}" swigjecq_avx2_lib = f"{prefix}_swigjecq_avx2{ext}" swigjecq_avx512_lib = f"{prefix}_swigjecq_avx512{ext}" swigjecq_avx512_spr_lib = f"{prefix}_swigjecq_avx512_spr{ext}" -callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}" swigjecq_sve_lib = f"{prefix}_swigjecq_sve{ext}" found_swigjecq_generic = os.path.exists(swigjecq_generic_lib) found_swigjecq_avx2 = os.path.exists(swigjecq_avx2_lib) found_swigjecq_avx512 = os.path.exists(swigjecq_avx512_lib) found_swigjecq_avx512_spr = os.path.exists(swigjecq_avx512_spr_lib) -found_callbacks = os.path.exists(callbacks_lib) found_swigjecq_sve = os.path.exists(swigjecq_sve_lib) assert ( @@ -64,50 +64,93 @@ f"Jecq may not be compiled yet." ) +libs = [] + if found_swigjecq_generic: print(f"Copying {swigjecq_generic_lib}") shutil.copyfile("swigjecq.py", "jecq/swigjecq.py") shutil.copyfile(swigjecq_generic_lib, f"jecq/_swigjecq{ext}") + libs.append("_swigjecq") if found_swigjecq_avx2: print(f"Copying {swigjecq_avx2_lib}") shutil.copyfile("swigjecq_avx2.py", "jecq/swigjecq_avx2.py") shutil.copyfile(swigjecq_avx2_lib, f"jecq/_swigjecq_avx2{ext}") + libs.append("_swigjecq_avx2") if found_swigjecq_avx512: print(f"Copying {swigjecq_avx512_lib}") shutil.copyfile("swigjecq_avx512.py", "jecq/swigjecq_avx512.py") shutil.copyfile(swigjecq_avx512_lib, f"jecq/_swigjecq_avx512{ext}") + libs.append("_swigjecq_avx512") + if found_swigjecq_avx512_spr: print(f"Copying {swigjecq_avx512_spr_lib}") shutil.copyfile("swigjecq_avx512_spr.py", "jecq/swigjecq_avx512_spr.py") shutil.copyfile(swigjecq_avx512_spr_lib, f"jecq/_swigjecq_avx512_spr{ext}") + libs.append("_swigjecq_avx512_spr") -if found_callbacks: - print(f"Copying {callbacks_lib}") - shutil.copyfile(callbacks_lib, f"jecq/{callbacks_lib}") if found_swigjecq_sve: print(f"Copying {swigjecq_sve_lib}") shutil.copyfile("swigjecq_sve.py", "jecq/swigjecq_sve.py") shutil.copyfile(swigjecq_sve_lib, f"jecq/_swigjecq_sve{ext}") + libs.append("_swigjecq_sve") + +ext_modules = [Extension(f"jecq.{mod}", sources=[]) for mod in libs] + + +# 2) CopyBuildExt just copies the .so into the build directory without compiling +class CopyBuildExt(_build_ext): + def run(self): + # skip the normal compiler run + for extension in self.extensions: + self.build_extension(extension) + + def build_extension(self, extension): + name = extension.name.split(".", 1)[1] # e.g. "_swigjecq" + src = f"{prefix}{name}{ext}" + if not os.path.exists(src): + raise FileNotFoundError(f"Binary output '{src}' not found") + dst = self.get_ext_fullpath(extension.name) + self.mkpath(os.path.dirname(dst)) + shutil.copyfile(src, dst) + print(f"Copied {src} to {dst}") + long_description = """ Jecq is a library for efficient similarity search based on the Faiss library. """ + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + + """Custom bdist_wheel to ensure that the root is not pure Python.""" + + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False + +except ImportError: + print("Not using custom bdist_wheel; wheel package not installed") + setup( name="jecq", version="0.0.1", description="A Faiss-based library for efficient similarity search " "and clustering of dense vectors", long_description=long_description, - license="TODO", + url="https://github.com/JaneaSystems/jecq", + license="MIT", keywords="search nearest neighbors", install_requires=["numpy", "packaging", "faiss-cpu"], packages=["jecq"], package_data={ - "jecq": ["*.so", "*.pyd"], + "jecq": ["*.pyd"] if is_windows else [], }, zip_safe=False, + ext_modules=[] if is_windows else ext_modules, + cmdclass={"bdist_wheel": bdist_wheel, "build_ext": CopyBuildExt}, ) diff --git a/requirements.linux b/requirements.linux index 7dbcf24..a35f7f0 100644 --- a/requirements.linux +++ b/requirements.linux @@ -7,4 +7,5 @@ cmake build-essential autoconf automake -libtool \ No newline at end of file +libtool +patchelf \ No newline at end of file