release-python.yml

name: Release Python

on:
  workflow_dispatch:
    inputs:
      # Latest commit to include with the release. If omitted, use the latest commit on the main branch.
      sha:
        description: Commit SHA
        type: string
      # Create the sdist and build the wheels, but do not publish to PyPI / GitHub.
      dry-run:
        description: Dry run
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  PYTHON_VERSION: '3.9'
  PYTHON_VERSION_WIN_ARM64: '3.11' # ARM64 Windows doesn't have older versions
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  RUSTUP_MAX_RETRIES: 10

defaults:
  run:
    shell: bash

jobs:
  create-sdist:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        package: [polars, polars-lts-cpu, polars-u64-idx]

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.sha }}

      # Avoid potential out-of-memory errors
      - name: Set swap space for Linux
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install yq
        if: matrix.package != 'polars'
        run: pip install yq
      - name: Update package name
        if: matrix.package != 'polars'
        run: tomlq -i -t ".project.name = \"${{ matrix.package }}\"" py-polars/pyproject.toml
      - name: Add bigidx feature
        if: matrix.package == 'polars-u64-idx'
        run: tomlq -i -t '.dependencies.polars.features += ["bigidx"]' py-polars/Cargo.toml
      - name: Update optional dependencies
        if: matrix.package != 'polars'
        run: sed -i 's/polars\[/${{ matrix.package }}\[/' py-polars/pyproject.toml

      - name: Create source distribution
        uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: >
            --manifest-path py-polars/Cargo.toml
            --out dist
          maturin-version: 1.8.3

      - name: Test sdist
        run: |
          pip install --force-reinstall --verbose dist/*.tar.gz
          python -c 'import polars'

      - name: Upload sdist
        uses: actions/upload-artifact@v4
        with:
          name: sdist-${{ matrix.package }}
          path: dist/*.tar.gz

  build-wheels:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        package: [polars, polars-lts-cpu, polars-u64-idx]
        # macos-13 is x86-64
        # macos-15 is aarch64
        os: [ubuntu-latest, macos-13, macos-15, windows-latest, windows-11-arm]
        architecture: [x86-64, aarch64]
        exclude:
          - os: windows-latest
            architecture: aarch64
          - os: windows-11-arm
            architecture: x86-64
          - os: macos-15
            architecture: x86-64
          - os: macos-13
            architecture: aarch64

    env:
      SED_INPLACE: ${{ startsWith(matrix.os, 'macos') && '-i ''''' || '-i' }}
      CPU_CHECK_MODULE: py-polars/polars/_cpu_check.py

    steps:
      - name: Setup build environment (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        shell:
          powershell
          # Notes
          # * We update `Expand-Archive` to avoid "" is not a supported archive file format when extracting
          #   files that don't end in `.zip`
        run: |
          Write-Output "> Update Expand-Archive (Microsoft.PowerShell.Archive)"
          Install-PackageProvider -Name NuGet -Force
          Install-Module -Name Microsoft.PowerShell.Archive -Force

          Write-Output "> Setup bash.exe (git-for-windows/PortableGit)"
          Invoke-WebRequest "https://github.com/git-for-windows/git/releases/download/v2.47.1.windows.1/PortableGit-2.47.1-arm64.7z.exe" -OutFile /git.7z.exe
          /git.7z.exe -o/git -y

          Write-Output "> Setup Rust"
          Invoke-WebRequest "https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe" -OutFile /rustup-init.exe
          /rustup-init.exe --default-host aarch64-pc-windows-msvc -y

          Write-Output "> Setup VS Build Tools"
          Invoke-WebRequest "https://aka.ms/vs/17/release/vs_BuildTools.exe" -OutFile /vs_BuildTools.exe
          Start-Process C:/vs_BuildTools.exe -ArgumentList " `
              --add Microsoft.VisualStudio.Workload.NativeDesktop `
              --add Microsoft.VisualStudio.Workload.VCTools `
              --add Microsoft.VisualStudio.Component.VC.Tools.arm64 `
              --add Microsoft.VisualStudio.Component.VC.Llvm.Clang `
              --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset `
              --includeRecommended --quiet --norestart --wait" -Wait

          Write-Output "> Setup CMake"
          Invoke-WebRequest "https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-3.31.2-windows-arm64.zip" -OutFile /cmake.zip
          Expand-Archive /cmake.zip -DestinationPath /

          Write-Output "> Download jq.exe (github.com/jqlang) (needed for tomlq / yq)"
          Invoke-WebRequest https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-windows-i386.exe -OutFile /jq.exe

          Write-Output "> Update GITHUB_PATH"
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/git/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + $Env:USERPROFILE + "/.cargo/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/cmake-3.31.2-windows-arm64/bin")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n")

          Get-Content $Env:GITHUB_PATH | Out-Host

      - name: Check build environment (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        run: |
          set -x
          bash --version
          rustup show
          clang -v
          cmake --version

      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.sha }}

      # Avoid potential out-of-memory errors
      - name: Set swap space for Linux
        if: matrix.os == 'ubuntu-latest'
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Set up Python
        if: matrix.os != 'windows-11-arm'
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Set up Python (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION_WIN_ARM64 }}

      # Otherwise can't find `tomlq` after `pip install yq`
      - name: Add Python scripts folder to GITHUB_PATH (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        run: |
          python -c "import sysconfig; print(sysconfig.get_path('scripts'))" >> $GITHUB_PATH

      - name: Install yq
        if: matrix.package != 'polars'
        run: pip install yq

      - name: Update package name
        if: matrix.package != 'polars'
        run: tomlq -i -t ".project.name = \"${{ matrix.package }}\"" py-polars/pyproject.toml

      - name: Add bigidx feature
        if: matrix.package == 'polars-u64-idx'
        run: tomlq -i -t '.dependencies.polars.features += ["bigidx"]' py-polars/Cargo.toml

      - name: Update optional dependencies
        if: matrix.package != 'polars'
        run: sed $SED_INPLACE 's/polars\[/${{ matrix.package }}\[/' py-polars/pyproject.toml

      - name: Determine CPU features for x86-64
        id: features
        if: matrix.architecture == 'x86-64'
        env:
          IS_LTS_CPU: ${{ matrix.package == 'polars-lts-cpu' }}
        # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py
        run: |
          if [[ "$IS_LTS_CPU" = true ]]; then
            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b
            CC_FEATURES="-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16"
          else
            TUNE_CPU=skylake
            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe
            CC_FEATURES="-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe"
          fi
          echo "features=$FEATURES" >> $GITHUB_OUTPUT
          echo "tune_cpu=$TUNE_CPU" >> $GITHUB_OUTPUT
          echo "cc_features=$CC_FEATURES" >> $GITHUB_OUTPUT

      - name: Set RUSTFLAGS for x86-64
        if: matrix.architecture == 'x86-64'
        env:
          FEATURES: ${{ steps.features.outputs.features }}
          TUNE_CPU: ${{ steps.features.outputs.tune_cpu }}
          CC_FEATURES: ${{ steps.features.outputs.cc_features }}
          CFG: ${{ matrix.package == 'polars-lts-cpu' && '--cfg allocator="default"' || '' }}
        run: |
          if [[ -z "$TUNE_CPU" ]]; then
            echo "RUSTFLAGS=-C target-feature=$FEATURES $CFG" >> $GITHUB_ENV
            echo "CFLAGS=$CC_FEATURES" >> $GITHUB_ENV
          else
            echo "RUSTFLAGS=-C target-feature=$FEATURES -Z tune-cpu=$TUNE_CPU $CFG" >> $GITHUB_ENV
            echo "CFLAGS=$CC_FEATURES -mtune=$TUNE_CPU" >> $GITHUB_ENV
          fi

      - name: Set variables in CPU check module
        run: |
          sed $SED_INPLACE 's/^_POLARS_ARCH = \"unknown\"$/_POLARS_ARCH = \"${{ matrix.architecture }}\"/g' $CPU_CHECK_MODULE
          sed $SED_INPLACE 's/^_POLARS_FEATURE_FLAGS = \"\"$/_POLARS_FEATURE_FLAGS = \"${{ steps.features.outputs.features }}\"/g' $CPU_CHECK_MODULE
      - name: Set variables in CPU check module - LTS_CPU
        if: matrix.package == 'polars-lts-cpu'
        run: |
          sed $SED_INPLACE 's/^_POLARS_LTS_CPU = False$/_POLARS_LTS_CPU = True/g' $CPU_CHECK_MODULE

      - name: Set Rust target for aarch64
        if: matrix.architecture == 'aarch64'
        id: target
        run: |
          TARGET=$(
            if [[ "${{ matrix.os }}" == "macos-15" ]]; then
              echo "aarch64-apple-darwin";
            elif [[ "${{ matrix.os }}" == "windows-11-arm" ]]; then
              echo "aarch64-pc-windows-msvc";
            else
              echo "aarch64-unknown-linux-gnu";
            fi
          )
          echo "target=$TARGET" >> $GITHUB_OUTPUT

      - name: Set jemalloc for aarch64 Linux
        if: matrix.architecture == 'aarch64' && matrix.os == 'ubuntu-latest'
        run: |
          echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV

      - name: Copy toolchain to py-polars/ (ARM64 Windows)
        # Manual fix for:
        #   TomlError: Unknown character "46" at row 1, col 2, pos 1:
        #   1> ../rust-toolchain.toml
        if: matrix.os == 'windows-11-arm'
        run: cp rust-toolchain.toml py-polars/

      - name: Build wheel
        uses: PyO3/maturin-action@v1
        with:
          command: build
          target: ${{ steps.target.outputs.target }}
          args: >
            --profile dist-release
            --manifest-path py-polars/Cargo.toml
            --out dist
          manylinux: ${{ matrix.architecture == 'aarch64' && '2_24' || 'auto' }}
          maturin-version: 1.8.3

      - name: Test wheel
        # For linux; only test on x86-64 for now as this matches the runner architecture
        if: matrix.architecture == 'x86-64' || startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'windows')
        run: |
          pip install --force-reinstall --verbose dist/*.whl
          python -c 'import polars; polars.show_versions()'

      - name: Upload wheel
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ matrix.package }}-${{ matrix.os }}-${{ matrix.architecture }}
          path: dist/*.whl

  build-wheel-pyodide:
    name: build-wheels (polars, pyodide, wasm32)
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.sha }}

      # Avoid potential out-of-memory errors
      - name: Set swap space for Linux
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Disable incompatible features
        env:
            FEATURES: csv|ipc|ipc_streaming|parquet|async|json|extract_jsonpath|catalog|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
        run: |
          sed -i 's/serde_json = { workspace = true, optional = true }/serde_json = { workspace = true }/' crates/polars-python/Cargo.toml
          sed -i 's/"serde_json", //' crates/polars-python/Cargo.toml
          sed -E -i "/^  \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/Cargo.toml

      - name: Setup emsdk
        uses: mymindstorm/setup-emsdk@v14
        with:
          # This should match the exact version of Emscripten used by Pyodide
          # UPDATE; set to latest due to breaking build
          version: latest

      - name: Set CFLAGS and RUSTFLAGS for wasm32
        run: |
            echo "CFLAGS=-fPIC" >> $GITHUB_ENV
            echo "RUSTFLAGS=-C link-self-contained=no" >> $GITHUB_ENV

      - name: Build wheel
        uses: PyO3/maturin-action@v1
        with:
          command: build
          target: wasm32-unknown-emscripten
          args: >
            --profile dist-release
            --manifest-path py-polars/Cargo.toml
            --interpreter python3.10
            --out wasm-dist
          maturin-version: 1.8.3

      - name: Upload wheel
        uses: actions/upload-artifact@v4
        with:
          name: wheel-polars-emscripten-wasm32
          path: wasm-dist/*.whl

  publish-to-pypi:
    needs: [create-sdist, build-wheels, build-wheel-pyodide]
    environment:
      name: release-python
      url: https://pypi.org/project/polars
    runs-on: ubuntu-latest
    permissions:
      id-token: write

    steps:
      - name: Download sdists and wheels
        uses: actions/download-artifact@v4
        with:
          path: dist
          merge-multiple: true

      - name: Remove Emscripten wheel
        run: rm -f dist/*emscripten*.whl

      - name: Publish to PyPI
        if: inputs.dry-run == false
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true

  publish-to-github:
    needs: [publish-to-pypi, build-wheel-pyodide]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.sha }}

      - name: Download sdist
        uses: actions/download-artifact@v4
        with:
          name: sdist-polars
          path: dist

      - name: Download Pyodide wheel
        uses: actions/download-artifact@v4
        with:
          name: wheel-polars-emscripten-wasm32
          path: wasm-dist

      - name: Get version from Cargo.toml
        id: version
        working-directory: py-polars
        run: |
          VERSION=$(grep -m 1 -oP 'version = "\K[^"]+' Cargo.toml)
          if [[ "$VERSION" == *"-"* ]]; then
            IS_PRERELEASE=true
          else
            IS_PRERELEASE=false
          fi
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "is_prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT

      - name: Create GitHub release
        id: github-release
        uses: release-drafter/release-drafter@v6
        with:
          config-name: release-drafter-python.yml
          name: Python Polars ${{ steps.version.outputs.version }}
          tag: py-${{ steps.version.outputs.version }}
          version: ${{ steps.version.outputs.version }}
          prerelease: ${{ steps.version.outputs.is_prerelease }}
          commitish: ${{ inputs.sha || github.sha }}
          disable-autolabeler: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Upload sdist to GitHub release
        run: gh release upload $TAG $FILES --clobber
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ steps.github-release.outputs.tag_name }}
          FILES: dist/polars-*.tar.gz wasm-dist/polars-*.whl

      - name: Publish GitHub release
        if: inputs.dry-run == false
        run: gh release edit $TAG --draft=false
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ steps.github-release.outputs.tag_name }}

      - name: Trigger other workflows related to the release
        if: inputs.dry-run == false
        uses: peter-evans/repository-dispatch@v3
        with:
          event-type: python-release
          client-payload: >
            {
              "version": "${{ steps.version.outputs.version }}",
              "is_prerelease": "${{ steps.version.outputs.is_prerelease }}",
              "tag": "${{ steps.github-release.outputs.tag_name }}",
              "sha": "${{ inputs.sha || github.sha }}"
            }