From 8e1fd4dd36f689b835ce113c116f7cabd20ef296 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 28 Jan 2026 15:56:01 +0800 Subject: [PATCH 1/4] Expand wheel matrix for Python 3.13/3.14 Add macOS/Windows entries for free-threaded variants. Include distinct artifact suffixes and use explicit maturin interpreter settings. Add manylinux entries for the latest Python versions with similar configurations. --- .github/workflows/build.yml | 91 +++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2dc8b96fe..60e6adae1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,8 +74,49 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] - os: [macos-latest, windows-latest] + include: + - os: macos-latest + python-version: "3.10" + artifact-suffix: "" + - os: macos-latest + python-version: "3.11" + artifact-suffix: "" + - os: macos-latest + python-version: "3.12" + artifact-suffix: "" + - os: macos-latest + python-version: "3.13" + artifact-suffix: "" + - os: macos-latest + python-version: "3.14" + artifact-suffix: "" + - os: macos-latest + python-version: "3.13t" + artifact-suffix: "-freethreaded" + - os: macos-latest + python-version: "3.14t" + artifact-suffix: "-freethreaded" + - os: windows-latest + python-version: "3.10" + artifact-suffix: "" + - os: windows-latest + python-version: "3.11" + artifact-suffix: "" + - os: windows-latest + python-version: "3.12" + artifact-suffix: "" + - os: windows-latest + python-version: "3.13" + artifact-suffix: "" + - os: windows-latest + python-version: "3.14" + artifact-suffix: "" + - os: windows-latest + python-version: "3.13t" + artifact-suffix: "-freethreaded" + - os: windows-latest + python-version: "3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 @@ -105,7 +146,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -121,7 +162,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-${{ matrix.os }} + name: dist-${{ matrix.os }}${{ matrix.artifact-suffix }} path: target/wheels/* build-macos-x86_64: @@ -131,7 +172,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + include: + - python-version: "3.10" + artifact-suffix: "" + - python-version: "3.11" + artifact-suffix: "" + - python-version: "3.12" + artifact-suffix: "" + - python-version: "3.13" + artifact-suffix: "" + - python-version: "3.14" + artifact-suffix: "" + - python-version: "3.13t" + artifact-suffix: "-freethreaded" + - python-version: "3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 @@ -161,7 +216,7 @@ jobs: - name: Build Python package run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features substrait --interpreter python - name: List Mac wheels run: find target/wheels/ @@ -169,13 +224,21 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-macos-aarch64 + name: dist-macos-aarch64${{ matrix.artifact-suffix }} path: target/wheels/* build-manylinux-x86_64: needs: [generate-license] name: Manylinux x86_64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-suffix: "" + - interpreter: "3.13t 3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -194,17 +257,26 @@ jobs: target: x86_64 manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --manylinux 2014 --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64${{ matrix.artifact-suffix }} path: target/wheels/* build-manylinux-aarch64: needs: [generate-license] name: Manylinux arm64 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - interpreter: "3.10 3.11 3.12 3.13 3.14" + artifact-suffix: "" + - interpreter: "3.13t 3.14t" + artifact-suffix: "-freethreaded" steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -224,11 +296,12 @@ jobs: # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail manylinux: 2_28 rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + interpreter: ${{ matrix.interpreter }} args: --release --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-aarch64 + name: dist-manylinux-aarch64${{ matrix.artifact-suffix }} path: target/wheels/* build-sdist: From 6efe8789c92d642bbff2e8b786ffd256a397f437 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 28 Jan 2026 15:56:35 +0800 Subject: [PATCH 2/4] Update PyO3 dependency and document wheel rationale Rely on explicit abi3-py310 flag for ABI3 compatibility in the PyO3 dependency feature set while retaining extension-module enabled. Document the rationale for free-threaded 3.13/3.14 wheels alongside ABI3 wheel guidance in the release notes. --- Cargo.toml | 1 - dev/release/README.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 364713964..7fe9ee617 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,6 @@ tokio = { version = "1.47", features = [ ] } pyo3 = { version = "0.26", features = [ "extension-module", - "abi3", "abi3-py310", ] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } diff --git a/dev/release/README.md b/dev/release/README.md index 5d2fae5a7..5a76da810 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -130,6 +130,10 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl datafusion-22.0.0-cp37-abi3-win_amd64.whl ``` +Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and ignore the `abi3` setting. We keep +the `abi3-py310` feature enabled to preserve the stable-ABI wheels for GIL-enabled Python, and we additionally ship +version-specific `cp313t`/`cp314t` wheels for the free-threaded builds. + Upload the wheels to testpypi. ```bash From c03f56d0a1e78426a1513700969cfef3f306920e Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 4 Feb 2026 14:01:33 +0800 Subject: [PATCH 3/4] Update build.yml to use python-tag for artifact naming and enhance README with artifact naming conventions --- .github/workflows/build.yml | 60 ++++++++++++++++++------------------- dev/release/README.md | 6 ++++ 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60e6adae1..bbaf81183 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,46 +77,46 @@ jobs: include: - os: macos-latest python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - os: macos-latest python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - os: macos-latest python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - os: macos-latest python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - os: macos-latest python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - os: macos-latest python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - os: macos-latest python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t - os: windows-latest python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - os: windows-latest python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - os: windows-latest python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - os: windows-latest python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - os: windows-latest python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - os: windows-latest python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - os: windows-latest python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -162,7 +162,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-${{ matrix.os }}${{ matrix.artifact-suffix }} + name: dist-${{ matrix.os }}-${{ matrix.python-tag }} path: target/wheels/* build-macos-x86_64: @@ -174,19 +174,19 @@ jobs: matrix: include: - python-version: "3.10" - artifact-suffix: "" + python-tag: cp310 - python-version: "3.11" - artifact-suffix: "" + python-tag: cp311 - python-version: "3.12" - artifact-suffix: "" + python-tag: cp312 - python-version: "3.13" - artifact-suffix: "" + python-tag: cp313 - python-version: "3.14" - artifact-suffix: "" + python-tag: cp314 - python-version: "3.13t" - artifact-suffix: "-freethreaded" + python-tag: cp313t - python-version: "3.14t" - artifact-suffix: "-freethreaded" + python-tag: cp314t steps: - uses: actions/checkout@v5 @@ -224,7 +224,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-macos-aarch64${{ matrix.artifact-suffix }} + name: dist-macos-aarch64-${{ matrix.python-tag }} path: target/wheels/* build-manylinux-x86_64: @@ -236,9 +236,9 @@ jobs: matrix: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" - artifact-suffix: "" + artifact-label: cp310-314 - interpreter: "3.13t 3.14t" - artifact-suffix: "-freethreaded" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -262,7 +262,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-x86_64${{ matrix.artifact-suffix }} + name: dist-manylinux-x86_64-${{ matrix.artifact-label }} path: target/wheels/* build-manylinux-aarch64: @@ -274,9 +274,9 @@ jobs: matrix: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" - artifact-suffix: "" + artifact-label: cp310-314 - interpreter: "3.13t 3.14t" - artifact-suffix: "-freethreaded" + artifact-label: cp313t-314t steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -301,7 +301,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v4 with: - name: dist-manylinux-aarch64${{ matrix.artifact-suffix }} + name: dist-manylinux-aarch64-${{ matrix.artifact-label }} path: target/wheels/* build-sdist: @@ -396,7 +396,7 @@ jobs: - name: Download pre-built Linux wheel uses: actions/download-artifact@v5 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64-cp310-314 path: wheels/ # Install from the pre-built wheel diff --git a/dev/release/README.md b/dev/release/README.md index 5a76da810..2568caa06 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -222,6 +222,12 @@ cargo publish ### Publishing Python Artifacts to PyPi +GitHub Actions groups wheel artifacts by platform and interpreter tag using the pattern `dist--`. +For example, standard manylinux wheels live under `dist-manylinux-x86_64-cp310-314` while the free-threaded builds +use `dist-manylinux-x86_64-cp313t-314t`. macOS and Windows jobs publish one artifact per CPython version as well +(`dist-macos-latest-cp311`, `dist-windows-latest-cp313t`, etc.). Download the exact tags you intend to push to PyPI, +and remember that the docs workflow currently installs from the `cp310-314` manylinux artifact. + Go to the Test PyPI page of Datafusion, and download [all published artifacts](https://test.pypi.org/project/datafusion/#files) under `dist-release/` directory. Then proceed uploading them using `twine`: From 0ebd47623edee5b3978c83ef130f2a9ea6982499 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Wed, 11 Feb 2026 13:49:39 +0800 Subject: [PATCH 4/4] Enhance build workflow to validate interpreter and wheel mode compatibility; update README for clarity on free-threaded builds and limited API usage --- .github/workflows/build.yml | 114 ++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 +- dev/release/README.md | 6 +- 3 files changed, 114 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bbaf81183..59731ba8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,10 +143,45 @@ jobs: with: enable-cache: true + - name: Validate interpreter/wheel mode + shell: bash + env: + WHEEL_MODE: ${{ endsWith(matrix.python-version, 't') && 'freethreaded' || 'abi3' }} + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} + run: | + python - <<'PY' + import os + import sys + import sysconfig + + mode = os.environ["WHEEL_MODE"] + features = os.environ["MATURIN_FEATURES"] + is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + print(sys.version) + print(f"Py_GIL_DISABLED={int(is_free_threaded)}") + print(f"wheel_mode={mode}") + print(f"maturin_features={features}") + + if mode == "abi3" and is_free_threaded: + raise SystemExit("Invalid matrix: free-threaded interpreter cannot build abi3 wheels") + if mode == "freethreaded" and not is_free_threaded: + raise SystemExit("Invalid matrix: freethreaded wheel mode requires a free-threaded interpreter") + PY + + - name: Show resolved PyO3 versions + shell: bash + run: | + cargo tree -e no-dev -i pyo3 + cargo tree -e no-dev -i pyo3-build-config + - name: Build Python package + shell: bash + env: + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait --interpreter python + uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter python - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -213,10 +248,45 @@ jobs: with: enable-cache: true + - name: Validate interpreter/wheel mode + shell: bash + env: + WHEEL_MODE: ${{ endsWith(matrix.python-version, 't') && 'freethreaded' || 'abi3' }} + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} + run: | + python - <<'PY' + import os + import sys + import sysconfig + + mode = os.environ["WHEEL_MODE"] + features = os.environ["MATURIN_FEATURES"] + is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + print(sys.version) + print(f"Py_GIL_DISABLED={int(is_free_threaded)}") + print(f"wheel_mode={mode}") + print(f"maturin_features={features}") + + if mode == "abi3" and is_free_threaded: + raise SystemExit("Invalid matrix: free-threaded interpreter cannot build abi3 wheels") + if mode == "freethreaded" and not is_free_threaded: + raise SystemExit("Invalid matrix: freethreaded wheel mode requires a free-threaded interpreter") + PY + + - name: Show resolved PyO3 versions + shell: bash + run: | + cargo tree -e no-dev -i pyo3 + cargo tree -e no-dev -i pyo3-build-config + - name: Build Python package + shell: bash + env: + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} run: | uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait --interpreter python + uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter python - name: List Mac wheels run: find target/wheels/ @@ -237,8 +307,12 @@ jobs: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" artifact-label: cp310-314 + wheel-mode: abi3 + maturin-features: protoc,substrait,py-limited-api - interpreter: "3.13t 3.14t" artifact-label: cp313t-314t + wheel-mode: freethreaded + maturin-features: protoc,substrait steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -248,6 +322,20 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt + - name: Validate wheel mode matrix + shell: bash + run: | + echo "interpreter=${{ matrix.interpreter }}" + echo "wheel_mode=${{ matrix.wheel-mode }}" + echo "maturin_features=${{ matrix.maturin-features }}" + if [[ "${{ matrix.wheel-mode }}" == "abi3" && "${{ matrix.interpreter }}" == *t* ]]; then + echo "Invalid matrix: abi3 mode cannot use free-threaded interpreters" + exit 1 + fi + if [[ "${{ matrix.wheel-mode }}" == "freethreaded" && "${{ matrix.interpreter }}" != *t* ]]; then + echo "Invalid matrix: freethreaded mode requires free-threaded interpreters" + exit 1 + fi - name: Build wheels uses: PyO3/maturin-action@v1 env: @@ -258,7 +346,7 @@ jobs: manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 interpreter: ${{ matrix.interpreter }} - args: --release --manylinux 2014 --features protoc,substrait + args: --release --manylinux 2014 --features ${{ matrix.maturin-features }} - name: Archive wheels uses: actions/upload-artifact@v4 with: @@ -275,8 +363,12 @@ jobs: include: - interpreter: "3.10 3.11 3.12 3.13 3.14" artifact-label: cp310-314 + wheel-mode: abi3 + maturin-features: protoc,substrait,py-limited-api - interpreter: "3.13t 3.14t" artifact-label: cp313t-314t + wheel-mode: freethreaded + maturin-features: protoc,substrait steps: - uses: actions/checkout@v5 - run: rm LICENSE.txt @@ -286,6 +378,20 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt + - name: Validate wheel mode matrix + shell: bash + run: | + echo "interpreter=${{ matrix.interpreter }}" + echo "wheel_mode=${{ matrix.wheel-mode }}" + echo "maturin_features=${{ matrix.maturin-features }}" + if [[ "${{ matrix.wheel-mode }}" == "abi3" && "${{ matrix.interpreter }}" == *t* ]]; then + echo "Invalid matrix: abi3 mode cannot use free-threaded interpreters" + exit 1 + fi + if [[ "${{ matrix.wheel-mode }}" == "freethreaded" && "${{ matrix.interpreter }}" != *t* ]]; then + echo "Invalid matrix: freethreaded mode requires free-threaded interpreters" + exit 1 + fi - name: Build wheels uses: PyO3/maturin-action@v1 env: @@ -297,7 +403,7 @@ jobs: manylinux: 2_28 rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 interpreter: ${{ matrix.interpreter }} - args: --release --features protoc,substrait + args: --release --features ${{ matrix.maturin-features }} - name: Archive wheels uses: actions/upload-artifact@v4 with: diff --git a/Cargo.toml b/Cargo.toml index 7fe9ee617..ad7f4048f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ include = [ default = ["mimalloc"] protoc = ["datafusion-substrait/protoc"] substrait = ["dep:datafusion-substrait"] +py-limited-api = ["pyo3/abi3-py310"] [dependencies] tokio = { version = "1.47", features = [ @@ -50,7 +51,6 @@ tokio = { version = "1.47", features = [ ] } pyo3 = { version = "0.26", features = [ "extension-module", - "abi3-py310", ] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } pyo3-log = "0.13.2" diff --git a/dev/release/README.md b/dev/release/README.md index 2568caa06..a11148134 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -130,9 +130,9 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl datafusion-22.0.0-cp37-abi3-win_amd64.whl ``` -Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and ignore the `abi3` setting. We keep -the `abi3-py310` feature enabled to preserve the stable-ABI wheels for GIL-enabled Python, and we additionally ship -version-specific `cp313t`/`cp314t` wheels for the free-threaded builds. +Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and cannot use the limited API (`abi3`). +The release workflow enables Cargo feature `py-limited-api` for GIL-enabled wheels and disables it for free-threaded +builds, producing version-specific `cp313t`/`cp314t` wheels. Upload the wheels to testpypi.