diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c35801b11..d7af9b663 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,7 +186,7 @@ jobs: manylinux: "2_28" # FFI test wheel only needs to be built once per platform; gate to abi3. - - name: Build FFI test library + - name: Build FFI provider test library if: matrix.python-tag == 'abi3' uses: PyO3/maturin-action@v1 with: @@ -196,6 +196,16 @@ jobs: args: --out dist rustup-components: rust-std + - name: Build FFI query planner test library + if: matrix.python-tag == 'abi3' + uses: PyO3/maturin-action@v1 + with: + target: x86_64-unknown-linux-gnu + manylinux: "2_28" + working-directory: examples/datafusion-ffi-query-planner-example + args: --out dist + rustup-components: rust-std + - name: Archive wheels uses: actions/upload-artifact@v7 with: @@ -207,7 +217,9 @@ jobs: uses: actions/upload-artifact@v7 with: name: test-ffi-manylinux-x86_64 - path: examples/datafusion-ffi-example/dist/* + path: | + examples/datafusion-ffi-example/dist/* + examples/datafusion-ffi-query-planner-example/dist/* # ============================================ # Build - Linux ARM64 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 558e751c8..047b35039 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -93,11 +93,15 @@ jobs: uv venv --python "${{ steps.setup-python.outputs.python-path }}" VENV_PY="$PWD/.venv/bin/python" uv sync --python "$VENV_PY" --dev --no-install-package datafusion + # Search recursively: the FFI artifact bundles more than one + # project, so upload-artifact keeps a `/dist/` prefix + # and the wheels are not all at the top of wheels/. WHEELS=$(find wheels/ -name "*.whl") if [ -n "$WHEELS" ]; then echo "Installing wheels:" echo "$WHEELS" - uv pip install --python "$VENV_PY" wheels/*.whl + # shellcheck disable=SC2086 # intentional split on newlines + uv pip install --python "$VENV_PY" $WHEELS else echo "ERROR: No wheels found!" exit 1 @@ -121,6 +125,8 @@ jobs: run: | cd examples/datafusion-ffi-example uv run --no-project pytest python/tests/_test*.py + cd ../datafusion-ffi-query-planner-example + uv run --no-project pytest python/tests/_test*.py - name: Run tpchgen-cli to create 1 Gb dataset if: matrix.wheel-tag == 'abi3' diff --git a/Cargo.lock b/Cargo.lock index ab222b177..5549ae8ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,7 +202,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.23.1", + "base64 0.23.0", "chrono", "comfy-table", "half", @@ -429,9 +429,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64" -version = "0.23.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" [[package]] name = "bigdecimal" @@ -792,7 +792,7 @@ dependencies = [ [[package]] name = "datafusion" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-schema", @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-trait", @@ -869,7 +869,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-trait", @@ -892,7 +892,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-ipc", @@ -918,7 +918,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "futures", "log", @@ -928,7 +928,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-compression", @@ -964,7 +964,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-ipc", @@ -988,7 +988,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-avro", @@ -1006,7 +1006,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-trait", @@ -1029,7 +1029,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-trait", @@ -1052,7 +1052,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-schema", @@ -1084,12 +1084,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" [[package]] name = "datafusion-execution" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-buffer", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-schema", @@ -1138,7 +1138,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1149,7 +1149,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-schema", @@ -1200,14 +1200,31 @@ dependencies = [ "pyo3-log", ] +[[package]] +name = "datafusion-ffi-query-planner-example" +version = "54.0.0" +dependencies = [ + "async-trait", + "datafusion", + "datafusion-catalog", + "datafusion-common", + "datafusion-ffi", + "datafusion-proto", + "datafusion-python-util", + "datafusion-session", + "pyo3", + "pyo3-build-config", + "pyo3-log", +] + [[package]] name = "datafusion-functions" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-buffer", - "base64 0.23.1", + "base64 0.23.0", "blake2", "blake3", "chrono", @@ -1234,7 +1251,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1254,7 +1271,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1265,7 +1282,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-ord", @@ -1289,7 +1306,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "async-trait", @@ -1304,7 +1321,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1320,7 +1337,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1329,7 +1346,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "datafusion-doc", "quote", @@ -1339,7 +1356,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "chrono", @@ -1358,7 +1375,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1380,7 +1397,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1394,7 +1411,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "chrono", @@ -1411,7 +1428,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1430,7 +1447,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "arrow-data", @@ -1466,7 +1483,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-catalog", @@ -1492,7 +1509,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1502,7 +1519,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -1512,7 +1529,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "datafusion-common", @@ -1572,7 +1589,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow-schema", "async-trait", @@ -1586,7 +1603,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "bigdecimal", @@ -1615,7 +1632,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "arrow", "bigdecimal", @@ -1634,7 +1651,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "55.0.0" -source = "git+https://github.com/apache/datafusion?rev=55.0.0-rc3#d5552342012888b7d1a3ab88d92e3d292fc0cde0" +source = "git+https://github.com/timsaucer/datafusion?branch=fix%2Fffi-session-create-physical-plan-codec-55#24d8903b0e4530642feb53319d7e7de1be244f27" dependencies = [ "async-recursion", "async-trait", @@ -2667,7 +2684,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.23.1", + "base64 0.23.0", "brotli", "bytes", "chrono", diff --git a/Cargo.toml b/Cargo.toml index a9e15d7e9..7efe64a7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,12 @@ edition = "2024" rust-version = "1.88" [workspace] -members = ["crates/core", "crates/util", "examples/datafusion-ffi-example"] +members = [ + "crates/core", + "crates/util", + "examples/datafusion-ffi-example", + "examples/datafusion-ffi-query-planner-example", +] resolver = "3" [workspace.dependencies] @@ -50,6 +55,7 @@ datafusion-functions-aggregate = { version = "55.0.0" } datafusion-functions-window = { version = "55.0.0" } datafusion-spark = { version = "55.0.0" } datafusion-expr = { version = "55.0.0" } +datafusion-session = { version = "55.0.0" } prost = "0.14.3" serde_json = "1" uuid = { version = "1.23" } @@ -72,13 +78,14 @@ codegen-units = 2 # We cannot publish to crates.io with any patches in the below section. Developers # must remove any entries in this section before creating a release candidate. [patch.crates-io] -datafusion = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-substrait = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-ffi = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-common = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } -datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "55.0.0-rc3" } +datafusion = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-substrait = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-proto = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-ffi = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-catalog = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-common = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-functions-aggregate = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-functions-window = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-spark = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-expr = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } +datafusion-session = { git = "https://github.com/timsaucer/datafusion", branch = "fix/ffi-session-create-physical-plan-codec-55" } diff --git a/crates/core/src/codec.rs b/crates/core/src/codec.rs index 26853e69f..1c6658939 100644 --- a/crates/core/src/codec.rs +++ b/crates/core/src/codec.rs @@ -103,6 +103,7 @@ use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionContext; use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec}; use datafusion_proto::physical_plan::{ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension, @@ -233,10 +234,31 @@ fn strip_wire_header<'a>( /// Sitting at the top of the session's logical codec stack means /// every serializer that reads `session.logical_codec()` automatically /// picks up Python-aware encoding for free. -#[derive(Debug)] pub struct PythonLogicalCodec { inner: Arc, python_udf_inlining: bool, + /// Keeps the exporting session alive for a codec handed across the FFI + /// boundary. + /// + /// `FFI_TaskContextProvider` stores its provider in a `Weak`, so an exported + /// codec stops working the moment the object that produced it goes out of + /// scope. Retaining the session in the inner codec survives both that and + /// `clone`, which clones the inner codec's `Arc` and so carries this along. + /// + /// Set this only on codecs that are leaving this library. A codec installed + /// *in* a session must not hold one, or the session would own itself: + /// `SessionContext -> SessionState -> query planner -> FFI codec -> here`. + exported_session: Option>, +} + +impl std::fmt::Debug for PythonLogicalCodec { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PythonLogicalCodec") + .field("inner", &self.inner) + .field("python_udf_inlining", &self.python_udf_inlining) + .finish_non_exhaustive() + } } impl PythonLogicalCodec { @@ -244,9 +266,19 @@ impl PythonLogicalCodec { Self { inner, python_udf_inlining: true, + exported_session: None, } } + /// Retain `ctx` so this codec keeps working after the exporting + /// `SessionContext` goes out of scope. Only for codecs being exported over + /// FFI; see the `exported_session` field for why installed codecs must not + /// use this. + pub fn with_exported_session(mut self, ctx: Arc) -> Self { + self.exported_session = Some(ctx); + self + } + pub fn inner(&self) -> &Arc { &self.inner } @@ -443,10 +475,31 @@ fn refuse_inline_payload(kind: &str, name: &str) -> datafusion::error::DataFusio /// would round-trip at the logical level but break at the physical /// level. Both layers reuse the shared payload framing /// ([`PY_SCALAR_UDF_FAMILY`] et al.) so the wire format is identical. -#[derive(Debug)] pub struct PythonPhysicalCodec { inner: Arc, python_udf_inlining: bool, + /// Keeps the exporting session alive for a codec handed across the FFI + /// boundary. + /// + /// `FFI_TaskContextProvider` stores its provider in a `Weak`, so an exported + /// codec stops working the moment the object that produced it goes out of + /// scope. Retaining the session in the inner codec survives both that and + /// `clone`, which clones the inner codec's `Arc` and so carries this along. + /// + /// Set this only on codecs that are leaving this library. A codec installed + /// *in* a session must not hold one, or the session would own itself: + /// `SessionContext -> SessionState -> query planner -> FFI codec -> here`. + exported_session: Option>, +} + +impl std::fmt::Debug for PythonPhysicalCodec { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PythonPhysicalCodec") + .field("inner", &self.inner) + .field("python_udf_inlining", &self.python_udf_inlining) + .finish_non_exhaustive() + } } impl PythonPhysicalCodec { @@ -454,9 +507,19 @@ impl PythonPhysicalCodec { Self { inner, python_udf_inlining: true, + exported_session: None, } } + /// Retain `ctx` so this codec keeps working after the exporting + /// `SessionContext` goes out of scope. Only for codecs being exported over + /// FFI; see the `exported_session` field for why installed codecs must not + /// use this. + pub fn with_exported_session(mut self, ctx: Arc) -> Self { + self.exported_session = Some(ctx); + self + } + pub fn inner(&self) -> &Arc { &self.inner } diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index 7bbeed2f1..5b9f09a06 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -36,7 +36,7 @@ use datafusion::datasource::listing::{ }; use datafusion::datasource::{MemTable, TableProvider}; use datafusion::execution::context::{ - DataFilePaths, SQLOptions, SessionConfig, SessionContext, TaskContext, + DataFilePaths, QueryPlanner, SQLOptions, SessionConfig, SessionContext, TaskContext, }; use datafusion::execution::disk_manager::DiskManagerMode; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; @@ -53,12 +53,14 @@ use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; use datafusion_ffi::execution::FFI_TaskContextProvider; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::{FFI_QueryPlanner, ForeignQueryPlanner}; use datafusion_ffi::table_provider_factory::FFI_TableProviderFactory; use datafusion_proto::logical_plan::LogicalExtensionCodec; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_python_util::{ create_logical_extension_capsule, create_physical_extension_capsule, - ffi_logical_codec_from_pycapsule, get_global_ctx, get_tokio_runtime, + create_query_planner_capsule, ffi_logical_codec_from_pycapsule, + ffi_query_planner_from_pycapsule, get_global_ctx, get_tokio_runtime, physical_codec_from_pycapsule, physical_optimizer_rule_from_pycapsule, spawn_future, wait_for_future, }; @@ -1211,6 +1213,18 @@ impl PySessionContext { Ok(()) } + pub fn with_query_planner(&self, planner: Bound<'_, PyAny>) -> PyDataFusionResult { + let planner = ffi_query_planner_from_pycapsule(&planner)?; + let ctx = + self.ctx_with_rebound_planner(&self.logical_codec, &self.physical_codec, Some(planner)); + + Ok(Self { + ctx, + logical_codec: Arc::clone(&self.logical_codec), + physical_codec: Arc::clone(&self.physical_codec), + }) + } + pub fn table_provider(&self, name: &str, py: Python) -> PyResult { let provider = wait_for_future(py, self.ctx.table_provider(name)) // Outer error: runtime/async failure @@ -1381,8 +1395,25 @@ impl PySessionContext { &self, py: Python<'py>, ) -> PyResult> { - let ffi = self.ffi_logical_codec(); - create_logical_extension_capsule(py, ffi.as_ref()) + create_logical_extension_capsule(py, &self.exported_ffi_logical_codec()) + } + + pub fn __datafusion_query_planner__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + // An already-foreign planner is re-exported as its original handle + // rather than gaining another layer, because `new_with_ffi_codecs` + // unwraps a `ForeignQueryPlanner`. It still adopts the codecs supplied + // here, so a consumer that wraps this capsule decodes our plans with + // our codecs. + let planner = Arc::clone(self.ctx.state().query_planner()); + let ffi = FFI_QueryPlanner::new_with_ffi_codecs( + planner, + self.exported_ffi_logical_codec(), + self.exported_ffi_physical_codec(), + ); + create_query_planner_capsule(py, &ffi) } pub fn with_logical_extension_codec<'py>( @@ -1393,10 +1424,12 @@ impl PySessionContext { let inner: Arc = (&inner_ffi).into(); let logical_codec = Arc::new(PythonLogicalCodec::new(inner)); + let physical_codec = Arc::clone(&self.physical_codec); + let ctx = self.ctx_with_rebound_planner(&logical_codec, &physical_codec, None); Ok(Self { - ctx: Arc::clone(&self.ctx), + ctx, logical_codec, - physical_codec: Arc::clone(&self.physical_codec), + physical_codec, }) } @@ -1404,8 +1437,7 @@ impl PySessionContext { &self, py: Python<'py>, ) -> PyResult> { - let ffi = self.ffi_physical_codec(); - create_physical_extension_capsule(py, ffi.as_ref()) + create_physical_extension_capsule(py, &self.exported_ffi_physical_codec()) } pub fn with_physical_extension_codec<'py>( @@ -1415,9 +1447,11 @@ impl PySessionContext { let inner = physical_codec_from_pycapsule(&codec)?; let physical_codec = Arc::new(PythonPhysicalCodec::new(inner)); + let logical_codec = Arc::clone(&self.logical_codec); + let ctx = self.ctx_with_rebound_planner(&logical_codec, &physical_codec, None); Ok(Self { - ctx: Arc::clone(&self.ctx), - logical_codec: Arc::clone(&self.logical_codec), + ctx, + logical_codec, physical_codec, }) } @@ -1431,8 +1465,9 @@ impl PySessionContext { PythonPhysicalCodec::new(Arc::clone(self.physical_codec.inner())) .with_python_udf_inlining(enabled), ); + let ctx = self.ctx_with_rebound_planner(&logical_codec, &physical_codec, None); Self { - ctx: Arc::clone(&self.ctx), + ctx, logical_codec, physical_codec, } @@ -1440,6 +1475,64 @@ impl PySessionContext { } impl PySessionContext { + /// Return the [`SessionContext`] a derived `PySessionContext` should use, + /// binding a foreign query planner to `logical_codec` and `physical_codec`. + /// + /// Pass `Some(planner)` to install one, or `None` to rebind whichever + /// planner the session already holds. + /// + /// With no foreign planner in play there is nothing to rebind, so the + /// existing context is shared and swapping codecs alone does not change + /// what a derived context observes. A foreign planner does have to be + /// rebound, and the FFI codecs capture the context they are built against, + /// so that case forks the session: the codecs are built from the fork and + /// then the fork's state is overwritten in place, because rebuilding the + /// context afterwards would leave the codecs pointing at a session that is + /// no longer used for planning. + /// + /// A fork is not a deep copy. `SessionState` keeps its catalog list behind + /// an `Arc`, so catalogs and tables stay shared with the original session, + /// while registered functions and the configuration are snapshotted at the + /// time of the call. The session id is deliberately carried over. + fn ctx_with_rebound_planner( + &self, + logical_codec: &Arc, + physical_codec: &Arc, + planner: Option, + ) -> Arc { + let state = self.ctx.state(); + + // When the caller is only replacing codecs, recover the handle behind + // the installed planner so it can be rebound below. A planner this + // library owns is not a `ForeignQueryPlanner` and needs no rebinding, + // because it does not carry codecs of its own. + let planner = planner.or_else(|| { + let installed: &dyn std::any::Any = state.query_planner().as_ref(); + installed + .downcast_ref::() + .map(|planner| planner.0.clone()) + }); + + let Some(planner) = planner else { + return Arc::clone(&self.ctx); + }; + + let ctx = Arc::new(SessionContext::new_with_state(state)); + let inner: Arc = (&planner).into(); + let planner: Arc = (&FFI_QueryPlanner::new_with_ffi_codecs( + inner, + Self::ffi_logical_codec_for(&ctx, logical_codec), + Self::ffi_physical_codec_for(&ctx, physical_codec), + )) + .into(); + let state = SessionStateBuilder::new_from_existing(ctx.state()) + .with_query_planner(planner) + .build(); + *ctx.state_ref().write() = state; + + ctx + } + async fn _table(&self, name: &str) -> datafusion::common::Result { self.ctx.table(name).await } @@ -1501,28 +1594,58 @@ impl PySessionContext { /// Used at every site that exports the codec across an FFI boundary /// (capsule getters, Rust wrappers for Python-defined providers, etc.). pub(crate) fn ffi_logical_codec(&self) -> Arc { - let inner: Arc = - Arc::clone(&self.logical_codec) as Arc; + Arc::new(Self::ffi_logical_codec_for(&self.ctx, &self.logical_codec)) + } + + fn ffi_logical_codec_for( + ctx: &Arc, + codec: &Arc, + ) -> FFI_LogicalExtensionCodec { + let codec: Arc = + Arc::clone(codec) as Arc; let runtime = get_tokio_runtime().handle().clone(); - let ctx_provider = Arc::clone(&self.ctx) as Arc; - Arc::new(FFI_LogicalExtensionCodec::new( - inner, - Some(runtime), - &ctx_provider, - )) + let ctx_provider = Arc::clone(ctx) as Arc; + FFI_LogicalExtensionCodec::new(codec, Some(runtime), &ctx_provider) } - /// Build an FFI-wrapped clone of the session's physical codec on demand. - pub(crate) fn ffi_physical_codec(&self) -> Arc { - let inner: Arc = - Arc::clone(&self.physical_codec) as Arc; + fn ffi_physical_codec_for( + ctx: &Arc, + codec: &Arc, + ) -> FFI_PhysicalExtensionCodec { + let codec: Arc = + Arc::clone(codec) as Arc; let runtime = get_tokio_runtime().handle().clone(); - let ctx_provider = Arc::clone(&self.ctx) as Arc; - Arc::new(FFI_PhysicalExtensionCodec::new( - inner, - Some(runtime), - &ctx_provider, - )) + let ctx_provider = Arc::clone(ctx) as Arc; + FFI_PhysicalExtensionCodec::new(codec, Some(runtime), &ctx_provider) + } + + /// Build an FFI-wrapped logical codec for handing out in a PyCapsule. + /// + /// Same as [`Self::ffi_logical_codec`] except the inner codec retains this + /// session. The FFI task-context handle is weak, so without that a capsule + /// stops working as soon as the exporting `SessionContext` goes out of + /// scope, which the natural `ctx = ctx.with_query_planner(planner)` does. + /// + /// Only for codecs leaving this library. The plain builder is still correct + /// for codecs attached to objects that end up back inside this session, + /// which would otherwise make the session own itself. + fn exported_ffi_logical_codec(&self) -> FFI_LogicalExtensionCodec { + let codec = Arc::new( + PythonLogicalCodec::new(Arc::clone(self.logical_codec.inner())) + .with_python_udf_inlining(self.logical_codec.python_udf_inlining()) + .with_exported_session(Arc::clone(&self.ctx)), + ); + Self::ffi_logical_codec_for(&self.ctx, &codec) + } + + /// Physical companion to [`Self::exported_ffi_logical_codec`]. + fn exported_ffi_physical_codec(&self) -> FFI_PhysicalExtensionCodec { + let codec = Arc::new( + PythonPhysicalCodec::new(Arc::clone(self.physical_codec.inner())) + .with_python_udf_inlining(self.physical_codec.python_udf_inlining()) + .with_exported_session(Arc::clone(&self.ctx)), + ); + Self::ffi_physical_codec_for(&self.ctx, &codec) } } diff --git a/crates/util/src/lib.rs b/crates/util/src/lib.rs index 9327d7f2f..7375a034c 100644 --- a/crates/util/src/lib.rs +++ b/crates/util/src/lib.rs @@ -29,6 +29,7 @@ use datafusion_ffi::execution::FFI_TaskContextProvider; use datafusion_ffi::physical_optimizer::FFI_PhysicalOptimizerRule; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::FFI_QueryPlanner; use datafusion_ffi::table_provider::FFI_TableProvider; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use pyo3::exceptions::{PyImportError, PyTypeError, PyValueError}; @@ -231,6 +232,38 @@ pub fn ffi_logical_codec_from_pycapsule(obj: Bound) -> PyResult( + py: Python<'py>, + planner: &FFI_QueryPlanner, +) -> PyResult> { + PyCapsule::new_with_value(py, planner.clone(), cr"datafusion_query_planner") +} + +pub fn ffi_query_planner_from_pycapsule(obj: &Bound) -> PyResult { + let attr_name = "__datafusion_query_planner__"; + let capsule = if obj.hasattr(attr_name)? { + obj.getattr(attr_name)?.call0()? + } else { + obj.clone() + }; + + let capsule = capsule.cast::()?; + validate_pycapsule(capsule, "datafusion_query_planner")?; + let data: NonNull = capsule + .pointer_checked(Some(c"datafusion_query_planner"))? + .cast(); + let planner = unsafe { data.as_ref() }; + let planner_version = unsafe { (planner.version)() }; + let expected_version = datafusion_ffi::version(); + if planner_version != expected_version { + return Err(PyImportError::new_err(format!( + "Incompatible DataFusion query planner version {planner_version}; expected major version {expected_version}." + ))); + } + + Ok(planner.clone()) +} + pub fn create_physical_extension_capsule<'py>( py: Python<'py>, codec: &FFI_PhysicalExtensionCodec, diff --git a/docs/source/contributor-guide/ffi.md b/docs/source/contributor-guide/ffi.md index bf65cad2a..ded7b4295 100644 --- a/docs/source/contributor-guide/ffi.md +++ b/docs/source/contributor-guide/ffi.md @@ -232,6 +232,73 @@ extension that has been written using this approach and the most thoroughly impl As we continue to expose more of the DataFusion features, we intend to follow this same design pattern. +## Query Planners Across Multiple Libraries + +A query can involve three independent native libraries: `datafusion-python`, a library +that owns table providers or functions, and a library that owns the query planner. The +examples use two separate extension crates so each role has a distinct shared-library +identity: + +- [`datafusion-ffi-example`] owns providers, functions, and their codecs. +- [`datafusion-ffi-query-planner-example`] owns the planner and its configuration. + +The `SessionContext` owns the codecs used for the exchange and supplies them to the +foreign planner. This lets the planner decode provider-owned objects and lets +`datafusion-python` decode the physical plan returned by the planner. The examples use +process-local tokens to demonstrate ownership; production codecs should serialize +durable metadata instead. + +The current Python API has one external logical codec and one external physical codec. +Installing another codec replaces the prior codec rather than composing a registry. +The example therefore has one external codec owner, and the planner uses built-in +physical nodes. Install the provider codecs before the planner where possible. + +The current FFI logical codec supports providers and UDFs but not arbitrary custom +`LogicalPlan::Extension` nodes. See both example READMEs for the supported flow and +local build commands. + +### What a derived context shares + +`with_query_planner`, `with_logical_extension_codec`, `with_physical_extension_codec`, +and `with_python_udf_inlining` all return a new `SessionContext` rather than mutating +the receiver. How much the two contexts then share depends on whether a foreign query +planner is involved. + +Without one, the derived context wraps the *same* underlying session, so a registration +on either side is visible to both. + +`with_query_planner` is different, and so is any codec change made on a session that +already has a foreign planner installed. A foreign planner holds the FFI codecs it was +built with, so changing the codecs means rebuilding the planner against the context +that will actually run the query. That forks the session state, and the two halves of +the fork behave differently: + +- **Shared.** Catalogs, schemas, and tables. `SessionState` holds its catalog list + behind an `Arc`, so a table registered on either context is visible to both. The + runtime environment is shared for the same reason. +- **Copied.** Registered scalar, aggregate, and window functions, table functions, the + session configuration, and the analyzer and optimizer rule lists. These are + snapshotted when the derived context is created, so a UDF registered on the original + context afterwards is not visible to the derived one, and a `SET` applied to one does + not reach the other. + +The session id is carried over to the fork, so both contexts report the same id. + +Register functions before deriving, or register them directly on the derived context: + +```python +ctx = SessionContext(config) +ctx = ctx.with_logical_extension_codec(provider_logical_codec) +ctx = ctx.with_physical_extension_codec(provider_physical_codec) +ctx = ctx.with_query_planner(planner) +ctx.register_udf(my_udf) # registered on the context that will run the query +``` + +A session holds exactly one query planner. Calling `with_query_planner` again replaces +the installed planner instead of layering another one. To chain planners, have the new +planner wrap the capsule returned by `SessionContext.__datafusion_query_planner__()` +and delegate to it explicitly. + ## Alternative Approach Suppose you needed to expose some other features of DataFusion and you could not wait @@ -257,3 +324,5 @@ At the time of this writing, the FFI features are under active development. To s the latest status, we recommend reviewing the code in the [datafusion-ffi] crate. [datafusion-ffi]: https://crates.io/crates/datafusion-ffi +[`datafusion-ffi-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example +[`datafusion-ffi-query-planner-example`]: https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-query-planner-example diff --git a/examples/README.md b/examples/README.md index e0e3056d9..7bbb45dcf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -49,6 +49,15 @@ Here is a direct link to the file used in the examples: - [Fan out distinct expressions to a multiprocessing pool](./multiprocessing_pickle_expr.py) - [Distribute expression evaluation across Ray actors](./ray_pickle_expr.py) +### Rust FFI Extensions + +- [Table providers, functions, and codecs](./datafusion-ffi-example/) +- [Independent query planner and planner configuration](./datafusion-ffi-query-planner-example/) + +These two crates form a three-library interoperability example with +`datafusion-python`. They are separate shared libraries so the tests exercise real FFI +type and codec boundaries rather than same-library Rust downcasts. + ### Substrait Support - [Serialize query plans using Substrait](./substrait.py) diff --git a/examples/datafusion-ffi-example/README.md b/examples/datafusion-ffi-example/README.md new file mode 100644 index 000000000..f2945251f --- /dev/null +++ b/examples/datafusion-ffi-example/README.md @@ -0,0 +1,48 @@ + + +# DataFusion Python FFI provider example + +This crate is the **provider library** in the three-library query-planning example. It exports table providers, functions, and the logical and physical codecs needed to serialize objects owned by this library. The companion planner is in [`../datafusion-ffi-query-planner-example`](../datafusion-ffi-query-planner-example/). + +The example intentionally uses separate `cdylib` crates for these roles: + +1. **A — `datafusion-python`:** owns the `SessionContext` and executes the result. +2. **B — this crate:** owns table providers, functions, and provider execution plans. +3. **C — the planner crate:** receives the logical plan and returns a physical plan. + +Separate shared libraries guarantee distinct DataFusion library markers. This catches type-identity mistakes that a planner and provider compiled into one shared library would hide. + +## Codec behavior + +`MyLogicalExtensionCodec` serializes this example's in-memory table providers, and `MyPhysicalExtensionCodec` serializes provider-owned memory scans and opaque FFI wrappers around them. Both use documented, process-local, one-shot token registries. The registries make ownership and callback routing visible without pretending to be a portable format. They assume trusted in-process payloads and consume each token during decoding. A production provider should instead encode durable metadata from which its provider and plans can be reconstructed. + +The example codecs do not inspect the callback `TaskContext`. A production codec that depends on session configuration or registered functions must ensure its exported FFI codec is bound to, and retains, the appropriate host `TaskContextProvider`. + +This example makes the provider library the sole external codec owner. Register both provider codecs before installing the planner: + +```python +ctx = ctx.with_logical_extension_codec(provider_logical_codec) +ctx = ctx.with_physical_extension_codec(provider_physical_codec) +ctx = ctx.with_query_planner(planner) +``` + +Derived contexts also rebind an installed planner when codecs change, so this order is a recommendation rather than a requirement. Planner-last states the ownership flow more clearly. + +For the limits behind that choice — why there is one external codec owner rather than a registry, which node kinds survive the boundary, and what a derived context shares with the context it came from — see [Query Planners Across Multiple Libraries](../../docs/source/contributor-guide/ffi.md#query-planners-across-multiple-libraries) in the contributor guide. diff --git a/examples/datafusion-ffi-example/pyproject.toml b/examples/datafusion-ffi-example/pyproject.toml index 7f85e9487..c51fa8a8d 100644 --- a/examples/datafusion-ffi-example/pyproject.toml +++ b/examples/datafusion-ffi-example/pyproject.toml @@ -21,7 +21,8 @@ build-backend = "maturin" [project] name = "datafusion_ffi_example" -requires-python = ">=3.9" +# Matches the abi3-py310 feature the crate builds against. +requires-python = ">=3.10" classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", diff --git a/examples/datafusion-ffi-example/src/logical_extension_codec.rs b/examples/datafusion-ffi-example/src/logical_extension_codec.rs index 8c3976d37..0474a8d39 100644 --- a/examples/datafusion-ffi-example/src/logical_extension_codec.rs +++ b/examples/datafusion-ffi-example/src/logical_extension_codec.rs @@ -15,11 +15,14 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::collections::HashMap; +use std::fmt; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; use arrow::datatypes::SchemaRef; -use datafusion::common::{Result, TableReference}; +use datafusion::catalog::MemTable; +use datafusion::common::{DataFusionError, Result, TableReference}; use datafusion::datasource::TableProvider; use datafusion::execution::{TaskContext, TaskContextProvider}; use datafusion::logical_expr::{Extension, LogicalPlan, ScalarUDF}; @@ -30,25 +33,49 @@ use datafusion_python_util::get_tokio_runtime; use pyo3::prelude::*; use pyo3::types::PyCapsule; -/// Tracks how often each `try_*_udf` entry point fires. Surface for -/// Python tests to assert the session routed UDF -/// encode/decode through this user-supplied codec rather than the -/// upstream default. +const TABLE_PROVIDER_TOKEN: &[u8] = b"DFPYEXTP"; +static NEXT_TABLE_PROVIDER_ID: AtomicU64 = AtomicU64::new(1); +static TABLE_PROVIDERS: OnceLock>>> = OnceLock::new(); + +fn table_providers() -> &'static Mutex>> { + TABLE_PROVIDERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn token_id(buf: &[u8], prefix: &[u8]) -> Option { + let id: [u8; 8] = buf.strip_prefix(prefix)?.try_into().ok()?; + Some(u64::from_le_bytes(id)) +} + #[derive(Debug, Default)] pub(crate) struct CallCounters { pub encode_udf: AtomicUsize, pub decode_udf: AtomicUsize, + pub encode_table_provider: AtomicUsize, + pub decode_table_provider: AtomicUsize, } -/// Minimal user-supplied `LogicalExtensionCodec` for integration tests. -/// Delegates everything to `DefaultLogicalExtensionCodec` and bumps -/// counters on the UDF entry points so tests can prove the wrapper -/// installed via `SessionContext.with_logical_extension_codec(...)` -/// actually gets consulted. -#[derive(Debug)] +/// Example codec for objects owned by this extension library. +/// +/// The table-provider token registry is intentionally process-local. It is a compact +/// example of preserving Rust type identity across three loaded libraries, not a +/// network serialization format. Production libraries should encode reconstructible +/// provider metadata rather than retaining objects in a global registry. struct CountingLogicalExtensionCodec { inner: DefaultLogicalExtensionCodec, counters: Arc, + // The FFI task-context handle is weak. Retain its provider for as long as + // this codec can be called, even if Python drops the exporter object. + _ctx_provider: Arc, +} + +impl fmt::Debug for CountingLogicalExtensionCodec { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CountingLogicalExtensionCodec") + .field("inner", &self.inner) + .field("counters", &self.counters) + .finish_non_exhaustive() + } } impl LogicalExtensionCodec for CountingLogicalExtensionCodec { @@ -72,6 +99,20 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { schema: SchemaRef, ctx: &TaskContext, ) -> Result> { + if let Some(id) = token_id(buf, TABLE_PROVIDER_TOKEN) { + self.counters + .decode_table_provider + .fetch_add(1, Ordering::SeqCst); + return table_providers() + .lock() + .map_err(|err| DataFusionError::Internal(err.to_string()))? + .remove(&id) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "Unknown datafusion-ffi-example table provider token {id}" + )) + }); + } self.inner .try_decode_table_provider(buf, table_ref, schema, ctx) } @@ -82,6 +123,19 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { node: Arc, buf: &mut Vec, ) -> Result<()> { + if node.downcast_ref::().is_some() { + self.counters + .encode_table_provider + .fetch_add(1, Ordering::SeqCst); + let id = NEXT_TABLE_PROVIDER_ID.fetch_add(1, Ordering::SeqCst); + table_providers() + .lock() + .map_err(|err| DataFusionError::Internal(err.to_string()))? + .insert(id, node); + buf.extend_from_slice(TABLE_PROVIDER_TOKEN); + buf.extend_from_slice(&id.to_le_bytes()); + return Ok(()); + } self.inner.try_encode_table_provider(table_ref, node, buf) } @@ -105,6 +159,7 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { #[derive(Clone)] pub(crate) struct MyLogicalExtensionCodec { counters: Arc, + ctx_provider: Arc, } #[pymethods] @@ -113,26 +168,26 @@ impl MyLogicalExtensionCodec { fn new() -> Self { Self { counters: Arc::new(CallCounters::default()), + ctx_provider: Arc::new(SessionContext::new()), } } - /// Number of `try_encode_udf` invocations observed since - /// construction. fn encode_udf_calls(&self) -> usize { self.counters.encode_udf.load(Ordering::SeqCst) } - /// Number of `try_decode_udf` invocations observed. fn decode_udf_calls(&self) -> usize { self.counters.decode_udf.load(Ordering::SeqCst) } - /// Capsule entry point consumed by - /// `datafusion_python_util::ffi_logical_codec_from_pycapsule`. - /// datafusion-python invokes this with no arguments when the user - /// calls `ctx.with_logical_extension_codec(my_codec)`. The codec - /// owns its own bare `SessionContext` as a TaskContextProvider — - /// good enough for tests that only exercise UDF encode/decode. + fn table_provider_encode_calls(&self) -> usize { + self.counters.encode_table_provider.load(Ordering::SeqCst) + } + + fn table_provider_decode_calls(&self) -> usize { + self.counters.decode_table_provider.load(Ordering::SeqCst) + } + fn __datafusion_logical_extension_codec__<'py>( &self, py: Python<'py>, @@ -140,11 +195,11 @@ impl MyLogicalExtensionCodec { let inner: Arc = Arc::new(CountingLogicalExtensionCodec { inner: DefaultLogicalExtensionCodec {}, counters: Arc::clone(&self.counters), + _ctx_provider: Arc::clone(&self.ctx_provider), }); let runtime = get_tokio_runtime().handle().clone(); - let bare_session: Arc = Arc::new(SessionContext::new()); - let ctx_provider = bare_session as Arc; + let ctx_provider: Arc = self.ctx_provider.clone(); let ffi = FFI_LogicalExtensionCodec::new(inner, Some(runtime), &ctx_provider); PyCapsule::new_with_value(py, ffi, cr"datafusion_logical_extension_codec") diff --git a/examples/datafusion-ffi-example/src/physical_extension_codec.rs b/examples/datafusion-ffi-example/src/physical_extension_codec.rs index 35ef77f6b..d1b9ed63b 100644 --- a/examples/datafusion-ffi-example/src/physical_extension_codec.rs +++ b/examples/datafusion-ffi-example/src/physical_extension_codec.rs @@ -15,14 +15,18 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::collections::HashMap; +use std::fmt; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; -use datafusion::common::Result; +use datafusion::common::{DataFusionError, Result}; +use datafusion::datasource::source::DataSourceExec; use datafusion::execution::{TaskContext, TaskContextProvider}; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; +use datafusion_ffi::execution_plan::ForeignExecutionPlan; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; use datafusion_proto::physical_plan::{ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension, @@ -31,20 +35,48 @@ use datafusion_python_util::get_tokio_runtime; use pyo3::prelude::*; use pyo3::types::PyCapsule; +const EXECUTION_PLAN_TOKEN: &[u8] = b"DFPYEXEP"; +static NEXT_EXECUTION_PLAN_ID: AtomicU64 = AtomicU64::new(1); +static EXECUTION_PLANS: OnceLock>>> = OnceLock::new(); + +fn execution_plans() -> &'static Mutex>> { + EXECUTION_PLANS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn token_id(buf: &[u8]) -> Option { + let id: [u8; 8] = buf.strip_prefix(EXECUTION_PLAN_TOKEN)?.try_into().ok()?; + Some(u64::from_le_bytes(id)) +} + #[derive(Debug, Default)] pub(crate) struct PhysicalCallCounters { pub encode_udf: AtomicUsize, pub decode_udf: AtomicUsize, + pub encode_execution_plan: AtomicUsize, + pub decode_execution_plan: AtomicUsize, } -/// Mirror of [`super::logical_extension_codec::CountingLogicalExtensionCodec`] -/// for the physical layer. Delegates to `DefaultPhysicalExtensionCodec` -/// and bumps counters on UDF encode/decode so tests can prove the -/// session routed through a user-supplied physical codec. -#[derive(Debug)] +/// Physical companion to the logical example codec. +/// +/// Provider-owned memory scan plans use a same-process token registry so the +/// owning cdylib can restore their concrete Rust type after the plan travels +/// through the independent query-planner and datafusion-python libraries. struct CountingPhysicalExtensionCodec { inner: DefaultPhysicalExtensionCodec, counters: Arc, + // The FFI task-context handle is weak. Keep its provider alive with the + // codec rather than relying on the lifetime of the Python exporter. + _ctx_provider: Arc, +} + +impl fmt::Debug for CountingPhysicalExtensionCodec { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CountingPhysicalExtensionCodec") + .field("inner", &self.inner) + .field("counters", &self.counters) + .finish_non_exhaustive() + } } impl PhysicalExtensionCodec for CountingPhysicalExtensionCodec { @@ -55,6 +87,20 @@ impl PhysicalExtensionCodec for CountingPhysicalExtensionCodec { ctx: &TaskContext, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { + if let Some(id) = token_id(buf) { + self.counters + .decode_execution_plan + .fetch_add(1, Ordering::SeqCst); + return execution_plans() + .lock() + .map_err(|err| DataFusionError::Internal(err.to_string()))? + .remove(&id) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "Unknown datafusion-ffi-example execution plan token {id}" + )) + }); + } self.inner.try_decode(buf, inputs, ctx, proto_converter) } @@ -64,6 +110,22 @@ impl PhysicalExtensionCodec for CountingPhysicalExtensionCodec { buf: &mut Vec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { + // The provider owns DataSourceExec. A ForeignExecutionPlan can wrap a + // host-added execution decorator around that scan; retaining the opaque + // wrapper preserves its original library identity without downcasting it. + if node.is::() || node.is::() { + self.counters + .encode_execution_plan + .fetch_add(1, Ordering::SeqCst); + let id = NEXT_EXECUTION_PLAN_ID.fetch_add(1, Ordering::SeqCst); + execution_plans() + .lock() + .map_err(|err| DataFusionError::Internal(err.to_string()))? + .insert(id, node); + buf.extend_from_slice(EXECUTION_PLAN_TOKEN); + buf.extend_from_slice(&id.to_le_bytes()); + return Ok(()); + } self.inner.try_encode(node, buf, proto_converter) } @@ -87,6 +149,7 @@ impl PhysicalExtensionCodec for CountingPhysicalExtensionCodec { #[derive(Clone)] pub(crate) struct MyPhysicalExtensionCodec { counters: Arc, + ctx_provider: Arc, } #[pymethods] @@ -95,6 +158,7 @@ impl MyPhysicalExtensionCodec { fn new() -> Self { Self { counters: Arc::new(PhysicalCallCounters::default()), + ctx_provider: Arc::new(SessionContext::new()), } } @@ -106,6 +170,14 @@ impl MyPhysicalExtensionCodec { self.counters.decode_udf.load(Ordering::SeqCst) } + fn execution_plan_encode_calls(&self) -> usize { + self.counters.encode_execution_plan.load(Ordering::SeqCst) + } + + fn execution_plan_decode_calls(&self) -> usize { + self.counters.decode_execution_plan.load(Ordering::SeqCst) + } + fn __datafusion_physical_extension_codec__<'py>( &self, py: Python<'py>, @@ -114,11 +186,11 @@ impl MyPhysicalExtensionCodec { Arc::new(CountingPhysicalExtensionCodec { inner: DefaultPhysicalExtensionCodec {}, counters: Arc::clone(&self.counters), + _ctx_provider: Arc::clone(&self.ctx_provider), }); let runtime = get_tokio_runtime().handle().clone(); - let bare_session: Arc = Arc::new(SessionContext::new()); - let ctx_provider = bare_session as Arc; + let ctx_provider: Arc = self.ctx_provider.clone(); let ffi = FFI_PhysicalExtensionCodec::new(inner, Some(runtime), &ctx_provider); PyCapsule::new_with_value(py, ffi, cr"datafusion_physical_extension_codec") diff --git a/examples/datafusion-ffi-query-planner-example/Cargo.toml b/examples/datafusion-ffi-query-planner-example/Cargo.toml new file mode 100644 index 000000000..263f034b8 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/Cargo.toml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-ffi-query-planner-example" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description.workspace = true +homepage.workspace = true +repository.workspace = true +publish = false + +[dependencies] +datafusion = { workspace = true } +datafusion-catalog = { workspace = true, default-features = false } +datafusion-common = { workspace = true, default-features = false } +datafusion-ffi = { workspace = true } +datafusion-proto = { workspace = true } +datafusion-session = { workspace = true } +async-trait = { workspace = true } +datafusion-python-util.workspace = true +pyo3 = { workspace = true, features = [ + "extension-module", + "abi3", + "abi3-py310", +] } +pyo3-log = { workspace = true } + +[build-dependencies] +pyo3-build-config = { workspace = true } + +[lib] +name = "datafusion_ffi_query_planner_example" +crate-type = ["cdylib", "rlib"] diff --git a/examples/datafusion-ffi-query-planner-example/README.md b/examples/datafusion-ffi-query-planner-example/README.md new file mode 100644 index 000000000..4d04b7ab8 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/README.md @@ -0,0 +1,60 @@ + + +# DataFusion Python FFI query planner example + +This crate is an independent query-planner Python extension. Together with [`../datafusion-ffi-example`](../datafusion-ffi-example/) it demonstrates a real three-library plan exchange: + +- **A — `datafusion-python`:** owns the session and final execution. +- **B — `datafusion-ffi-example`:** owns a table provider, UDF, and provider codecs. +- **C — this crate:** owns the query planner and its custom configuration. + +Two extension crates are used rather than placing the planner in the provider crate. Loading distinct `cdylib` images gives each library a distinct DataFusion marker and proves that foreign sessions, providers, and plans survive the actual ABI boundary. + +## Running the example + +From the repository root, build and install all three extensions, then run the +integration tests: + +```bash +maturin develop --uv +uv run maturin develop --manifest-path examples/datafusion-ffi-example/Cargo.toml +uv run maturin develop \ + --manifest-path examples/datafusion-ffi-query-planner-example/Cargo.toml +uv run pytest \ + examples/datafusion-ffi-query-planner-example/python/tests/_test*.py +``` + +The integration test follows this setup: + +```python +config = SessionConfig().with_extension(PlannerConfig(max_rows=3)) +ctx = SessionContext(config) +ctx = ctx.with_logical_extension_codec(provider_logical_codec) +ctx = ctx.with_physical_extension_codec(provider_physical_codec) +ctx.register_table("numbers", provider) +ctx.register_udf(provider_udf) +ctx = ctx.with_query_planner(MyQueryPlanner()) +``` + +`PlannerConfig` is transferred through the foreign session. `MyQueryPlanner` reads `ffi_query_planner.max_rows`, creates the plan with `DefaultPhysicalPlanner`, and adds a built-in `GlobalLimitExec`. The test changes the setting with `SET` and verifies the new row limit. + +The provider's codec pair is attached to the planner when the derived context is created and is also used to decode the returned physical plan in `datafusion-python`. This planner deliberately uses only built-in physical nodes. Install the codecs before the planner where possible; derived contexts rebind codecs after planner installation, but planner-last order is easier to audit. + +For the limits behind that choice — why there is one external codec owner rather than a registry, which node kinds survive the boundary, and what a derived context shares with the context it came from — see [Query Planners Across Multiple Libraries](../../docs/source/contributor-guide/ffi.md#query-planners-across-multiple-libraries) in the contributor guide. diff --git a/examples/datafusion-ffi-query-planner-example/build.rs b/examples/datafusion-ffi-query-planner-example/build.rs new file mode 100644 index 000000000..4878d8b0e --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/build.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +fn main() { + pyo3_build_config::add_extension_module_link_args(); +} diff --git a/examples/datafusion-ffi-query-planner-example/pyproject.toml b/examples/datafusion-ffi-query-planner-example/pyproject.toml new file mode 100644 index 000000000..9e34b4cd4 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/pyproject.toml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["maturin>=1.6,<2.0"] +build-backend = "maturin" + +[project] +name = "datafusion_ffi_query_planner_example" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", +] +dynamic = ["version"] + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py b/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py new file mode 100644 index 000000000..15f46bf90 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import gc + +import pytest +from datafusion import SessionConfig, SessionContext, udf +from datafusion_ffi_example import ( + IsNullUDF, + MyLogicalExtensionCodec, + MyPhysicalExtensionCodec, + MyTableProvider, +) +from datafusion_ffi_query_planner_example import MyQueryPlanner, PlannerConfig + + +def configured_context(max_rows: int): + config = SessionConfig().with_extension(PlannerConfig(max_rows=max_rows)) + logical_codec = MyLogicalExtensionCodec() + physical_codec = MyPhysicalExtensionCodec() + ctx = SessionContext(config) + ctx = ctx.with_logical_extension_codec(logical_codec) + ctx = ctx.with_physical_extension_codec(physical_codec) + ctx.register_table("numbers", MyTableProvider(1, 6, 1)) + ctx.register_udf(udf(IsNullUDF())) + return ctx, logical_codec, physical_codec + + +@pytest.mark.parametrize("raw_capsule", [False, True]) +def test_three_library_query_planner(raw_capsule: bool): + """Host, provider, and planner exchange a real non-empty plan over FFI.""" + ctx, logical_codec, physical_codec = configured_context(max_rows=3) + planner = MyQueryPlanner() + exported_planner = ( + planner.__datafusion_query_planner__() if raw_capsule else planner + ) + ctx = ctx.with_query_planner(exported_planner) + + batches = ctx.sql( + 'SELECT "A", my_custom_is_null("A") AS is_null FROM numbers ORDER BY "A"' + ).collect() + assert batches[0].column(0).to_pylist() == [0, 1, 2] + assert batches[0].column(1).to_pylist() == [False, False, False] + assert planner.last_max_rows() == 3 + + ctx.sql("SET ffi_query_planner.max_rows = 2").collect() + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1] + assert planner.last_max_rows() == 2 + + assert planner.plan_calls() >= 2 + assert planner.foreign_session_observed() + assert planner.foreign_provider_observed() + assert planner.foreign_plan_observed() + assert logical_codec.table_provider_encode_calls() > 0 + assert logical_codec.table_provider_decode_calls() > 0 + assert physical_codec.execution_plan_encode_calls() > 0 + assert physical_codec.execution_plan_decode_calls() > 0 + + +def test_spawning_plan_across_three_libraries(): + """A plan that spawns Tokio tasks survives the full three-library round trip. + + ``target_partitions`` above one puts a ``RepartitionExec`` under the + aggregate, and that operator spawns tasks while it runs. This exercises the + codecs on a multi-node plan rather than the bare scan the other tests use. + """ + config = SessionConfig().with_extension(PlannerConfig(max_rows=100)) + config = config.with_target_partitions(4) + logical_codec = MyLogicalExtensionCodec() + physical_codec = MyPhysicalExtensionCodec() + ctx = SessionContext(config) + ctx = ctx.with_logical_extension_codec(logical_codec) + ctx = ctx.with_physical_extension_codec(physical_codec) + ctx.register_table("numbers", MyTableProvider(1, 6, 3)) + + planner = MyQueryPlanner() + ctx = ctx.with_query_planner(planner) + + batches = ctx.sql( + 'SELECT "A" % 2 AS parity, count(*) AS n FROM numbers GROUP BY 1 ORDER BY 1' + ).collect() + counts = { + row[0]: row[1] + for batch in batches + for row in zip( + batch.column(0).to_pylist(), batch.column(1).to_pylist(), strict=True + ) + } + assert sum(counts.values()) == 6 + 7 + 8 + assert planner.plan_calls() > 0 + assert planner.foreign_provider_observed() + + +def test_planner_layers_on_the_session_planner(): + """A planner can wrap the one already installed and delegate to it. + + The capsule has to be captured before this planner is installed, because + ``__datafusion_query_planner__`` exports whichever planner is installed when + it is called. Capturing it afterwards would hand the planner a handle to + itself, and planning would recurse. + """ + ctx, logical_codec, physical_codec = configured_context(max_rows=3) + fallback = ctx.__datafusion_query_planner__() + planner = MyQueryPlanner(fallback=fallback) + # Rebinding `ctx` drops the context that produced the capsule. The exported + # codecs retain it, so the capsule stays usable. Without that the FFI + # task-context handle is weak and planning fails with "TaskContextProvider + # went out of scope over FFI boundary". + ctx = ctx.with_query_planner(planner) + gc.collect() + + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1, 2] + assert planner.plan_calls() > 0 + assert planner.used_fallback() + assert logical_codec.table_provider_decode_calls() > 0 + assert physical_codec.execution_plan_decode_calls() > 0 + + +def test_second_planner_replaces_the_first(): + """A session holds exactly one planner, so installing another replaces it.""" + ctx, _logical_codec, _physical_codec = configured_context(max_rows=2) + first = MyQueryPlanner() + second = MyQueryPlanner() + ctx = ctx.with_query_planner(first).with_query_planner(second) + + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1] + assert second.plan_calls() > 0 + assert first.plan_calls() == 0 + + +def test_planner_is_not_installed_on_the_original_context(): + """``with_query_planner`` returns a fork; the receiver keeps its planner.""" + ctx, _logical_codec, _physical_codec = configured_context(max_rows=2) + planner = MyQueryPlanner() + derived = ctx.with_query_planner(planner) + + ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert planner.plan_calls() == 0 + + derived.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert planner.plan_calls() > 0 + + +def test_installed_codecs_outlive_python_exporters(): + ctx, logical_codec, physical_codec = configured_context(max_rows=2) + del logical_codec, physical_codec + gc.collect() + + ctx = ctx.with_query_planner(MyQueryPlanner()) + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1] + + +def test_provider_codecs_can_be_installed_after_planner(): + config = SessionConfig().with_extension(PlannerConfig(max_rows=2)) + planner = MyQueryPlanner() + logical_codec = MyLogicalExtensionCodec() + physical_codec = MyPhysicalExtensionCodec() + ctx = SessionContext(config).with_query_planner(planner) + ctx = ctx.with_logical_extension_codec(logical_codec) + ctx = ctx.with_physical_extension_codec(physical_codec) + ctx.register_table("numbers", MyTableProvider(1, 4, 1)) + + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1] + assert planner.last_max_rows() == 2 + assert logical_codec.table_provider_decode_calls() > 0 + assert physical_codec.execution_plan_decode_calls() > 0 + + +def test_query_planner_requires_provider_codec(): + config = SessionConfig().with_extension(PlannerConfig(max_rows=2)) + ctx = SessionContext(config) + ctx.register_table("numbers", MyTableProvider(1, 3, 1)) + ctx = ctx.with_query_planner(MyQueryPlanner()) + + with pytest.raises(Exception, match=r"LogicalExtensionCodec|TableProvider"): + ctx.sql('SELECT "A" FROM numbers').collect() + + +@pytest.mark.parametrize("max_rows", ["0", "oops"]) +def test_query_planner_rejects_invalid_config(max_rows: str): + ctx, _logical_codec, _physical_codec = configured_context(max_rows=2) + ctx = ctx.with_query_planner(MyQueryPlanner()) + + with pytest.raises(Exception, match=r"max_rows|Invalid value"): + ctx.sql(f"SET ffi_query_planner.max_rows = '{max_rows}'").collect() diff --git a/examples/datafusion-ffi-query-planner-example/src/config.rs b/examples/datafusion-ffi-query-planner-example/src/config.rs new file mode 100644 index 000000000..801cee9a0 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/src/config.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use datafusion_common::config::{ + ConfigEntry, ConfigExtension, ConfigField, ExtensionOptions, Visit, +}; +use datafusion_common::{DataFusionError, config_err}; +use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use pyo3::types::PyCapsule; + +#[pyclass( + from_py_object, + name = "PlannerConfig", + module = "datafusion_ffi_query_planner_example", + subclass +)] +#[derive(Clone, Debug)] +pub(crate) struct PlannerConfig { + pub max_rows: usize, +} + +#[pymethods] +impl PlannerConfig { + #[new] + #[pyo3(signature = (max_rows=10))] + fn new(max_rows: usize) -> Self { + Self { max_rows } + } + + fn __datafusion_extension_options__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let mut config = FFI_ExtensionOptions::default(); + config + .add_config(self) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + PyCapsule::new_with_value(py, config, cr"datafusion_extension_options") + } +} + +impl Default for PlannerConfig { + fn default() -> Self { + Self { max_rows: 10 } + } +} + +impl ConfigExtension for PlannerConfig { + const PREFIX: &'static str = "ffi_query_planner"; +} + +impl ExtensionOptions for PlannerConfig { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn cloned(&self) -> Box { + Box::new(self.clone()) + } + + fn set(&mut self, key: &str, value: &str) -> datafusion_common::Result<()> { + ConfigField::set(self, key, value) + } + + fn entries(&self) -> Vec { + vec![ConfigEntry { + key: "max_rows".to_owned(), + value: Some(self.max_rows.to_string()), + description: "Maximum rows returned by the example query planner", + }] + } +} + +impl ConfigField for PlannerConfig { + fn visit(&self, visitor: &mut V, _key: &str, _description: &'static str) { + self.max_rows.visit( + visitor, + "max_rows", + "Maximum rows returned by the example query planner", + ); + } + + fn set(&mut self, key: &str, value: &str) -> Result<(), DataFusionError> { + let (key, rem) = key.split_once('.').unwrap_or((key, "")); + match key { + "max_rows" => self.max_rows.set(rem, value), + _ => config_err!("Config value '{key}' not found on PlannerConfig"), + } + } +} diff --git a/examples/datafusion-ffi-query-planner-example/src/lib.rs b/examples/datafusion-ffi-query-planner-example/src/lib.rs new file mode 100644 index 000000000..7635c2992 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/src/lib.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use pyo3::prelude::*; + +use crate::config::PlannerConfig; +use crate::planner::MyQueryPlanner; + +mod config; +mod planner; + +#[pymodule] +fn datafusion_ffi_query_planner_example(m: &Bound<'_, PyModule>) -> PyResult<()> { + pyo3_log::init(); + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/examples/datafusion-ffi-query-planner-example/src/planner.rs b/examples/datafusion-ffi-query-planner-example/src/planner.rs new file mode 100644 index 000000000..59791f160 --- /dev/null +++ b/examples/datafusion-ffi-query-planner-example/src/planner.rs @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use async_trait::async_trait; +use datafusion::common::DataFusionError; +use datafusion::execution::TaskContextProvider; +use datafusion::execution::context::SessionContext; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::limit::GlobalLimitExec; +use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; +use datafusion_catalog::default_table_source::source_as_provider; +use datafusion_ffi::config::ExtensionOptionsFFIProvider; +use datafusion_ffi::execution_plan::ForeignExecutionPlan; +use datafusion_ffi::query_planner::FFI_QueryPlanner; +use datafusion_ffi::session::ForeignSession; +use datafusion_ffi::table_provider::ForeignTableProvider; +use datafusion_proto::logical_plan::DefaultLogicalExtensionCodec; +use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec; +use datafusion_python_util::{ffi_query_planner_from_pycapsule, get_tokio_runtime}; +use datafusion_session::{QueryPlanner, Session}; +use pyo3::prelude::*; +use pyo3::types::PyCapsule; + +use crate::config::PlannerConfig; + +#[derive(Default)] +struct PlannerObservations { + plan_calls: AtomicUsize, + last_max_rows: AtomicUsize, + foreign_session: AtomicBool, + foreign_provider: AtomicBool, + foreign_plan: AtomicBool, + used_fallback: AtomicBool, +} + +impl fmt::Debug for PlannerObservations { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("PlannerObservations") + .field("plan_calls", &self.plan_calls) + .field("last_max_rows", &self.last_max_rows) + .finish_non_exhaustive() + } +} + +fn logical_plan_has_foreign_provider(plan: &LogicalPlan) -> bool { + if let LogicalPlan::TableScan(scan) = plan + && let Ok(provider) = source_as_provider(&scan.source) + && provider.downcast_ref::().is_some() + { + return true; + } + plan.inputs() + .iter() + .any(|input| logical_plan_has_foreign_provider(input)) +} + +fn physical_plan_has_foreign_plan(plan: &Arc) -> bool { + plan.is::() + || plan + .children() + .iter() + .any(|child| physical_plan_has_foreign_plan(child)) +} + +/// The row limit as the host spells it, where `PlannerConfig` is registered as +/// an ordinary config extension under its own `ConfigExtension::PREFIX`. +const MAX_ROWS_KEY: &str = "ffi_query_planner.max_rows"; + +/// The same setting as it appears once the session has crossed the FFI +/// boundary. Rebuilding a `ConfigOptions` on this side parks every foreign +/// extension inside a single `FFI_ExtensionOptions`, which is itself a config +/// extension namespaced under `datafusion_ffi`, so `ConfigOptions::entries` +/// reports the key with both prefixes. +const FFI_MAX_ROWS_KEY: &str = "datafusion_ffi.ffi_query_planner.max_rows"; + +fn planner_config(session: &dyn Session) -> datafusion::common::Result { + let options = session.config_options(); + + // Prefer the raw entry. `local_or_ffi_extension` discards a value it cannot + // parse and hands back `PlannerConfig::default()`, which would quietly turn + // a typo into a different row limit instead of reporting it. + let config = match options + .entries() + .into_iter() + .find(|entry| entry.key == MAX_ROWS_KEY || entry.key == FFI_MAX_ROWS_KEY) + { + Some(entry) => { + let value = entry.value.ok_or_else(|| { + DataFusionError::Configuration(format!("{} must have a value", entry.key)) + })?; + let max_rows = value.parse::().map_err(|err| { + DataFusionError::Configuration(format!( + "Invalid value '{value}' for {}: {err}", + entry.key + )) + })?; + PlannerConfig { max_rows } + } + None => options + .local_or_ffi_extension::() + .unwrap_or_default(), + }; + + // Validate after both paths so the fallback cannot smuggle in a limit that + // the direct path rejects. + if config.max_rows == 0 { + return Err(DataFusionError::Configuration(format!( + "{MAX_ROWS_KEY} must be greater than zero" + ))); + } + + Ok(config) +} + +#[derive(Debug)] +struct DistributedQueryPlanner { + observations: Arc, + /// Planner to hand the work to instead of planning here. + /// + /// This is how a real planner layers on top of an existing one. The capsule + /// must be captured from the session *before* this planner is installed: + /// `SessionContext.__datafusion_query_planner__` exports whatever planner + /// is installed at the time it is called, so capturing it afterwards would + /// hand this planner a handle to itself. + /// + /// Note that `Session::create_physical_plan` cannot be used for this. It + /// dispatches through the session's installed query planner, so calling it + /// from inside that planner recurses until the stack overflows. + fallback: Option>, +} + +#[async_trait] +impl QueryPlanner for DistributedQueryPlanner { + async fn create_physical_plan( + &self, + logical_plan: &LogicalPlan, + session: &dyn Session, + ) -> datafusion::common::Result> { + self.observations.plan_calls.fetch_add(1, Ordering::SeqCst); + self.observations + .foreign_session + .store(session.as_any().is::(), Ordering::SeqCst); + self.observations.foreign_provider.store( + logical_plan_has_foreign_provider(logical_plan), + Ordering::SeqCst, + ); + + let config = planner_config(session)?; + self.observations + .last_max_rows + .store(config.max_rows, Ordering::SeqCst); + + let plan = match self.fallback.as_ref() { + Some(fallback) => { + self.observations + .used_fallback + .store(true, Ordering::SeqCst); + fallback.create_physical_plan(logical_plan, session).await? + } + None => { + DefaultPhysicalPlanner::default() + .create_physical_plan(logical_plan, session) + .await? + } + }; + self.observations + .foreign_plan + .store(physical_plan_has_foreign_plan(&plan), Ordering::SeqCst); + + Ok(Arc::new(GlobalLimitExec::new( + plan, + 0, + Some(config.max_rows), + ))) + } +} + +#[pyclass( + from_py_object, + name = "MyQueryPlanner", + module = "datafusion_ffi_query_planner_example", + subclass +)] +#[derive(Debug, Default, Clone)] +pub(crate) struct MyQueryPlanner { + observations: Arc, + fallback: Option>, +} + +#[pymethods] +impl MyQueryPlanner { + /// Build a planner, optionally layered on top of an existing one. + /// + /// `fallback` takes anything exporting `__datafusion_query_planner__`, + /// including a `SessionContext`. Capture it *before* installing this + /// planner on that context, or the capsule will describe this planner and + /// planning will recurse. + #[new] + #[pyo3(signature = (fallback=None))] + fn new(fallback: Option>) -> PyResult { + let fallback = fallback + .map(|planner| { + ffi_query_planner_from_pycapsule(&planner) + .map(|ffi| -> Arc { (&ffi).into() }) + }) + .transpose()?; + Ok(Self { + fallback, + ..Self::default() + }) + } + + fn used_fallback(&self) -> bool { + self.observations.used_fallback.load(Ordering::SeqCst) + } + + fn plan_calls(&self) -> usize { + self.observations.plan_calls.load(Ordering::SeqCst) + } + + fn last_max_rows(&self) -> usize { + self.observations.last_max_rows.load(Ordering::SeqCst) + } + + fn foreign_session_observed(&self) -> bool { + self.observations.foreign_session.load(Ordering::SeqCst) + } + + fn foreign_provider_observed(&self) -> bool { + self.observations.foreign_provider.load(Ordering::SeqCst) + } + + fn foreign_plan_observed(&self) -> bool { + self.observations.foreign_plan.load(Ordering::SeqCst) + } + + fn __datafusion_query_planner__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let planner: Arc = Arc::new(DistributedQueryPlanner { + observations: Arc::clone(&self.observations), + fallback: self.fallback.clone(), + }); + let runtime = get_tokio_runtime().handle().clone(); + let ctx_provider = Arc::new(SessionContext::new()) as Arc; + let ffi = FFI_QueryPlanner::new( + planner, + Some(runtime), + &ctx_provider, + Arc::new(DefaultLogicalExtensionCodec {}), + Arc::new(DefaultPhysicalExtensionCodec {}), + ); + PyCapsule::new_with_value(py, ffi, cr"datafusion_query_planner") + } +} diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 94b2bb1c6..b1f4f74f0 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -145,6 +145,16 @@ class PhysicalOptimizerRuleExportable(Protocol): def __datafusion_physical_optimizer_rule__(self) -> object: ... # noqa: D105 +class QueryPlannerExportable(Protocol): + """Type hint for object that has a __datafusion_query_planner__ PyCapsule. + + The method returns a PyCapsule wrapping an ``FFI_QueryPlanner``, typically + produced by a separate compiled extension. + """ + + def __datafusion_query_planner__(self) -> object: ... # noqa: D105 + + class SessionConfig: """Session configuration options.""" @@ -1759,6 +1769,50 @@ def add_physical_optimizer_rule( """ self.ctx.add_physical_optimizer_rule(rule) + def with_query_planner( + self, planner: QueryPlannerExportable | _PyCapsule + ) -> SessionContext: + """Create a new session context with a custom query planner. + + The planner is imported through its ``__datafusion_query_planner__`` + PyCapsule. The returned context carries over the current session state + and the logical and physical extension codec settings. Codec changes + made on a derived context are rebound to the planner before planning. + + A session holds exactly one planner, so calling this again replaces the + previous one rather than layering. To chain planners, have the new + planner wrap the capsule from + :meth:`~SessionContext.__datafusion_query_planner__`. + + .. note:: Derived contexts share catalogs, not registrations + The returned context is a fork. Catalogs, schemas, and tables stay + shared with the original context, but registered functions and + configuration are copied at the time of the call. A UDF registered + on the original context afterwards is **not** visible here, while a + table registered on either context is visible to both. Register + functions before deriving, or register them on the derived context. + + Args: + planner: Object exposing ``__datafusion_query_planner__`` (see + :class:`QueryPlannerExportable`) or a raw + ``datafusion_query_planner`` PyCapsule. + + Returns: + A new context that uses the specified query planner. + + Examples: + >>> from my_extension import DistributedQueryPlanner # doctest: +SKIP + >>> ctx = SessionContext() + >>> planner = DistributedQueryPlanner() # doctest: +SKIP + >>> planner_ctx = ctx.with_query_planner(planner) # doctest: +SKIP + >>> query = planner_ctx.sql("SELECT * FROM remote_table") # doctest: +SKIP + >>> query.collect() # doctest: +SKIP + """ + new_internal = self.ctx.with_query_planner(planner) + new = SessionContext.__new__(SessionContext) + new.ctx = new_internal + return new + def table_provider(self, name: str) -> Table: """Return the :py:class:`~datafusion.catalog.Table` for the given table name. @@ -2182,6 +2236,10 @@ def __datafusion_logical_extension_codec__(self) -> Any: """Access the PyCapsule FFI_LogicalExtensionCodec.""" return self.ctx.__datafusion_logical_extension_codec__() + def __datafusion_query_planner__(self) -> Any: + """Access the ``FFI_QueryPlanner`` PyCapsule for the current planner.""" + return self.ctx.__datafusion_query_planner__() + def with_logical_extension_codec( self, codec: LogicalExtensionCodecExportable | _PyCapsule ) -> SessionContext: @@ -2190,6 +2248,12 @@ def with_logical_extension_codec( Only FFI codecs are supported. Pass any object implementing ``__datafusion_logical_extension_codec__`` (see :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`). + + The returned context shares its session state with the original, so a + later registration on either is visible to both. The exception is a + session with a custom query planner installed: that planner has to be + rebound to the new codec, which forks the state. See + :meth:`~SessionContext.with_query_planner` for what a fork shares. """ new_internal = self.ctx.with_logical_extension_codec(codec) new = SessionContext.__new__(SessionContext) @@ -2208,6 +2272,12 @@ def with_physical_extension_codec( Only FFI codecs are supported. Pass any object implementing ``__datafusion_physical_extension_codec__`` (see :py:class:`~datafusion.user_defined.PhysicalExtensionCodecExportable`). + + The returned context shares its session state with the original, so a + later registration on either is visible to both. The exception is a + session with a custom query planner installed: that planner has to be + rebound to the new codec, which forks the state. See + :meth:`~SessionContext.with_query_planner` for what a fork shares. """ new_internal = self.ctx.with_physical_extension_codec(codec) new = SessionContext.__new__(SessionContext) @@ -2250,7 +2320,13 @@ def with_python_udf_inlining(self, *, enabled: bool) -> SessionContext: regardless of the toggle. Returns a new :class:`SessionContext` with the toggle applied; - the original session is unchanged. + the original session is unchanged. The returned context shares + its session state with the original, so a later registration on + either is visible to both. The exception is a session with a + custom query planner installed: that planner has to be rebound + to the new codecs, which forks the state. See + :meth:`~SessionContext.with_query_planner` for what a fork + shares. Examples: >>> import pyarrow as pa diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 7d038c7a5..5ceb14e0e 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import ctypes import datetime as dt import gzip import pathlib @@ -731,6 +732,80 @@ def test_remove_optimizer_rule(ctx): assert ctx.remove_optimizer_rule("nonexistent_rule") is False +def test_with_query_planner_rejects_wrong_capsule(ctx): + with pytest.raises(ValueError, match="datafusion_query_planner"): + ctx.with_query_planner(ctx.__datafusion_task_context_provider__()) + + +def test_with_query_planner_capsule(ctx): + capsule = ctx.__datafusion_query_planner__() + get_name = ctypes.pythonapi.PyCapsule_GetName + get_name.argtypes = [ctypes.py_object] + get_name.restype = ctypes.c_char_p + assert get_name(capsule) == b"datafusion_query_planner" + + ctx.register_record_batches( + "query_planner_test", + [[pa.RecordBatch.from_pydict({"value": [1, 2, 3]})]], + ) + planner_context = ctx.with_query_planner(capsule) + assert planner_context.table_exist("query_planner_test") + batches = planner_context.sql("SELECT 1 AS value").collect() + assert batches[0].column(0) == pa.array([1]) + + +def test_derived_context_shares_catalogs(ctx): + """Catalogs live behind an Arc, so tables cross the fork in both directions.""" + derived = ctx.with_query_planner(ctx.__datafusion_query_planner__()) + + ctx.register_record_batches( + "registered_on_parent", + [[pa.RecordBatch.from_pydict({"value": [1]})]], + ) + derived.register_record_batches( + "registered_on_derived", + [[pa.RecordBatch.from_pydict({"value": [2]})]], + ) + + assert derived.table_exist("registered_on_parent") + assert ctx.table_exist("registered_on_derived") + + +def test_derived_context_snapshots_functions(ctx): + """Function registries are copied at fork time, unlike catalogs. + + A UDF registered on the parent before the fork is carried over; one + registered afterwards is not. Guards the caveat documented on + ``SessionContext.with_query_planner``. + """ + before = udf( + lambda arr: arr, + [pa.int64()], + pa.int64(), + volatility="immutable", + name="registered_before_fork", + ) + ctx.register_udf(before) + + derived = ctx.with_query_planner(ctx.__datafusion_query_planner__()) + + after = udf( + lambda arr: arr, + [pa.int64()], + pa.int64(), + volatility="immutable", + name="registered_after_fork", + ) + ctx.register_udf(after) + + assert derived.sql("SELECT registered_before_fork(1)").collect() + with pytest.raises(Exception, match="registered_after_fork"): + derived.sql("SELECT registered_after_fork(1)").collect() + + # The parent is unaffected by the fork. + assert ctx.sql("SELECT registered_after_fork(1)").collect() + + def test_table_provider(ctx): batch = pa.RecordBatch.from_pydict({"x": [10, 20, 30]}) ctx.register_record_batches("provider_test", [[batch]])