From 12d15aa08c469efb2348523d5c51613226852772 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 15 Sep 2026 10:33:00 +0200 Subject: [PATCH] chore: remove migrated DataFusion integration crates --- Cargo.lock | 1771 ++--------------- Cargo.toml | 19 - README.md | 7 - bindings/python/DEPENDENCIES.rust.tsv | 1 - crates/integrations/datafusion/Cargo.toml | 49 - .../datafusion/DEPENDENCIES.rust.tsv | 412 ---- crates/integrations/datafusion/LICENSE | 1 - crates/integrations/datafusion/NOTICE | 1 - crates/integrations/datafusion/README.md | 22 - crates/integrations/datafusion/public-api.txt | 139 -- crates/integrations/datafusion/src/catalog.rs | 93 - crates/integrations/datafusion/src/error.rs | 32 - crates/integrations/datafusion/src/lib.rs | 30 - .../datafusion/src/physical_plan/commit.rs | 713 ------- .../src/physical_plan/expr_to_predicate.rs | 927 --------- .../src/physical_plan/metadata_scan.rs | 98 - .../datafusion/src/physical_plan/mod.rs | 31 - .../datafusion/src/physical_plan/project.rs | 590 ------ .../src/physical_plan/repartition.rs | 853 -------- .../datafusion/src/physical_plan/scan.rs | 252 --- .../datafusion/src/physical_plan/sort.rs | 240 --- .../datafusion/src/physical_plan/write.rs | 653 ------ crates/integrations/datafusion/src/schema.rs | 447 ----- .../datafusion/src/table/metadata_table.rs | 84 - .../integrations/datafusion/src/table/mod.rs | 893 --------- .../src/table/table_provider_factory.rs | 331 --- .../datafusion/src/task_writer.rs | 517 ----- .../table_metadata/TableMetadataV2.json | 121 -- .../tests/integration_datafusion_test.rs | 948 --------- .../tests/test_data/TableMetadataV2Valid.json | 122 -- crates/integrations/playground/Cargo.toml | 53 - .../playground/DEPENDENCIES.rust.tsv | 517 ----- crates/integrations/playground/README.md | 25 - crates/integrations/playground/src/catalog.rs | 204 -- crates/integrations/playground/src/lib.rs | 22 - crates/integrations/playground/src/main.rs | 112 -- crates/sqllogictest/Cargo.toml | 58 - crates/sqllogictest/DEPENDENCIES.rust.tsv | 481 ----- crates/sqllogictest/README.md | 35 - crates/sqllogictest/src/engine/datafusion.rs | 204 -- crates/sqllogictest/src/engine/mod.rs | 143 -- crates/sqllogictest/src/error.rs | 52 - crates/sqllogictest/src/lib.rs | 23 - crates/sqllogictest/src/schedule.rs | 255 --- .../testdata/schedules/df_basic_queries.toml | 23 - .../df_binary_predicate_pushdown.toml | 23 - .../df_boolean_predicate_pushdown.toml | 23 - .../testdata/schedules/df_create_table.toml | 23 - .../testdata/schedules/df_drop_table.toml | 23 - .../df_encrypted_insert_round_trip.toml | 23 - .../schedules/df_insert_and_like.toml | 27 - .../testdata/schedules/df_scan_all_types.toml | 23 - .../testdata/schedules/df_show_tables.toml | 23 - .../df_timestamp_predicate_pushdown.toml | 23 - .../testdata/slts/df_test/basic_queries.slt | 224 --- .../df_test/binary_predicate_pushdown.slt | 67 - .../df_test/boolean_predicate_pushdown.slt | 124 -- .../testdata/slts/df_test/create_table.slt | 90 - .../testdata/slts/df_test/drop_table.slt | 63 - .../df_test/encrypted_insert_round_trip.slt | 31 - .../testdata/slts/df_test/insert_into.slt | 153 -- .../slts/df_test/like_predicate_pushdown.slt | 109 - .../testdata/slts/df_test/scan_all_types.slt | 128 -- .../testdata/slts/df_test/show_tables.slt | 46 - .../df_test/timestamp_predicate_pushdown.slt | 183 -- crates/sqllogictest/tests/sqllogictests.rs | 87 - 66 files changed, 143 insertions(+), 13977 deletions(-) delete mode 100644 crates/integrations/datafusion/Cargo.toml delete mode 100644 crates/integrations/datafusion/DEPENDENCIES.rust.tsv delete mode 120000 crates/integrations/datafusion/LICENSE delete mode 120000 crates/integrations/datafusion/NOTICE delete mode 100644 crates/integrations/datafusion/README.md delete mode 100644 crates/integrations/datafusion/public-api.txt delete mode 100644 crates/integrations/datafusion/src/catalog.rs delete mode 100644 crates/integrations/datafusion/src/error.rs delete mode 100644 crates/integrations/datafusion/src/lib.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/commit.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/metadata_scan.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/mod.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/project.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/repartition.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/scan.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/sort.rs delete mode 100644 crates/integrations/datafusion/src/physical_plan/write.rs delete mode 100644 crates/integrations/datafusion/src/schema.rs delete mode 100644 crates/integrations/datafusion/src/table/metadata_table.rs delete mode 100644 crates/integrations/datafusion/src/table/mod.rs delete mode 100644 crates/integrations/datafusion/src/table/table_provider_factory.rs delete mode 100644 crates/integrations/datafusion/src/task_writer.rs delete mode 100644 crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json delete mode 100644 crates/integrations/datafusion/tests/integration_datafusion_test.rs delete mode 100644 crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json delete mode 100644 crates/integrations/playground/Cargo.toml delete mode 100644 crates/integrations/playground/DEPENDENCIES.rust.tsv delete mode 100644 crates/integrations/playground/README.md delete mode 100644 crates/integrations/playground/src/catalog.rs delete mode 100644 crates/integrations/playground/src/lib.rs delete mode 100644 crates/integrations/playground/src/main.rs delete mode 100644 crates/sqllogictest/Cargo.toml delete mode 100644 crates/sqllogictest/DEPENDENCIES.rust.tsv delete mode 100644 crates/sqllogictest/README.md delete mode 100644 crates/sqllogictest/src/engine/datafusion.rs delete mode 100644 crates/sqllogictest/src/engine/mod.rs delete mode 100644 crates/sqllogictest/src/error.rs delete mode 100644 crates/sqllogictest/src/lib.rs delete mode 100644 crates/sqllogictest/src/schedule.rs delete mode 100644 crates/sqllogictest/testdata/schedules/df_basic_queries.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_create_table.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_drop_table.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_insert_and_like.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_scan_all_types.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_show_tables.toml delete mode 100644 crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml delete mode 100644 crates/sqllogictest/testdata/slts/df_test/basic_queries.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/create_table.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/drop_table.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/insert_into.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/show_tables.slt delete mode 100644 crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt delete mode 100644 crates/sqllogictest/tests/sqllogictests.rs diff --git a/Cargo.lock b/Cargo.lock index 010f8985eb..d739b0ce52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,7 +174,7 @@ dependencies = [ "serde_json", "snap", "strum", - "strum_macros 0.27.2", + "strum_macros", "thiserror 2.0.18", "uuid", "zstd", @@ -189,15 +189,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ar_archive_writer" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" -dependencies = [ - "object", -] - [[package]] name = "arc-swap" version = "1.9.2" @@ -281,30 +272,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-avro" -version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", - "bytes", - "bzip2", - "crc", - "flate2", - "indexmap 2.14.0", - "liblzma", - "rand 0.9.5", - "serde", - "serde_json", - "snap", - "strum_macros 0.28.0", - "uuid", - "zstd", -] - [[package]] name = "arrow-buffer" version = "59.2.0" @@ -332,7 +299,6 @@ dependencies = [ "atoi", "base64 0.23.0", "chrono", - "comfy-table", "half", "lexical-core", "num-traits", @@ -379,8 +345,6 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", - "lz4_flex 0.14.0", - "zstd", ] [[package]] @@ -453,8 +417,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "bitflags 2.13.0", - "serde_core", - "serde_json", ] [[package]] @@ -516,18 +478,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-compression" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" -dependencies = [ - "compression-codecs", - "compression-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-lock" version = "3.4.2" @@ -547,7 +497,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -558,7 +508,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -989,7 +939,7 @@ checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1138,15 +1088,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "blake3" version = "1.8.5" @@ -1220,7 +1161,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.119", + "syn", ] [[package]] @@ -1287,7 +1228,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1315,15 +1256,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" -dependencies = [ - "libbz2-rs-sys", -] - [[package]] name = "cbc" version = "0.1.2" @@ -1439,7 +1371,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1448,15 +1380,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "clipboard-win" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" -dependencies = [ - "error-code", -] - [[package]] name = "cmake" version = "0.1.58" @@ -1497,37 +1420,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "comfy-table" -version = "7.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" -dependencies = [ - "unicode-segmentation", - "unicode-width 0.2.2", -] - -[[package]] -name = "compression-codecs" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" -dependencies = [ - "bzip2", - "compression-core", - "flate2", - "liblzma", - "memchr", - "zstd", - "zstd-safe", -] - -[[package]] -name = "compression-core" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1537,18 +1429,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "console" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" -dependencies = [ - "encode_unicode", - "libc", - "unicode-width 0.2.2", - "windows-sys 0.61.2", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -1844,7 +1724,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn", ] [[package]] @@ -1857,7 +1737,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn", ] [[package]] @@ -1868,7 +1748,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1879,7 +1759,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1897,921 +1777,147 @@ dependencies = [ ] [[package]] -name = "datafusion" -version = "55.0.0" +name = "defmt" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "bzip2", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "flate2", - "futures", - "indexmap 2.14.0", - "itertools 0.15.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "parquet", - "sqlparser", - "tempfile", - "tokio", - "url", - "uuid", - "zstd", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "datafusion-catalog" -version = "55.0.0" +name = "defmt-macros" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "parking_lot", - "tokio", + "defmt-parser", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "datafusion-catalog-listing" -version = "55.0.0" +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "percent-encoding", + "thiserror 2.0.18", ] [[package]] -name = "datafusion-cli" -version = "55.0.0" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8b87ff78e243da48b836ced274799a1799b4a2fd321700e4b8f3426c01aa7e" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "arrow", - "async-trait", - "aws-config", - "aws-credential-types", - "chrono", - "clap", - "datafusion", - "datafusion-common", - "dirs", - "env_logger", - "futures", - "log", - "mimalloc", - "object_store", - "parking_lot", - "parquet", - "regex", - "rustyline", - "tokio", - "url", + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", ] [[package]] -name = "datafusion-common" -version = "55.0.0" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.1", - "hex", - "indexmap 2.14.0", - "itertools 0.15.0", - "libc", - "log", - "num-traits", - "object_store", - "parquet", - "recursive", - "sqlparser", - "tokio", - "uuid", - "web-time", + "serde_core", ] [[package]] -name = "datafusion-common-runtime" -version = "55.0.0" +name = "derive_builder" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" dependencies = [ - "futures", - "log", - "tokio", + "derive_builder_macro", ] [[package]] -name = "datafusion-datasource" -version = "55.0.0" +name = "derive_builder_core" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "arrow", - "async-compression", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "flate2", - "futures", - "glob", - "itertools 0.15.0", - "liblzma", - "log", - "object_store", - "parking_lot", - "rand 0.9.5", - "tokio", - "tokio-util", - "url", - "zstd", + "darling 0.20.11", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "datafusion-datasource-arrow" -version = "55.0.0" +name = "derive_builder_macro" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ - "arrow", - "arrow-ipc", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.15.0", - "object_store", - "tokio", + "derive_builder_core", + "syn", ] [[package]] -name = "datafusion-datasource-avro" -version = "55.0.0" +name = "diff" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c9587cff8163f9bfcb186e8664ba85cb327031b8ff14330307f961ea2fc196" -dependencies = [ - "arrow", - "arrow-avro", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-adapter", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", -] +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] -name = "datafusion-datasource-csv" -version = "55.0.0" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "regex", - "tokio", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", ] [[package]] -name = "datafusion-datasource-json" -version = "55.0.0" +name = "digest" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "tokio", - "tokio-stream", + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", ] [[package]] -name = "datafusion-datasource-parquet" -version = "55.0.0" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", - "futures", - "itertools 0.15.0", - "log", - "object_store", - "parking_lot", - "parquet", - "tokio", + "dirs-sys", ] [[package]] -name = "datafusion-doc" -version = "55.0.0" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] [[package]] -name = "datafusion-execution" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" -dependencies = [ - "arrow", - "arrow-buffer", - "async-trait", - "bytes", - "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", - "futures", - "log", - "object_store", - "parking_lot", - "parquet", - "pin-project-lite", - "rand 0.9.5", - "tempfile", - "tokio", - "tokio-util", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" -dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.14.0", - "itertools 0.15.0", - "recursive", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.14.0", - "itertools 0.15.0", -] - -[[package]] -name = "datafusion-functions" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" -dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.23.0", - "blake2", - "blake3", - "chrono", - "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hex", - "itertools 0.15.0", - "log", - "md-5 0.11.0", - "memchr", - "num-traits", - "rand 0.9.5", - "regex", - "sha2 0.11.0", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.17.1", - "log", - "num-traits", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-functions-nested" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "hashbrown 0.17.1", - "itertools 0.15.0", - "itoa", - "log", - "memchr", -] - -[[package]] -name = "datafusion-functions-table" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-functions-window" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" -dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-macros" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" -dependencies = [ - "datafusion-doc", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "datafusion-optimizer" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "indexmap 2.14.0", - "itertools 0.15.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "parking_lot", - "petgraph", - "recursive", - "tokio", -] - -[[package]] -name = "datafusion-physical-expr-adapter" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "itertools 0.15.0", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "parking_lot", - "pin-project", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", - "itertools 0.15.0", - "recursive", -] - -[[package]] -name = "datafusion-physical-plan" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" -dependencies = [ - "arrow", - "arrow-data", - "arrow-ipc", - "arrow-ord", - "arrow-schema", - "async-trait", - "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.17.1", - "indexmap 2.14.0", - "itertools 0.15.0", - "log", - "num-traits", - "parking_lot", - "pin-project-lite", - "serde_json", - "tokio", -] - -[[package]] -name = "datafusion-pruning" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "log", -] - -[[package]] -name = "datafusion-session" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" -dependencies = [ - "arrow-schema", - "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", -] - -[[package]] -name = "datafusion-spark" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "992dd0b954f24cb576cbeee3555c3083b9cf7a5a5f2448ea25f7a437d444163f" -dependencies = [ - "arrow", - "bigdecimal", - "chrono", - "crc32fast", - "datafusion", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-functions-nested", - "log", - "num-traits", - "percent-encoding", - "rand 0.9.5", - "serde_json", - "sha1 0.11.0", - "sha2 0.11.0", - "twox-hash", - "url", -] - -[[package]] -name = "datafusion-sql" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" -dependencies = [ - "arrow", - "bigdecimal", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", - "indexmap 2.14.0", - "log", - "recursive", - "regex", - "sqlparser", - "stacker", -] - -[[package]] -name = "datafusion-sqllogictest" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c921345d249b784a513ad2be71fc3df18fd8f019fe6c82233f085a1871154fd" -dependencies = [ - "arrow", - "async-trait", - "bigdecimal", - "clap", - "datafusion", - "datafusion-spark", - "futures", - "half", - "indicatif", - "itertools 0.15.0", - "log", - "object_store", - "sqllogictest", - "sqlparser", - "tempfile", - "thiserror 2.0.18", - "tokio", -] - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror 2.0.18", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "serde_core", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.119", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "const-oid 0.9.6", - "crypto-common 0.1.7", - "subtle", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid 0.10.2", - "crypto-common 0.2.2", - "ctutils", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - -[[package]] -name = "displaydoc" -version = "0.2.6" +name = "displaydoc" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2868,18 +1974,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "educe" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "either" version = "1.16.0" @@ -2889,61 +1983,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - -[[package]] -name = "endian-type" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" - -[[package]] -name = "enum-ordinalize" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f808d588c10e464ea6f7d3eaed500049eff30aaac103460f61828c2d65b3eb" -dependencies = [ - "enum-ordinalize-derive", -] - -[[package]] -name = "enum-ordinalize-derive" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e528e2d34ba8a67a1a650b86beae8ef69fc5fdb638016f386b973226590432" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "env_filter" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -2971,18 +2010,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "error-code" -version = "3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" - -[[package]] -name = "escape8259" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" - [[package]] name = "etcetera" version = "0.8.0" @@ -3061,12 +2088,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "flatbuffers" version = "25.12.19" @@ -3111,12 +2132,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3135,15 +2150,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "fs-err" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" -dependencies = [ - "autocfg", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -3217,7 +2223,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3340,7 +2346,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3413,7 +2419,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash 0.1.5", + "foldhash", ] [[package]] @@ -3421,11 +2427,6 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "hashlink" @@ -3882,23 +2883,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iceberg-datafusion" -version = "0.10.1" -dependencies = [ - "anyhow", - "async-trait", - "dashmap", - "datafusion", - "expect-test", - "futures", - "iceberg", - "parquet", - "tempfile", - "tokio", - "uuid", -] - [[package]] name = "iceberg-examples" version = "0.10.1" @@ -3927,28 +2911,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "iceberg-playground" -version = "0.10.1" -dependencies = [ - "anyhow", - "clap", - "datafusion", - "datafusion-cli", - "dirs", - "fs-err", - "home", - "iceberg", - "iceberg-catalog-rest", - "iceberg-datafusion", - "mimalloc", - "stacker", - "tokio", - "toml 0.8.23", - "tracing", - "tracing-subscriber", -] - [[package]] name = "iceberg-property-macro" version = "0.10.1" @@ -3958,32 +2920,10 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.119", + "syn", "trybuild", ] -[[package]] -name = "iceberg-sqllogictest" -version = "0.10.1" -dependencies = [ - "anyhow", - "async-trait", - "datafusion", - "datafusion-sqllogictest", - "enum-ordinalize", - "env_logger", - "iceberg", - "iceberg-datafusion", - "indicatif", - "libtest-mimic", - "log", - "serde", - "sqllogictest", - "tokio", - "toml 0.8.23", - "tracing", -] - [[package]] name = "iceberg-storage-opendal" version = "0.10.1" @@ -4146,19 +3086,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "indicatif" -version = "0.18.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" -dependencies = [ - "console", - "portable-atomic", - "unicode-width 0.2.2", - "unit-prefix", - "web-time", -] - [[package]] name = "inout" version = "0.1.4" @@ -4218,15 +3145,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -4259,7 +3177,7 @@ checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4304,7 +3222,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.119", + "syn", ] [[package]] @@ -4323,7 +3241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4430,53 +3348,18 @@ dependencies = [ "lexical-util", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" - [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "liblzma" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba" -dependencies = [ - "liblzma-sys", -] - -[[package]] -name = "liblzma-sys" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "libm" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - [[package]] name = "libredox" version = "0.1.18" @@ -4497,19 +3380,7 @@ checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", - "vcpkg", -] - -[[package]] -name = "libtest-mimic" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" -dependencies = [ - "anstream", - "anstyle", - "clap", - "escape8259", + "vcpkg", ] [[package]] @@ -4679,15 +3550,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "minijinja" version = "2.24.0" @@ -4742,7 +3604,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4816,7 +3678,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4836,7 +3698,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4851,15 +3713,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec", -] - [[package]] name = "nix" version = "0.29.0" @@ -4873,18 +3726,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.0", - "cfg-if 1.0.4", - "cfg_aliases", - "libc", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -5003,7 +3844,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5034,54 +3875,6 @@ dependencies = [ "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "form_urlencoded", - "futures-channel", - "futures-core", - "futures-util", - "http 1.5.0", - "http-body-util", - "humantime", - "hyper", - "itertools 0.14.0", - "md-5 0.10.6", - "parking_lot", - "percent-encoding", - "quick-xml 0.39.4", - "rand 0.10.2", - "reqwest 0.12.28", - "ring", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.18", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -5143,7 +3936,7 @@ dependencies = [ "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "serde", "serde_json", @@ -5223,7 +4016,7 @@ dependencies = [ "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5267,7 +4060,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5302,7 +4095,7 @@ dependencies = [ "http 1.5.0", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5322,7 +4115,7 @@ dependencies = [ "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5387,12 +4180,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "owo-colors" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" - [[package]] name = "parking" version = "2.2.1" @@ -5447,7 +4234,6 @@ dependencies = [ "num-bigint 0.5.1", "num-integer", "num-traits", - "object_store", "ring", "seq-macro", "simdutf8", @@ -5498,18 +4284,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "serde", -] - [[package]] name = "phf" version = "0.12.1" @@ -5567,7 +4341,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5726,7 +4500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.119", + "syn", ] [[package]] @@ -5735,7 +4509,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.12+spec-1.1.0", + "toml_edit", ] [[package]] @@ -5747,16 +4521,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "psm" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" -dependencies = [ - "ar_archive_writer", - "cc", -] - [[package]] name = "ptr_meta" version = "0.3.1" @@ -5774,7 +4538,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5828,7 +4592,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5840,7 +4604,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5849,16 +4613,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" -[[package]] -name = "quick-xml" -version = "0.39.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -5947,16 +4701,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radix_trie" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" -dependencies = [ - "endian-type", - "nibble_vec", -] - [[package]] name = "rancor" version = "0.1.2" @@ -6053,26 +4797,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn 2.0.119", -] - [[package]] name = "redb" version = "3.1.3" @@ -6128,7 +4852,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6207,7 +4931,7 @@ dependencies = [ "http 1.5.0", "log", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "rust-ini", "serde", @@ -6225,7 +4949,7 @@ dependencies = [ "bytes", "http 1.5.0", "log", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-core", "reqsign-core", "serde", @@ -6315,27 +5039,20 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", "http 1.5.0", "http-body 1.0.1", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -6441,7 +5158,7 @@ checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6500,7 +5217,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.119", + "syn", "unicode-ident", ] @@ -6627,27 +5344,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "rustyline" -version = "18.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684" -dependencies = [ - "bitflags 2.13.0", - "cfg-if 1.0.4", - "clipboard-win", - "home", - "libc", - "log", - "memchr", - "nix 0.31.3", - "radix_trie", - "unicode-segmentation", - "unicode-width 0.2.2", - "utf8parse", - "windows-sys 0.61.2", -] - [[package]] name = "ryu" version = "1.0.23" @@ -6824,7 +5520,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -6833,7 +5529,6 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6849,16 +5544,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", + "syn", ] [[package]] @@ -6912,7 +5598,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7153,53 +5839,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqllogictest" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b2262a244037b0b510edbd25a8e6c9fb8d73ee0237fc6cc95a54c16f94a82" -dependencies = [ - "async-trait", - "educe", - "fs-err", - "futures", - "glob", - "humantime", - "itertools 0.13.0", - "libtest-mimic", - "md-5 0.10.6", - "owo-colors", - "rand 0.8.6", - "regex", - "similar", - "subst", - "tempfile", - "thiserror 2.0.18", - "tracing", -] - -[[package]] -name = "sqlparser" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" -dependencies = [ - "log", - "recursive", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "sqlx" version = "0.8.6" @@ -7259,7 +5898,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.119", + "syn", ] [[package]] @@ -7280,7 +5919,7 @@ dependencies = [ "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", - "syn 2.0.119", + "syn", "tokio", "url", ] @@ -7393,19 +6032,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "stacker" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" -dependencies = [ - "cc", - "cfg-if 1.0.4", - "libc", - "psm", - "windows-sys 0.61.2", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -7445,7 +6071,7 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", + "strum_macros", ] [[package]] @@ -7457,29 +6083,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "strum_macros" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "subst" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" -dependencies = [ - "memchr", - "unicode-width 0.1.14", + "syn", ] [[package]] @@ -7505,17 +6109,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -7533,7 +6126,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7643,7 +6236,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7654,7 +6247,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7755,7 +6348,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -7788,7 +6381,6 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util", ] [[package]] @@ -7804,18 +6396,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned 0.6.9", - "toml_datetime 0.6.11", - "toml_edit 0.22.27", -] - [[package]] name = "toml" version = "1.1.3+spec-1.1.0" @@ -7824,20 +6404,11 @@ checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ "indexmap 2.14.0", "serde_core", - "serde_spanned 1.1.1", - "toml_datetime 1.1.1+spec-1.1.0", + "serde_spanned", + "toml_datetime", "toml_parser", "toml_writer", - "winnow 1.0.3", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", + "winnow", ] [[package]] @@ -7849,20 +6420,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned 0.6.9", - "toml_datetime 0.6.11", - "toml_write", - "winnow 0.7.15", -] - [[package]] name = "toml_edit" version = "0.25.12+spec-1.1.0" @@ -7870,9 +6427,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap 2.14.0", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", - "winnow 1.0.3", + "winnow", ] [[package]] @@ -7881,15 +6438,9 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.3", + "winnow", ] -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -7974,7 +6525,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8047,7 +6598,7 @@ dependencies = [ "serde_json", "target-triple", "termcolor", - "toml 1.1.3+spec-1.1.0", + "toml", ] [[package]] @@ -8055,9 +6606,6 @@ name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -dependencies = [ - "rand 0.9.5", -] [[package]] name = "typed-builder" @@ -8076,7 +6624,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8112,7 +6660,7 @@ checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8148,30 +6696,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - [[package]] name = "universal-hash" version = "0.5.1" @@ -8262,7 +6786,7 @@ dependencies = [ "metainfo", "motore", "mur3", - "nix 0.29.0", + "nix", "once_cell", "pin-project", "rand 0.9.5", @@ -8410,7 +6934,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn", "wasm-bindgen-shared", ] @@ -8603,7 +7127,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8614,7 +7138,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -8819,15 +7343,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "1.0.3" @@ -9037,7 +7552,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -9058,7 +7573,7 @@ checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -9078,7 +7593,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -9118,7 +7633,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d1083a3dc7..661c56b094 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ members = [ "crates/integration_tests", "crates/integrations/*", "crates/property-macro", - "crates/sqllogictest", "crates/storage/*", "crates/test_utils", ] @@ -75,16 +74,8 @@ bimap = "0.6" bytes = "1.11" cfg-if = "1" chrono = "0.4.41" -clap = { version = "4.5.48", features = ["derive", "cargo"] } crc32fast = "1" -dashmap = "6" -datafusion = "55.0.0" -datafusion-cli = "55.0.0" -datafusion-sqllogictest = "55.0.0" derive_builder = "0.20" -dirs = "6" -enum-ordinalize = "4.3.0" -env_logger = "0.11.8" expect-test = "1" fastnum = { version = "0.7", default-features = false, features = [ "std", @@ -94,10 +85,8 @@ faststr = "0.2.31" flate2 = "1.1.5" fnv = "1.0.7" form_urlencoded = "1.2.2" -fs-err = "3.1.0" futures = "0.3" hive_metastore = "0.2.0" -home = "0.5.12" http = "1.2" iceberg = { version = "0.10.0", path = "./crates/iceberg" } iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" } @@ -105,16 +94,11 @@ iceberg-catalog-hms = { version = "0.10.0", path = "./crates/catalog/hms" } iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } -iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } -indicatif = "0.18" itertools = "0.13" -libtest-mimic = "0.8.1" linkedbytes = "0.1.8" -log = "0.4.28" metainfo = "0.7.14" -mimalloc = "0.1.46" minijinja = "2.12.0" mockall = "0.13.1" mockito = "1" @@ -140,9 +124,7 @@ serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" -sqllogictest = "0.29" sqlx = { version = "0.8.1", default-features = false } -stacker = "0.1.20" strum = "0.27.2" syn = "2" tempfile = "3.18" @@ -150,7 +132,6 @@ tokio = { version = "1.47", default-features = false, features = [ "sync", "rt-multi-thread", ] } -toml = "0.8" tracing = "0.1.41" tracing-subscriber = "0.3.20" trybuild = "1" diff --git a/README.md b/README.md index 536562b7b0..fa974dc767 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | -| [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -48,12 +47,6 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg release docs]: https://docs.rs/iceberg [iceberg dev docs]: https://rust.iceberg.apache.org/api/iceberg/ -[iceberg-datafusion]: crates/integrations/datafusion/README.md -[iceberg-datafusion image]: https://img.shields.io/crates/v/iceberg-datafusion.svg -[iceberg-datafusion link]: https://crates.io/crates/iceberg-datafusion -[iceberg-datafusion dev docs]: https://rust.iceberg.apache.org/api/iceberg_datafusion/ -[iceberg-datafusion release docs]: https://docs.rs/iceberg-datafusion - [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index d38f10dd3f..4f0b8daa21 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -238,7 +238,6 @@ hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X iceberg-storage-opendal@0.10.0 X iceberg_test_utils@0.10.0 X icu_collections@2.1.1 X diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml deleted file mode 100644 index 11fb7b1d0b..0000000000 --- a/crates/integrations/datafusion/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -name = "iceberg-datafusion" -publish = true -rust-version = { workspace = true } -version = { workspace = true } - -categories = ["database"] -description = "Apache Iceberg DataFusion Integration" -keywords = ["iceberg", "integrations", "datafusion"] -license = { workspace = true } -repository = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -dashmap = { workspace = true } -datafusion = { workspace = true } -futures = { workspace = true } -iceberg = { workspace = true } -parquet = { workspace = true } -tokio = { workspace = true } -uuid = { workspace = true } - -[dev-dependencies] -expect-test = { workspace = true } -parquet = { workspace = true } -tempfile = { workspace = true } - -[lints] -workspace = true diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv deleted file mode 100644 index 72d6d7fd2c..0000000000 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,412 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-sql@53.1.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -either@1.15.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -psm@0.1.30 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.2 X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/datafusion/LICENSE b/crates/integrations/datafusion/LICENSE deleted file mode 120000 index 5853aaea53..0000000000 --- a/crates/integrations/datafusion/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../LICENSE \ No newline at end of file diff --git a/crates/integrations/datafusion/NOTICE b/crates/integrations/datafusion/NOTICE deleted file mode 120000 index 295f6bdb3a..0000000000 --- a/crates/integrations/datafusion/NOTICE +++ /dev/null @@ -1 +0,0 @@ -../../../NOTICE \ No newline at end of file diff --git a/crates/integrations/datafusion/README.md b/crates/integrations/datafusion/README.md deleted file mode 100644 index 134a8eff4b..0000000000 --- a/crates/integrations/datafusion/README.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# Apache Iceberg DataFusion Integration - -This crate contains the integration of Apache DataFusion and Apache Iceberg. diff --git a/crates/integrations/datafusion/public-api.txt b/crates/integrations/datafusion/public-api.txt deleted file mode 100644 index 5fcb4e85d3..0000000000 --- a/crates/integrations/datafusion/public-api.txt +++ /dev/null @@ -1,139 +0,0 @@ -pub mod iceberg_datafusion -pub mod iceberg_datafusion::metadata_table -pub struct iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub async fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan(self) -> datafusion_common::error::Result>> -impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::physical_plan -pub struct iceberg_datafusion::physical_plan::IcebergTableScan -impl iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::limit(&self) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::predicates(&self) -> core::option::Option<&iceberg::expr::predicate::Predicate> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::projection(&self) -> core::option::Option<&[alloc::string::String]> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::snapshot_id(&self) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::table(&self) -> &iceberg::table::Table -impl core::fmt::Debug for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_physical_plan::display::DisplayAs for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::fmt_as(&self, _t: datafusion_physical_plan::display::DisplayFormatType, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_physical_plan::execution_plan::ExecutionPlan for iceberg_datafusion::physical_plan::IcebergTableScan -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::apply_expressions(&self, _f: &mut dyn core::ops::function::FnMut(&alloc::sync::Arc) -> datafusion_common::error::Result) -> datafusion_common::error::Result -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::children(&self) -> alloc::vec::Vec<&alloc::sync::Arc<(dyn datafusion_physical_plan::execution_plan::ExecutionPlan + 'static)>> -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::execute(&self, _partition: usize, _context: alloc::sync::Arc) -> datafusion_common::error::Result -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::name(&self) -> &str -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::properties(&self) -> &alloc::sync::Arc -pub fn iceberg_datafusion::physical_plan::IcebergTableScan::with_new_children(self: alloc::sync::Arc, _children: alloc::vec::Vec>) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::physical_plan::convert_filters_to_predicate(filters: &[datafusion_expr::expr::Expr]) -> core::option::Option -pub fn iceberg_datafusion::physical_plan::project_with_partition(input: alloc::sync::Arc, table: &iceberg::table::Table) -> datafusion_common::error::Result> -pub mod iceberg_datafusion::table -pub mod iceberg_datafusion::table::metadata_table -pub struct iceberg_datafusion::table::metadata_table::IcebergMetadataTableProvider -impl iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub async fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan(self) -> datafusion_common::error::Result>> -impl core::clone::Clone for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::clone(&self) -> iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -impl core::fmt::Debug for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::metadata_table::IcebergMetadataTableProvider -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _projection: core::option::Option<&'life2 alloc::vec::Vec>, _filters: &'life3 [datafusion_expr::expr::Expr], _limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::metadata_table::IcebergMetadataTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::table::table_provider_factory -pub struct iceberg_datafusion::table::table_provider_factory::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub struct iceberg_datafusion::table::IcebergStaticTableProvider -impl iceberg_datafusion::IcebergStaticTableProvider -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table(table: iceberg::table::Table) -> iceberg::error::Result -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table_snapshot(table: iceberg::table::Table, snapshot_id: i64) -> iceberg::error::Result -impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergStaticTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergStaticTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::table::IcebergTableProvider -impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub mod iceberg_datafusion::table_provider_factory -pub struct iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub struct iceberg_datafusion::IcebergCatalogProvider -impl iceberg_datafusion::IcebergCatalogProvider -pub async fn iceberg_datafusion::IcebergCatalogProvider::try_new(client: alloc::sync::Arc) -> iceberg::error::Result -impl core::fmt::Debug for iceberg_datafusion::IcebergCatalogProvider -pub fn iceberg_datafusion::IcebergCatalogProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::catalog::CatalogProvider for iceberg_datafusion::IcebergCatalogProvider -pub fn iceberg_datafusion::IcebergCatalogProvider::schema(&self, name: &str) -> core::option::Option> -pub fn iceberg_datafusion::IcebergCatalogProvider::schema_names(&self) -> alloc::vec::Vec -pub struct iceberg_datafusion::IcebergStaticTableProvider -impl iceberg_datafusion::IcebergStaticTableProvider -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table(table: iceberg::table::Table) -> iceberg::error::Result -pub async fn iceberg_datafusion::IcebergStaticTableProvider::try_new_from_table_snapshot(table: iceberg::table::Table, snapshot_id: i64) -> iceberg::error::Result -impl core::clone::Clone for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::clone(&self) -> iceberg_datafusion::IcebergStaticTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergStaticTableProvider -pub fn iceberg_datafusion::IcebergStaticTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, _input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergStaticTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergStaticTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergStaticTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::IcebergTableProvider -impl core::clone::Clone for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::clone(&self) -> iceberg_datafusion::IcebergTableProvider -impl core::fmt::Debug for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProvider for iceberg_datafusion::IcebergTableProvider -pub fn iceberg_datafusion::IcebergTableProvider::insert_into<'life0, 'life1, 'async_trait>(&'life0 self, state: &'life1 dyn datafusion_session::session::Session, input: alloc::sync::Arc, _insert_op: datafusion_expr::logical_plan::dml::InsertOp) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, projection: core::option::Option<&'life2 alloc::vec::Vec>, filters: &'life3 [datafusion_expr::expr::Expr], limit: core::option::Option) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait -pub fn iceberg_datafusion::IcebergTableProvider::schema(&self) -> arrow_schema::schema::SchemaRef -pub fn iceberg_datafusion::IcebergTableProvider::supports_filters_pushdown(&self, filters: &[&datafusion_expr::expr::Expr]) -> datafusion_common::error::Result> -pub fn iceberg_datafusion::IcebergTableProvider::table_type(&self) -> datafusion_expr::table_source::TableType -pub struct iceberg_datafusion::IcebergTableProviderFactory -impl iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new() -> Self -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::new_with_storage_factory(storage_factory: alloc::sync::Arc) -> Self -impl core::default::Default for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::default() -> iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -impl core::fmt::Debug for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl datafusion_session::table::TableProviderFactory for iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory -pub fn iceberg_datafusion::table_provider_factory::IcebergTableProviderFactory::create<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _state: &'life1 dyn datafusion_session::session::Session, cmd: &'life2 datafusion_expr::logical_plan::ddl::CreateExternalTable) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait -pub fn iceberg_datafusion::from_datafusion_error(error: datafusion_common::error::DataFusionError) -> iceberg::error::Error -pub fn iceberg_datafusion::to_datafusion_error(error: iceberg::error::Error) -> datafusion_common::error::DataFusionError diff --git a/crates/integrations/datafusion/src/catalog.rs b/crates/integrations/datafusion/src/catalog.rs deleted file mode 100644 index 2c6e1ff002..0000000000 --- a/crates/integrations/datafusion/src/catalog.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::sync::Arc; - -use datafusion::catalog::{CatalogProvider, SchemaProvider}; -use futures::future::try_join_all; -use iceberg::{Catalog, NamespaceIdent, Result}; - -use crate::schema::IcebergSchemaProvider; - -/// Provides an interface to manage and access multiple schemas -/// within an Iceberg [`Catalog`]. -/// -/// Acts as a centralized catalog provider that aggregates -/// multiple [`SchemaProvider`], each associated with distinct namespaces. -#[derive(Debug)] -pub struct IcebergCatalogProvider { - /// A `HashMap` where keys are namespace names - /// and values are dynamic references to objects implementing the - /// [`SchemaProvider`] trait. - schemas: HashMap>, -} - -impl IcebergCatalogProvider { - /// Asynchronously tries to construct a new [`IcebergCatalogProvider`] - /// using the given client to fetch and initialize schema providers for - /// each namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves the list of namespace names - /// attempts to create a schema provider for each namespace, and - /// collects these providers into a `HashMap`. - pub async fn try_new(client: Arc) -> Result { - // TODO: - // Schemas and providers should be cached and evicted based on time - // As of right now; schemas might become stale. - let schema_names: Vec<_> = client - .list_namespaces(None) - .await? - .iter() - .flat_map(|ns| ns.as_ref().clone()) - .collect(); - - let providers = try_join_all( - schema_names - .iter() - .map(|name| { - IcebergSchemaProvider::try_new( - client.clone(), - NamespaceIdent::new(name.clone()), - ) - }) - .collect::>(), - ) - .await?; - - let schemas: HashMap> = schema_names - .into_iter() - .zip(providers) - .map(|(name, provider)| { - let provider = Arc::new(provider) as Arc; - (name, provider) - }) - .collect(); - - Ok(IcebergCatalogProvider { schemas }) - } -} - -impl CatalogProvider for IcebergCatalogProvider { - fn schema_names(&self) -> Vec { - self.schemas.keys().cloned().collect() - } - - fn schema(&self, name: &str) -> Option> { - self.schemas.get(name).cloned() - } -} diff --git a/crates/integrations/datafusion/src/error.rs b/crates/integrations/datafusion/src/error.rs deleted file mode 100644 index 5bc47013dd..0000000000 --- a/crates/integrations/datafusion/src/error.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::anyhow; -use iceberg::{Error, ErrorKind}; - -/// Converts a datafusion error into an iceberg error. -pub fn from_datafusion_error(error: datafusion::error::DataFusionError) -> Error { - Error::new( - ErrorKind::Unexpected, - "Operation failed for hitting datafusion error".to_string(), - ) - .with_source(anyhow!("datafusion error: {error:?}")) -} -/// Converts an iceberg error into a datafusion error. -pub fn to_datafusion_error(error: Error) -> datafusion::error::DataFusionError { - datafusion::error::DataFusionError::External(error.into()) -} diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/integrations/datafusion/src/lib.rs deleted file mode 100644 index 4b0ea8606d..0000000000 --- a/crates/integrations/datafusion/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod catalog; -pub use catalog::*; - -mod error; -pub use error::*; - -pub mod physical_plan; -mod schema; -pub mod table; -pub use table::table_provider_factory::IcebergTableProviderFactory; -pub use table::*; - -pub(crate) mod task_writer; diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs deleted file mode 100644 index bb507217e2..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ /dev/null @@ -1,713 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; - -use datafusion::arrow::array::{Array, ArrayRef, RecordBatch, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::common::{DataFusionError, Result as DFResult}; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::StreamExt; -use iceberg::Catalog; -use iceberg::spec::{DataFile, deserialize_data_file_from_json}; -use iceberg::table::Table; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::to_datafusion_error; - -/// IcebergCommitExec is responsible for collecting the files written and use -/// [`Transaction::fast_append`] to commit the data files written. -#[derive(Debug)] -pub(crate) struct IcebergCommitExec { - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - count_schema: ArrowSchemaRef, - plan_properties: Arc, -} - -impl IcebergCommitExec { - pub fn new( - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - ) -> Self { - let count_schema = Self::make_count_schema(); - - let plan_properties = Self::compute_properties(Arc::clone(&count_schema)); - - Self { - table, - catalog, - input, - schema, - count_schema, - plan_properties, - } - } - - // Compute the plan properties for this execution plan - fn compute_properties(schema: ArrowSchemaRef) -> Arc { - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )) - } - - // Create a record batch with just the count of rows written - fn make_count_batch(count: u64) -> DFResult { - let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; - - RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError( - Box::new(e), - Some("Failed to make count batch!".to_string()), - ) - }) - } - - fn make_count_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - "count", - DataType::UInt64, - false, - )])) - } -} - -impl DisplayAs for IcebergCommitExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergCommitExec: table={}, schema={:?}", - self.table.identifier(), - self.schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergCommitExec { - fn name(&self) -> &str { - "IcebergCommitExec" - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn required_input_distribution(&self) -> Vec { - vec![datafusion::physical_plan::Distribution::SinglePartition; self.children().len()] - } - - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(IcebergCommitExec::new( - self.table.clone(), - self.catalog.clone(), - children[0].clone(), - self.schema.clone(), - ))) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - // IcebergCommitExec only has one partition (partition 0) - if partition != 0 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec only has one partition, but got partition {partition}" - ))); - } - - let table = self.table.clone(); - let input_plan = self.input.clone(); - - // todo revisit this - let spec_id = self.table.metadata().default_partition_spec_id(); - let partition_type = self.table.metadata().default_partition_type().clone(); - let current_schema = self.table.metadata().current_schema().clone(); - - let catalog = Arc::clone(&self.catalog); - - // Process the input streams from all partitions and commit the data files - let stream = futures::stream::once(async move { - let mut data_files: Vec = Vec::new(); - let mut total_record_count: u64 = 0; - - // Execute and collect results from the input coalesced plan - let mut batch_stream = input_plan.execute(0, context)?; - - while let Some(batch_result) = batch_stream.next().await { - let batch = batch_result?; - - let files_array = batch - .column_by_name(DATA_FILES_COL_NAME) - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column in input batch".to_string(), - ) - })? - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column to be StringArray".to_string(), - ) - })?; - - // Deserialize all data files from the StringArray - let batch_files: Vec = files_array - .into_iter() - .flatten() - .map(|f| -> DFResult { - // Parse JSON to DataFileSerde and convert to DataFile - deserialize_data_file_from_json( - f, - spec_id, - &partition_type, - ¤t_schema, - ) - .map_err(to_datafusion_error) - }) - .collect::>()?; - - // add record_counts from the current batch to total record count - total_record_count += batch_files.iter().map(|f| f.record_count()).sum::(); - - // Add all deserialized files to our collection - data_files.extend(batch_files); - } - - // If no data files were collected, return a single row with count = 0 - if data_files.is_empty() { - return Self::make_count_batch(0); - } - - // Create a transaction and commit the data files - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(data_files); - - // Apply the action and commit the transaction - let _updated_table = action - .apply(tx) - .map_err(to_datafusion_error)? - .commit(catalog.as_ref()) - .await - .map_err(to_datafusion_error)?; - - Self::make_count_batch(total_record_count) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.count_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::fmt; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, UInt64Array}; - use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; - use datafusion::datasource::MemTable; - use datafusion::execution::context::TaskContext; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::common::collect; - use datafusion::physical_plan::execution_plan::Boundedness; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use datafusion::prelude::*; - use futures::StreamExt; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, NestedField, PrimitiveType, Schema, - Struct, Type, - }; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; - - use super::*; - use crate::physical_plan::DATA_FILES_COL_NAME; - use crate::table::IcebergTableProvider; - - // A mock execution plan that returns record batches with serialized data files - #[derive(Debug)] - struct MockWriteExec { - schema: Arc, - data_files_json: Vec, - plan_properties: Arc, - } - - impl MockWriteExec { - fn new(data_files_json: Vec) -> Self { - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let plan_properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )); - - Self { - schema, - data_files_json, - plan_properties, - } - } - } - - impl ExecutionPlan for MockWriteExec { - fn name(&self) -> &str { - "MockWriteExec" - } - - fn schema(&self) -> Arc { - self.schema.clone() - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> datafusion::common::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> datafusion::common::Result { - // Create a record batch with the serialized data files - let array = Arc::new(StringArray::from(self.data_files_json.clone())) as ArrayRef; - let batch = RecordBatch::try_new(self.schema.clone(), vec![array])?; - - // Create a stream that returns this batch - let stream = futures::stream::once(async move { Ok(batch) }).boxed(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - stream, - ))) - } - } - - // Implement DisplayAs for MockDataFilesExec - impl DisplayAs for MockWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockDataFilesExec: files={}", self.data_files_json.len()) - } - } - } - } - - #[tokio::test] - async fn test_iceberg_commit_exec() -> Result<(), Box> { - // Create a memory catalog with in-memory file IO - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await - .unwrap(), - ); - - // Create a namespace - let namespace = NamespaceIdent::new("test_namespace".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - // Create a schema for the table - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - // Create a table - let table_creation = TableCreation::builder() - .name("test_table".to_string()) - .schema(schema) - .location("memory://root/test_table".to_string()) - .properties(HashMap::new()) - .build(); - - let table = catalog.create_table(&namespace, table_creation).await?; - - // Create data files - let data_file1 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file1.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(1024) - .record_count(100) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - let data_file2 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file2.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(2048) - .record_count(200) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - // Serialize data files to JSON - let partition_type = table.metadata().default_partition_type().clone(); - let data_file1_json = iceberg::spec::serialize_data_file_to_json( - data_file1.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - let data_file2_json = iceberg::spec::serialize_data_file_to_json( - data_file2.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - // Create a mock execution plan that returns the serialized data files - let input_exec = Arc::new(MockWriteExec::new(vec![data_file1_json, data_file2_json])); - - // Create the IcebergCommitExec - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let commit_exec = - IcebergCommitExec::new(table.clone(), catalog.clone(), input_exec, arrow_schema); - - // Verify Execution Plan schema matches the count schema - assert_eq!(commit_exec.schema(), IcebergCommitExec::make_count_schema()); - - // Execute the commit exec - let task_ctx = Arc::new(TaskContext::default()); - let stream = commit_exec.execute(0, task_ctx)?; - let batches = collect(stream).await?; - - // Verify the results - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert_eq!(batch.num_columns(), 1); - assert_eq!(batch.num_rows(), 1); - - // The output should be a record batch with a single column "count" and a single row - // with the total record count (100 + 200 = 300) - let count_array = batch.column(0); - assert_eq!(count_array.len(), 1); - assert_eq!(count_array.data_type(), &DataType::UInt64); - - // Verify that the count is correct - let count = count_array.as_any().downcast_ref::().unwrap(); - assert_eq!(count.value(0), 300); - - // Verify that the table has been updated with the new files - let updated_table = catalog - .load_table(&TableIdent::from_strs(["test_namespace", "test_table"]).unwrap()) - .await?; - let current_snapshot = updated_table.metadata().current_snapshot().unwrap(); - - // Load the manifest list to verify the data files were added - let manifest_list = updated_table - .manifest_list_reader(current_snapshot) - .load() - .await?; - - // There should be at least one manifest - assert!(!manifest_list.entries().is_empty()); - - // Load the first manifest and verify it contains our data files - let manifest = updated_table - .manifest_reader() - .read(&manifest_list.entries()[0]) - .await?; - - // Verify that the manifest contains our data files - let manifest_files: Vec = manifest - .entries() - .iter() - .map(|entry| entry.data_file().file_path().to_string()) - .collect(); - - assert!(manifest_files.contains(&"path/to/file1.parquet".to_string())); - assert!(manifest_files.contains(&"path/to/file2.parquet".to_string())); - - Ok(()) - } - - #[tokio::test] - async fn test_iceberg_commit_exec_empty_insert() -> Result<(), Box> { - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await - .unwrap(), - ); - - let namespace = NamespaceIdent::new("test_empty_insert".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - ]) - .build()?; - - let table_creation = TableCreation::builder() - .name("empty_insert_table".to_string()) - .schema(schema) - .location("memory://root/empty_insert_table".to_string()) - .properties(HashMap::new()) - .build(); - - let table = catalog.create_table(&namespace, table_creation).await?; - let snapshot_count_before = table.metadata().snapshots().len(); - - // Mock write plan produces no data files - let input_exec = Arc::new(MockWriteExec::new(vec![])); - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - let commit_exec = - IcebergCommitExec::new(table.clone(), catalog.clone(), input_exec, arrow_schema); - - let task_ctx = Arc::new(TaskContext::default()); - let stream = commit_exec.execute(0, task_ctx)?; - let batches = collect(stream).await?; - - // Must return exactly one batch with one row and one column - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert_eq!(batch.num_rows(), 1); - assert_eq!(batch.num_columns(), 1); - - // The count column must be UInt64 with value 0 - let count_array = batch.column(0); - assert_eq!(count_array.data_type(), &DataType::UInt64); - let count = count_array.as_any().downcast_ref::().unwrap(); - assert_eq!(count.value(0), 0); - - // No new snapshot should be created for an empty insert - let updated_table = catalog - .load_table( - &TableIdent::from_strs(["test_empty_insert", "empty_insert_table"]).unwrap(), - ) - .await?; - let snapshot_count_after = updated_table.metadata().snapshots().len(); - assert_eq!( - snapshot_count_after, snapshot_count_before, - "Empty insert must not create a new snapshot" - ); - assert!(updated_table.metadata().current_snapshot().is_none()); - - Ok(()) - } - - #[tokio::test] - async fn test_datafusion_execution_partitioned_source() -> Result<(), Box> - { - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await?, - ); - - let namespace = NamespaceIdent::new("test_namespace".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let table_name = "test_table"; - let table_creation = TableCreation::builder() - .name(table_name.to_string()) - .schema(schema) - .location("memory://root/test_table".to_string()) - .properties(HashMap::new()) - .build(); - let _ = catalog.create_table(&namespace, table_creation).await?; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let batches: Vec = (1..4) - .map(|idx| { - RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![idx])) as ArrayRef, - Arc::new(StringArray::from(vec![format!("Name{idx}")])) as ArrayRef, - ]) - }) - .collect::>()?; - - // Create DataFusion context with specific partition configuration - let mut config = SessionConfig::new(); - config = config.set_usize("datafusion.execution.target_partitions", 8); - let ctx = SessionContext::new_with_config(config); - - // Create multiple partitions - each batch becomes a separate partition - let partitions: Vec> = - batches.into_iter().map(|batch| vec![batch]).collect(); - let source_table = Arc::new(MemTable::try_new(Arc::clone(&arrow_schema), partitions)?); - ctx.register_table("source_table", source_table)?; - - let iceberg_table_provider = IcebergTableProvider::try_new( - catalog.clone(), - namespace.clone(), - table_name.to_string(), - ) - .await?; - ctx.register_table("iceberg_table", Arc::new(iceberg_table_provider))?; - - let insert_plan = ctx - .sql("INSERT INTO iceberg_table SELECT * FROM source_table") - .await?; - - let physical_plan = insert_plan.create_physical_plan().await?; - - let actual_plan = format!( - "{}", - datafusion::physical_plan::displayable(physical_plan.as_ref()).indent(false) - ); - - println!("Physical plan:\n{actual_plan}"); - - let expected_plan = "\ -IcebergCommitExec: table=test_namespace.test_table - CoalescePartitionsExec - IcebergWriteExec: table=test_namespace.test_table - DataSourceExec: partitions=3, partition_sizes=[1, 1, 1]"; - - assert_eq!( - actual_plan.trim(), - expected_plan.trim(), - "Physical plan does not match expected\n\nExpected:\n{}\n\nActual:\n{}", - expected_plan.trim(), - actual_plan.trim() - ); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs b/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs deleted file mode 100644 index fb5440a98a..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs +++ /dev/null @@ -1,927 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::vec; - -use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::expr::ScalarFunction; -use datafusion::logical_expr::{BinaryExpr, Expr, Like, Operator}; -use datafusion::scalar::ScalarValue; -use iceberg::expr::{BinaryExpression, Predicate, PredicateOperator, Reference, UnaryExpression}; -use iceberg::spec::{Datum, PrimitiveLiteral}; - -// A datafusion expression could be an Iceberg predicate, column, or literal. -enum TransformedResult { - Predicate(Predicate), - Column(Reference), - Literal(Datum), - NotTransformed, -} - -enum OpTransformedResult { - Operator(PredicateOperator), - And, - Or, - NotTransformed, -} - -/// Converts DataFusion filters ([`Expr`]) to an iceberg [`Predicate`]. -/// If none of the filters could be converted, return `None` which adds no predicates to the scan operation. -/// If the conversion was successful, return the converted predicates combined with an AND operator. -pub fn convert_filters_to_predicate(filters: &[Expr]) -> Option { - filters - .iter() - .filter_map(convert_filter_to_predicate) - .reduce(Predicate::and) -} - -fn convert_filter_to_predicate(expr: &Expr) -> Option { - match to_iceberg_predicate(expr) { - TransformedResult::Predicate(predicate) => Some(predicate), - TransformedResult::Column(column) => { - // A bare column in a filter context represents a boolean column check - // Convert it to: column = true - Some(Predicate::Binary(BinaryExpression::new( - PredicateOperator::Eq, - column, - Datum::bool(true), - ))) - } - TransformedResult::Literal(_) => { - // Literal values in filter context cannot be pushed down - None - } - _ => None, - } -} - -fn to_iceberg_predicate(expr: &Expr) -> TransformedResult { - match expr { - Expr::BinaryExpr(binary) => { - let left = to_iceberg_predicate(&binary.left); - let right = to_iceberg_predicate(&binary.right); - let op = to_iceberg_operation(binary.op); - match op { - OpTransformedResult::Operator(op) => to_iceberg_binary_predicate(left, right, op), - OpTransformedResult::And => to_iceberg_and_predicate(left, right), - OpTransformedResult::Or => to_iceberg_or_predicate(left, right), - OpTransformedResult::NotTransformed => TransformedResult::NotTransformed, - } - } - Expr::Not(exp) => { - let expr = to_iceberg_predicate(exp); - match expr { - TransformedResult::Predicate(p) => TransformedResult::Predicate(!p), - TransformedResult::Column(column) => { - // NOT of a bare boolean column: NOT col => col = false - TransformedResult::Predicate(Predicate::Binary(BinaryExpression::new( - PredicateOperator::Eq, - column, - Datum::bool(false), - ))) - } - _ => TransformedResult::NotTransformed, - } - } - Expr::Column(column) => TransformedResult::Column(Reference::new(column.name())), - Expr::Literal(literal, _) => match scalar_value_to_datum(literal) { - Some(data) => TransformedResult::Literal(data), - None => TransformedResult::NotTransformed, - }, - Expr::InList(inlist) => { - let mut datums = vec![]; - for expr in &inlist.list { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Literal(l) => datums.push(l), - _ => return TransformedResult::NotTransformed, - } - } - - let expr = to_iceberg_predicate(&inlist.expr); - match expr { - TransformedResult::Column(r) => match inlist.negated { - false => TransformedResult::Predicate(r.is_in(datums)), - true => TransformedResult::Predicate(r.is_not_in(datums)), - }, - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::IsNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNotNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::NotNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::Cast(c) => { - if *c.field.data_type() == DataType::Date32 || *c.field.data_type() == DataType::Date64 - { - // Casts to date truncate the expression, we cannot simply extract it as it - // can create erroneous predicates. - return TransformedResult::NotTransformed; - } - to_iceberg_predicate(&c.expr) - } - Expr::Like(Like { - negated, - expr, - pattern, - escape_char, - case_insensitive, - }) => { - // Only support simple prefix patterns (e.g., 'prefix%') - // Note: Iceberg's StartsWith operator is case-sensitive, so we cannot - // push down case-insensitive LIKE (ILIKE) patterns - // Escape characters are also not supported for pushdown - if escape_char.is_some() || *case_insensitive { - return TransformedResult::NotTransformed; - } - - // Extract the pattern string - let pattern_str = match to_iceberg_predicate(pattern) { - TransformedResult::Literal(d) => match d.literal() { - PrimitiveLiteral::String(s) => s.clone(), - _ => return TransformedResult::NotTransformed, - }, - _ => return TransformedResult::NotTransformed, - }; - - // Check if it's a simple prefix pattern (ends with % and no other wildcards) - if pattern_str.ends_with('%') - && !pattern_str[..pattern_str.len() - 1].contains(['%', '_']) - { - // Extract the prefix (remove trailing %) - let prefix = pattern_str[..pattern_str.len() - 1].to_string(); - - // Get the column reference - let column = match to_iceberg_predicate(expr) { - TransformedResult::Column(r) => r, - _ => return TransformedResult::NotTransformed, - }; - - // Create the appropriate predicate - let predicate = if *negated { - column.not_starts_with(Datum::string(prefix)) - } else { - column.starts_with(Datum::string(prefix)) - }; - - TransformedResult::Predicate(predicate) - } else { - // Complex LIKE patterns cannot be pushed down - TransformedResult::NotTransformed - } - } - Expr::ScalarFunction(ScalarFunction { func, args }) => { - scalar_function_to_iceberg_predicate(func.name(), args) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_operation(op: Operator) -> OpTransformedResult { - match op { - Operator::Eq => OpTransformedResult::Operator(PredicateOperator::Eq), - Operator::NotEq => OpTransformedResult::Operator(PredicateOperator::NotEq), - Operator::Lt => OpTransformedResult::Operator(PredicateOperator::LessThan), - Operator::LtEq => OpTransformedResult::Operator(PredicateOperator::LessThanOrEq), - Operator::Gt => OpTransformedResult::Operator(PredicateOperator::GreaterThan), - Operator::GtEq => OpTransformedResult::Operator(PredicateOperator::GreaterThanOrEq), - // AND OR - Operator::And => OpTransformedResult::And, - Operator::Or => OpTransformedResult::Or, - // Others not supported - _ => OpTransformedResult::NotTransformed, - } -} - -/// Translates a DataFusion scalar function into an Iceberg predicate. -/// Unlike dedicated Expr variants (e.g. `Expr::IsNull`), scalar functions are -/// identified by name at runtime, so we need to handle them here. -fn scalar_function_to_iceberg_predicate(func_name: &str, args: &[Expr]) -> TransformedResult { - match func_name { - "isnan" if args.len() == 1 => match resolve_nan_preserving_reference(&args[0]) { - Some(r) => TransformedResult::Predicate(r.is_nan()), - None => TransformedResult::NotTransformed, - }, - _ => TransformedResult::NotTransformed, - } -} - -/// Attempts to resolve a numeric expression argument down to a single column -/// [`Reference`] such that `isnan(arg)` is logically equivalent to -/// `isnan(reference)`. -/// -/// Filter pushdown is reported as `Inexact` (see -/// [`IcebergTableProvider::supports_filters_pushdown`]), so DataFusion -/// re-applies the original predicate after scanning. We therefore only need the -/// pushed-down predicate to be implied by the original filter (it may match -/// extra rows, but must never drop a matching one). Every transformation handled -/// here preserves NaN-ness *exactly* — the result is NaN if and only if the -/// wrapped column is NaN — so both `isnan(arg)` and `NOT isnan(arg)` are sound: -/// -/// * negation: `-x` is NaN iff `x` is NaN -/// * `abs(x)`: `abs(x)` is NaN iff `x` is NaN -/// * casts between numeric types preserve NaN -/// * `x + c`, `c + x`, `x - c`, `c - x` for a finite literal `c` -/// * `x * c`, `c * x`, `x / c` for a finite, non-zero literal `c` -/// -/// Multiplication/division by zero and `c / x` are intentionally rejected: e.g. -/// `x * 0` is NaN when `x` is `±inf`, so it does not imply `x` is NaN. -/// -/// [`IcebergTableProvider::supports_filters_pushdown`]: crate::table::IcebergTableProvider -fn resolve_nan_preserving_reference(expr: &Expr) -> Option { - match expr { - Expr::Column(column) => Some(Reference::new(column.name())), - Expr::Negative(inner) => resolve_nan_preserving_reference(inner), - Expr::Cast(cast) => { - // Casts to date truncate the value and are not numeric, so they - // cannot be treated as NaN-preserving. - if *cast.field.data_type() == DataType::Date32 - || *cast.field.data_type() == DataType::Date64 - { - return None; - } - resolve_nan_preserving_reference(&cast.expr) - } - Expr::ScalarFunction(ScalarFunction { func, args }) - if func.name() == "abs" && args.len() == 1 => - { - resolve_nan_preserving_reference(&args[0]) - } - Expr::BinaryExpr(binary) => resolve_nan_preserving_binary(binary), - _ => None, - } -} - -/// Resolves the column reference from an arithmetic expression that combines a -/// single column with a finite literal while preserving NaN-ness. See -/// [`resolve_nan_preserving_reference`] for the soundness argument. -/// -/// Expressions with column references on both sides (e.g. `(x + 1) * (x - 2)`) -/// are not supported. Handling them safely would require both operands to -/// resolve to the *same* column (`x + y` cannot be expressed as a single -/// `col IS NAN`) and the operator combination itself to be NaN-preserving: -/// `(x + 1) * (x - 2)` is NaN iff `x` is NaN, but `(x + 1) - (x - 2)` is NaN -/// for `x = inf` (`inf - inf`) even though `x` is not. -/// -/// TODO: support NaN-preserving expressions with column references on both -/// sides, see . -fn resolve_nan_preserving_binary(binary: &BinaryExpr) -> Option { - let (left, right) = (&binary.left, &binary.right); - match binary.op { - // `x + c`, `c + x`, `x - c` and `c - x` are NaN iff `x` is NaN, for any - // finite literal `c`. The column may be on either side. - Operator::Plus | Operator::Minus => { - if finite_literal(right).is_some() { - resolve_nan_preserving_reference(left) - } else if finite_literal(left).is_some() { - resolve_nan_preserving_reference(right) - } else { - None - } - } - - // `x * c` and `c * x` are NaN iff `x` is NaN, but only when `c` is - // non-zero. Per IEEE-754: - // - inf is not NaN - // - inf * 0 is NaN - // so multiplying by zero is rejected. The column may be on either side. - Operator::Multiply => { - if matches!(finite_literal(right), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(left) - } else if matches!(finite_literal(left), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(right) - } else { - None - } - } - - // `x / c` is NaN iff `x` is NaN, for a finite non-zero literal `c`. - // `c / x` is rejected and the column must be the dividend (left side). - // Per IEEE-754: - // - 0 is not NaN - // - 0 / 0 is NaN - // so `c / x` is not NaN-preserving. - Operator::Divide => { - if matches!(finite_literal(right), Some(c) if c != 0.0) { - resolve_nan_preserving_reference(left) - } else { - None - } - } - - _ => None, - } -} - -/// Returns the value of `expr` as an `f64` if it is a finite numeric literal -/// (i.e. not a non-literal, non-numeric, or infinite/NaN value). The numeric -/// conversion is delegated to DataFusion's [`ScalarValue::cast_to`]; the value -/// is only used to inspect finiteness and sign (precision loss is irrelevant). -fn finite_literal(expr: &Expr) -> Option { - let Expr::Literal(value, _) = expr else { - return None; - }; - match value.cast_to(&DataType::Float64).ok()? { - ScalarValue::Float64(Some(v)) if v.is_finite() => Some(v), - _ => None, - } -} - -fn to_iceberg_and_predicate( - left: TransformedResult, - right: TransformedResult, -) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.and(right)) - } - (TransformedResult::Predicate(left), _) => TransformedResult::Predicate(left), - (_, TransformedResult::Predicate(right)) => TransformedResult::Predicate(right), - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_or_predicate(left: TransformedResult, right: TransformedResult) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.or(right)) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_binary_predicate( - left: TransformedResult, - right: TransformedResult, - op: PredicateOperator, -) -> TransformedResult { - let (r, d, op) = match (left, right) { - (TransformedResult::NotTransformed, _) => return TransformedResult::NotTransformed, - (_, TransformedResult::NotTransformed) => return TransformedResult::NotTransformed, - (TransformedResult::Column(r), TransformedResult::Literal(d)) => (r, d, op), - (TransformedResult::Literal(d), TransformedResult::Column(r)) => { - (r, d, reverse_predicate_operator(op)) - } - _ => return TransformedResult::NotTransformed, - }; - TransformedResult::Predicate(Predicate::Binary(BinaryExpression::new(op, r, d))) -} - -fn reverse_predicate_operator(op: PredicateOperator) -> PredicateOperator { - match op { - PredicateOperator::Eq => PredicateOperator::Eq, - PredicateOperator::NotEq => PredicateOperator::NotEq, - PredicateOperator::GreaterThan => PredicateOperator::LessThan, - PredicateOperator::GreaterThanOrEq => PredicateOperator::LessThanOrEq, - PredicateOperator::LessThan => PredicateOperator::GreaterThan, - PredicateOperator::LessThanOrEq => PredicateOperator::GreaterThanOrEq, - _ => unreachable!("Reverse {}", op), - } -} - -const MILLIS_PER_DAY: i64 = 24 * 60 * 60 * 1000; - -/// Convert a scalar value to an iceberg datum. -fn scalar_value_to_datum(value: &ScalarValue) -> Option { - match value { - ScalarValue::Boolean(Some(v)) => Some(Datum::bool(*v)), - ScalarValue::Int8(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int16(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int32(Some(v)) => Some(Datum::int(*v)), - ScalarValue::Int64(Some(v)) => Some(Datum::long(*v)), - ScalarValue::Float32(Some(v)) => Some(Datum::double(*v as f64)), - ScalarValue::Float64(Some(v)) => Some(Datum::double(*v)), - ScalarValue::Utf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::LargeUtf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::Binary(Some(v)) => Some(Datum::binary(v.clone())), - ScalarValue::LargeBinary(Some(v)) => Some(Datum::binary(v.clone())), - ScalarValue::Date32(Some(v)) => Some(Datum::date(*v)), - ScalarValue::Date64(Some(v)) => Some(Datum::date((*v / MILLIS_PER_DAY) as i32)), - // Timestamp conversions - // Note: TimestampSecond and TimestampMillisecond are not handled here because - // DataFusion's type coercion always converts them to match the column type - // (either TimestampMicrosecond or TimestampNanosecond) before predicate pushdown. - // See unit tests for how those conversions would work if needed. - ScalarValue::TimestampMicrosecond(Some(v), _) => Some(Datum::timestamp_micros(*v)), - ScalarValue::TimestampNanosecond(Some(v), _) => Some(Datum::timestamp_nanos(*v)), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use datafusion::common::DFSchema; - use datafusion::logical_expr::utils::split_conjunction; - use datafusion::prelude::{Expr, SessionContext}; - use iceberg::expr::{Predicate, Reference}; - use iceberg::spec::Datum; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - - use super::convert_filters_to_predicate; - - fn create_test_schema() -> DFSchema { - let arrow_schema = Schema::new(vec![ - Field::new("foo", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("bar", DataType::Utf8, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("ts", DataType::Timestamp(TimeUnit::Second, None), true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]), - ), - Field::new("qux", DataType::Float64, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - ]); - DFSchema::try_from_qualified_schema("my_table", &arrow_schema).unwrap() - } - - fn convert_to_iceberg_predicate(sql: &str) -> Option { - let df_schema = create_test_schema(); - let expr = SessionContext::new() - .parse_sql_expr(sql, &df_schema) - .unwrap(); - let exprs: Vec = split_conjunction(&expr).into_iter().cloned().collect(); - convert_filters_to_predicate(&exprs[..]) - } - - #[test] - fn test_predicate_conversion_with_single_condition() { - let predicate = convert_to_iceberg_predicate("foo = 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").equal_to(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo != 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").not_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo > 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo >= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo < 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").less_than(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo <= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").less_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo is null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_null()); - - let predicate = convert_to_iceberg_predicate("foo is not null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_not_null()); - - let predicate = convert_to_iceberg_predicate("foo in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("foo not in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_not_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("not foo = 1").unwrap(); - assert_eq!(predicate, !Reference::new("foo").equal_to(Datum::long(1))); - } - - #[test] - fn test_predicate_conversion_with_single_unsupported_condition() { - let predicate = convert_to_iceberg_predicate("foo + 1 = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("length(bar) = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("foo in (1, 2, foo)"); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_single_condition_rev() { - let predicate = convert_to_iceberg_predicate("1 < foo").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - } - - #[test] - fn test_predicate_conversion_with_and_condition() { - let sql = "foo > 1 and bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_unsupported() { - let sql = "foo > 1 and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").greater_than(Datum::long(1)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_both_unsupported() { - let sql = "foo in (1, 2, foo) and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_unsupported() { - let sql = "foo > 1 or length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_supported() { - let sql = "foo > 1 or bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::or( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr() { - let sql = "(foo > 1 and bar = 'test') or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_one_and_expr_supported() { - let sql = "(foo > 1 and length(bar) = 1 ) or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Reference::new("foo").greater_than(Datum::long(1)); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr_unsupported() { - let sql = "(foo > 1 or length(bar) = 1 ) and foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").less_than(Datum::long(0)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_cast() { - let sql = "ts >= timestamp '2023-01-05T00:00:00'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = - Reference::new("ts").greater_than_or_equal_to(Datum::string("2023-01-05T00:00:00")); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_date_cast() { - let sql = "ts >= date '2023-01-05T11:00:00'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_scalar_value_to_datum_timestamp() { - use datafusion::common::ScalarValue; - - // Test TimestampMicrosecond - maps directly to Datum::timestamp_micros - let ts_micros = 1672876800000000i64; // 2023-01-05 00:00:00 UTC in microseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampMicrosecond(Some(ts_micros), None)); - assert_eq!(datum, Some(Datum::timestamp_micros(ts_micros))); - - // Test TimestampNanosecond - maps to Datum::timestamp_nanos to preserve precision - let ts_nanos = 1672876800000000500i64; // 2023-01-05 00:00:00.000000500 UTC in nanoseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampNanosecond(Some(ts_nanos), None)); - assert_eq!(datum, Some(Datum::timestamp_nanos(ts_nanos))); - - // Test None timestamp - let datum = super::scalar_value_to_datum(&ScalarValue::TimestampMicrosecond(None, None)); - assert_eq!(datum, None); - - // Note: TimestampSecond and TimestampMillisecond are not supported because - // DataFusion's type coercion converts them to TimestampMicrosecond or TimestampNanosecond - // before they reach scalar_value_to_datum in SQL queries. - // - // These return None (not pushed down): - let ts_seconds = 1672876800i64; // 2023-01-05 00:00:00 UTC in seconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampSecond(Some(ts_seconds), None)); - assert_eq!(datum, None); - - let ts_millis = 1672876800000i64; // 2023-01-05 00:00:00 UTC in milliseconds - let datum = - super::scalar_value_to_datum(&ScalarValue::TimestampMillisecond(Some(ts_millis), None)); - assert_eq!(datum, None); - } - - #[test] - fn test_scalar_value_to_datum_binary() { - use datafusion::common::ScalarValue; - - let bytes = vec![1u8, 2u8, 3u8]; - let datum = super::scalar_value_to_datum(&ScalarValue::Binary(Some(bytes.clone()))); - assert_eq!(datum, Some(Datum::binary(bytes.clone()))); - - let datum = super::scalar_value_to_datum(&ScalarValue::LargeBinary(Some(bytes.clone()))); - assert_eq!(datum, Some(Datum::binary(bytes))); - - let datum = super::scalar_value_to_datum(&ScalarValue::Binary(None)); - assert_eq!(datum, None); - } - - #[test] - fn test_predicate_conversion_with_binary() { - let sql = "foo = 1 and bar = X'0102'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - // Binary literals are converted to Datum::binary - // Note: SQL literal 1 is converted to Long by DataFusion - let expected_predicate = Reference::new("foo") - .equal_to(Datum::long(1)) - .and(Reference::new("bar").equal_to(Datum::binary(vec![1u8, 2u8]))); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_scalar_value_to_datum_boolean() { - use datafusion::common::ScalarValue; - - // Test boolean true - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(Some(true))); - assert_eq!(datum, Some(Datum::bool(true))); - - // Test boolean false - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(Some(false))); - assert_eq!(datum, Some(Datum::bool(false))); - - // Test None boolean - let datum = super::scalar_value_to_datum(&ScalarValue::Boolean(None)); - assert_eq!(datum, None); - } - - #[test] - fn test_predicate_conversion_with_like_starts_with() { - let sql = "bar LIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("test")) - ); - } - - #[test] - fn test_predicate_conversion_with_not_like_starts_with() { - let sql = "bar NOT LIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").not_starts_with(Datum::string("test")) - ); - } - - #[test] - fn test_predicate_conversion_with_like_empty_prefix() { - let sql = "bar LIKE '%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("")) - ); - } - - #[test] - fn test_predicate_conversion_with_like_complex_pattern() { - // Patterns with wildcards in the middle cannot be pushed down - let sql = "bar LIKE 'te%st'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_underscore_wildcard() { - // Patterns with underscore wildcard cannot be pushed down - let sql = "bar LIKE 'test_'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_no_wildcard() { - // Patterns without trailing % cannot be pushed down as StartsWith - let sql = "bar LIKE 'test'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_ilike() { - // Case-insensitive LIKE (ILIKE) is not supported - let sql = "bar ILIKE 'test%'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_and_other_conditions() { - let sql = "bar LIKE 'test%' AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("bar").starts_with(Datum::string("test")), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_like_special_characters() { - // Test LIKE with special characters in prefix - let sql = "bar LIKE 'test-abc_123%'"; - let predicate = convert_to_iceberg_predicate(sql); - // This should not be pushed down because it contains underscore - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_like_unicode() { - // Test LIKE with unicode characters in prefix - let sql = "bar LIKE '测试%'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!( - predicate, - Reference::new("bar").starts_with(Datum::string("测试")) - ); - } - - #[test] - fn test_predicate_conversion_with_isnan() { - let predicate = convert_to_iceberg_predicate("isnan(qux)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_not_isnan() { - let predicate = convert_to_iceberg_predicate("NOT isnan(qux)").unwrap(); - assert_eq!(predicate, !Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_and_other_condition() { - let sql = "isnan(qux) AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("qux").is_nan(), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_isnan_negation() { - // -x is NaN iff x is NaN - let predicate = convert_to_iceberg_predicate("isnan(-qux)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - - let predicate = convert_to_iceberg_predicate("NOT isnan(-qux)").unwrap(); - assert_eq!(predicate, !Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_abs() { - // abs(x) is NaN iff x is NaN - let predicate = convert_to_iceberg_predicate("isnan(abs(qux))").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_additive() { - // x + c, c + x, x - c, c - x are NaN iff x is NaN (for finite c) - for sql in [ - "isnan(qux + 1)", - "isnan(1 + qux)", - "isnan(qux - 1)", - "isnan(1 - qux)", - "isnan(qux + 1.5)", - ] { - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan(), "sql: {sql}"); - } - } - - #[test] - fn test_predicate_conversion_with_isnan_multiplicative() { - // x * c, c * x, x / c are NaN iff x is NaN (for finite non-zero c) - for sql in ["isnan(qux * 2)", "isnan(2 * qux)", "isnan(qux / 2)"] { - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan(), "sql: {sql}"); - } - } - - #[test] - fn test_predicate_conversion_with_isnan_nested_expr() { - // Nested NaN-preserving transformations resolve to the inner column - let predicate = convert_to_iceberg_predicate("isnan(-(abs(qux) + 1) * 3)").unwrap(); - assert_eq!(predicate, Reference::new("qux").is_nan()); - } - - #[test] - fn test_predicate_conversion_with_isnan_and_other_complex_condition() { - let sql = "isnan(qux + 1) AND foo > 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("qux").is_nan(), - Reference::new("foo").greater_than(Datum::long(1)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_isnan_unsupported_arg() { - // Multiplying/dividing by zero does not preserve NaN-ness: `x * 0` is NaN - // when `x` is ±inf, so it cannot be pushed down. - assert_eq!(convert_to_iceberg_predicate("isnan(qux * 0)"), None); - assert_eq!(convert_to_iceberg_predicate("isnan(qux / 0)"), None); - - // `c / x` is not NaN-preserving (e.g. `0 / 0` is NaN while `0` is not). - assert_eq!(convert_to_iceberg_predicate("isnan(1 / qux)"), None); - - // Expressions referencing more than one column cannot be reduced to a - // single column reference. - assert_eq!(convert_to_iceberg_predicate("isnan(qux + foo)"), None); - - // Unknown scalar functions are not pushed down. - assert_eq!(convert_to_iceberg_predicate("isnan(sqrt(qux))"), None); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs deleted file mode 100644 index 33c1143ab3..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use datafusion::catalog::TableProvider; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use futures::TryStreamExt; - -use crate::metadata_table::IcebergMetadataTableProvider; - -#[derive(Debug)] -pub struct IcebergMetadataScan { - provider: IcebergMetadataTableProvider, - properties: Arc, -} - -impl IcebergMetadataScan { - pub fn new(provider: IcebergMetadataTableProvider) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(provider.schema()), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - )); - Self { - provider, - properties, - } - } -} - -impl DisplayAs for IcebergMetadataScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "IcebergMetadataScan") - } -} - -impl ExecutionPlan for IcebergMetadataScan { - fn name(&self) -> &str { - "IcebergMetadataScan" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> datafusion::error::Result, - ) -> datafusion::error::Result { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> datafusion::error::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> datafusion::error::Result { - let fut = self.provider.clone().scan(); - let stream = futures::stream::once(fut).try_flatten(); - let schema = self.provider.schema(); - Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/mod.rs b/crates/integrations/datafusion/src/physical_plan/mod.rs deleted file mode 100644 index aeac30de32..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub(crate) mod commit; -pub(crate) mod expr_to_predicate; -pub(crate) mod metadata_scan; -pub(crate) mod project; -pub(crate) mod repartition; -pub(crate) mod scan; -pub(crate) mod sort; -pub(crate) mod write; - -pub(crate) const DATA_FILES_COL_NAME: &str = "data_files"; - -pub use expr_to_predicate::convert_filters_to_predicate; -pub use project::project_with_partition; -pub use scan::IcebergTableScan; diff --git a/crates/integrations/datafusion/src/physical_plan/project.rs b/crates/integrations/datafusion/src/physical_plan/project.rs deleted file mode 100644 index a77abd0eef..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/project.rs +++ /dev/null @@ -1,590 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Partition value projection for Iceberg tables. - -use std::sync::Arc; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; -use datafusion::common::{DataFusionError, Result as DFResult}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_expr::expressions::Column; -use datafusion::physical_plan::projection::ProjectionExec; -use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; -use iceberg::arrow::{ - PROJECTED_PARTITION_VALUE_COLUMN, PartitionValueCalculator, schema_to_arrow_schema, - strip_metadata_from_schema, -}; -use iceberg::spec::PartitionSpec; -use iceberg::table::Table; - -use crate::to_datafusion_error; - -/// Extends an ExecutionPlan with partition value calculations for Iceberg tables. -/// -/// This function takes an input ExecutionPlan and extends it with an additional column -/// containing calculated partition values based on the table's partition specification. -/// For unpartitioned tables, returns the original plan unchanged. -/// -/// # Arguments -/// * `input` - The input ExecutionPlan to extend -/// * `table` - The Iceberg table with partition specification -/// -/// # Returns -/// * `Ok(Arc)` - Extended plan with partition values column -/// * `Err` - If partition spec is not found or transformation fails -pub fn project_with_partition( - input: Arc, - table: &Table, -) -> DFResult> { - let metadata = table.metadata(); - let partition_spec = metadata.default_partition_spec(); - let table_schema = metadata.current_schema(); - - if partition_spec.is_unpartitioned() { - return Ok(input); - } - - let input_schema = input.schema(); - - // Validate that input_schema matches the Iceberg table schema - // Strip metadata from both schemas before comparison to ignore metadata differences - let expected_arrow_schema = - schema_to_arrow_schema(table_schema.as_ref()).map_err(to_datafusion_error)?; - let input_schema_cleaned = - strip_metadata_from_schema(&input_schema).map_err(to_datafusion_error)?; - let expected_schema_cleaned = - strip_metadata_from_schema(&expected_arrow_schema).map_err(to_datafusion_error)?; - - if input_schema_cleaned != expected_schema_cleaned { - return Err(DataFusionError::Plan(format!( - "Input schema does not match Iceberg table schema.\n\ - Expected schema: {expected_schema_cleaned}\n\ - Input schema: {input_schema_cleaned}" - ))); - } - - let calculator = - PartitionValueCalculator::try_new(partition_spec.as_ref(), table_schema.as_ref()) - .map_err(to_datafusion_error)?; - - let mut projection_exprs: Vec<(Arc, String)> = - Vec::with_capacity(input_schema.fields().len() + 1); - - for (index, field) in input_schema.fields().iter().enumerate() { - let column_expr = Arc::new(Column::new(field.name(), index)); - projection_exprs.push((column_expr, field.name().clone())); - } - - let partition_expr = Arc::new(PartitionExpr::new(calculator, partition_spec.clone())); - projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); - - let projection = ProjectionExec::try_new(projection_exprs, input)?; - Ok(Arc::new(projection)) -} - -/// PhysicalExpr implementation for partition value calculation -#[derive(Debug, Clone)] -struct PartitionExpr { - calculator: Arc, - partition_spec: Arc, -} - -impl PartitionExpr { - fn new(calculator: PartitionValueCalculator, partition_spec: Arc) -> Self { - Self { - calculator: Arc::new(calculator), - partition_spec, - } - } -} - -// Manual PartialEq/Eq implementations for pointer-based equality -// (two PartitionExpr are equal if they share the same calculator and partition_spec instances) -impl PartialEq for PartitionExpr { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.calculator, &other.calculator) - && Arc::ptr_eq(&self.partition_spec, &other.partition_spec) - } -} - -impl Eq for PartitionExpr {} - -impl PhysicalExpr for PartitionExpr { - fn data_type(&self, _input_schema: &ArrowSchema) -> DFResult { - Ok(self.calculator.partition_arrow_type().clone()) - } - - fn nullable(&self, _input_schema: &ArrowSchema) -> DFResult { - Ok(false) - } - - fn evaluate(&self, batch: &RecordBatch) -> DFResult { - let array = self - .calculator - .calculate(batch) - .map_err(to_datafusion_error)?; - Ok(ColumnarValue::Array(array)) - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec = self - .partition_spec - .fields() - .iter() - .map(|pf| format!("{}({})", pf.transform, pf.name)) - .collect(); - write!(f, "iceberg_partition_values[{}]", field_names.join(", ")) - } -} - -impl std::fmt::Display for PartitionExpr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let field_names: Vec<&str> = self - .partition_spec - .fields() - .iter() - .map(|pf| pf.name.as_str()) - .collect(); - write!(f, "iceberg_partition_values({})", field_names.join(", ")) - } -} - -impl std::hash::Hash for PartitionExpr { - fn hash(&self, state: &mut H) { - // Two PartitionExpr are equal if they share the same calculator and partition_spec Arcs - Arc::as_ptr(&self.calculator).hash(state); - Arc::as_ptr(&self.partition_spec).hash(state); - } -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::array::{ArrayRef, Int32Array, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Fields}; - use datafusion::physical_plan::empty::EmptyExec; - use iceberg::spec::{NestedField, PrimitiveType, Schema, StructType, Transform, Type}; - use iceberg::test_utils::test_runtime; - - use super::*; - - #[test] - fn test_partition_calculator_basic() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - - // Verify partition type - assert_eq!(calculator.partition_type().fields().len(), 1); - assert_eq!(calculator.partition_type().fields()[0].name, "id_partition"); - } - - #[test] - fn test_partition_expr_with_projection() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = Arc::new( - PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema.clone())); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - - let mut projection_exprs: Vec<(Arc, String)> = - Vec::with_capacity(arrow_schema.fields().len() + 1); - for (i, field) in arrow_schema.fields().iter().enumerate() { - let column_expr = Arc::new(Column::new(field.name(), i)); - projection_exprs.push((column_expr, field.name().clone())); - } - - let partition_expr = Arc::new(PartitionExpr::new(calculator, partition_spec)); - projection_exprs.push((partition_expr, PROJECTED_PARTITION_VALUE_COLUMN.to_string())); - - let projection = ProjectionExec::try_new(projection_exprs, input).unwrap(); - let result = Arc::new(projection); - - assert_eq!(result.schema().fields().len(), 3); - assert_eq!(result.schema().field(0).name(), "id"); - assert_eq!(result.schema().field(1).name(), "name"); - assert_eq!(result.schema().field(2).name(), "_partition"); - } - - #[test] - fn test_partition_expr_evaluate() { - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("data", DataType::Utf8, false), - ])); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![10, 20, 30])), - Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "a", "b", "c", - ])), - ]) - .unwrap(); - - let partition_spec = Arc::new(partition_spec); - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let partition_type = calculator.partition_arrow_type().clone(); - let expr = PartitionExpr::new(calculator, partition_spec); - - assert_eq!(expr.data_type(&arrow_schema).unwrap(), partition_type); - assert!(!expr.nullable(&arrow_schema).unwrap()); - - let result = expr.evaluate(&batch).unwrap(); - match result { - ColumnarValue::Array(array) => { - let struct_array = array.as_any().downcast_ref::().unwrap(); - let id_partition = struct_array - .column_by_name("id_partition") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(id_partition.value(0), 10); - assert_eq!(id_partition.value(1), 20); - assert_eq!(id_partition.value(2), 30); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_nested_partition() { - let address_struct = StructType::new(vec![ - NestedField::required(3, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(4, "city", Type::Primitive(PrimitiveType::String)).into(), - ]); - - let table_schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "address", Type::Struct(address_struct)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = PartitionSpec::builder(Arc::new(table_schema.clone())) - .add_partition_field("address.city", "city_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let struct_fields = Fields::from(vec![ - Field::new("street", DataType::Utf8, false), - Field::new("city", DataType::Utf8, false), - ]); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("address", DataType::Struct(struct_fields), false), - ])); - - let street_array = Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "123 Main St", - "456 Oak Ave", - ])); - let city_array = Arc::new(datafusion::arrow::array::StringArray::from(vec![ - "New York", - "Los Angeles", - ])); - - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("street", DataType::Utf8, false)), - street_array as ArrayRef, - ), - ( - Arc::new(Field::new("city", DataType::Utf8, false)), - city_array as ArrayRef, - ), - ]); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2])), - Arc::new(struct_array), - ]) - .unwrap(); - - let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); - let array = calculator.calculate(&batch).unwrap(); - - let struct_array = array.as_any().downcast_ref::().unwrap(); - let city_partition = struct_array - .column_by_name("city_partition") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(city_partition.value(0), "New York"); - assert_eq!(city_partition.value(1), "Los Angeles"); - } - - #[test] - fn test_schema_validation_matching_schemas() { - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema matching the table schema - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!(result.is_ok(), "Schema validation should pass"); - } - - #[test] - fn test_schema_validation_mismatched_schemas() { - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema with different field name (mismatched) - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("different_name", DataType::Utf8, false), // Wrong field name - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!( - result.is_err(), - "Schema validation should fail for mismatched schemas" - ); - assert!( - result - .unwrap_err() - .to_string() - .contains("Input schema does not match Iceberg table schema") - ); - } - - #[test] - fn test_schema_validation_with_metadata_differences() { - use std::collections::HashMap; - - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; - - let table_schema = Arc::new( - Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let partition_spec = PartitionSpec::builder(table_schema.clone()) - .add_partition_field("id", "id_partition", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = iceberg::spec::SortOrder::builder() - .build(&table_schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - (*table_schema).clone(), - partition_spec, - sort_order, - "/test/table".to_string(), - FormatVersion::V2, - HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - // Create Arrow schema with metadata (should be ignored in comparison) - let mut metadata = HashMap::new(); - metadata.insert("extra".to_string(), "metadata".to_string()); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(metadata.clone()), - Field::new("name", DataType::Utf8, false).with_metadata(metadata), - ])); - - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let result = project_with_partition(input, &table); - assert!( - result.is_ok(), - "Schema validation should pass even with metadata differences" - ); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/repartition.rs b/crates/integrations/datafusion/src/physical_plan/repartition.rs deleted file mode 100644 index de82fab2e8..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/repartition.rs +++ /dev/null @@ -1,853 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::num::NonZeroUsize; -use std::sync::Arc; - -use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_plan::expressions::Column; -use datafusion::physical_plan::repartition::RepartitionExec; -use datafusion::physical_plan::{ExecutionPlan, Partitioning}; -use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; -use iceberg::spec::{TableMetadata, TableMetadataRef, Transform}; -/// Creates an Iceberg-aware repartition execution plan that optimizes data distribution -/// for parallel processing while respecting Iceberg table partitioning semantics. -/// -/// Automatically determines the optimal partitioning strategy based on the table's -/// partition specification. -/// -/// ## Partitioning Strategies -/// -/// - **Partitioned tables with Identity/Bucket transforms** – Uses hash partitioning on the -/// `_partition` column for optimal data distribution and file clustering. Ensures that rows -/// with the same partition values are co-located in the same task. -/// -/// - **Partitioned tables with temporal transforms** – Uses round-robin partitioning for -/// temporal transforms (Year, Month, Day, Hour) that don't provide uniform hash distribution. -/// -/// - **Unpartitioned tables** – Uses round-robin distribution to balance load evenly across workers. -/// -/// ## Requirements -/// -/// - **For partitioned tables**: The input MUST include the `_partition` column. -/// Add it by calling [`project_with_partition`](crate::physical_plan::project_with_partition) before [`repartition`]. -/// - **For unpartitioned tables**: No special preparation needed. -/// - Returns an error if a partitioned table is missing the `_partition` column. -/// -/// ## Performance Notes -/// -/// - Only adds repartitioning when the input partitioning differs from the target. -/// - Requires an explicit target partition count for deterministic behavior. -/// -/// # Arguments -/// -/// * `input` - The input [`ExecutionPlan`]. For partitioned tables, must include the `_partition` -/// column (added via [`project_with_partition`](crate::physical_plan::project_with_partition)). -/// * `table_metadata` - Iceberg table metadata containing partition spec. -/// * `target_partitions` - Target number of partitions for parallel processing (must be > 0). -/// -/// # Returns -/// -/// An [`ExecutionPlan`] that applies the optimal partitioning strategy, or the original input plan -/// if repartitioning is not needed. -/// -/// # Errors -/// -/// Returns [`DataFusionError::Plan`] if a partitioned table input is missing the `_partition` column. -/// -/// # Examples -/// -/// For partitioned tables, first add the `_partition` column: -/// -/// ```ignore -/// use std::num::NonZeroUsize; -/// use iceberg_datafusion::physical_plan::project_with_partition; -/// -/// let plan_with_partition = project_with_partition(input_plan, &table)?; -/// -/// let repartitioned_plan = repartition( -/// plan_with_partition, -/// table.metadata_ref(), -/// NonZeroUsize::new(4).unwrap(), -/// )?; -/// ``` -pub(crate) fn repartition( - input: Arc, - table_metadata: TableMetadataRef, - target_partitions: NonZeroUsize, -) -> DFResult> { - let partitioning_strategy = - determine_partitioning_strategy(&input, &table_metadata, target_partitions)?; - - Ok(Arc::new(RepartitionExec::try_new( - input, - partitioning_strategy, - )?)) -} - -/// Determine the optimal partitioning strategy based on table metadata. -/// -/// Analyzes the table's partition specification to select the most appropriate -/// DataFusion partitioning strategy for insert operations. -/// -/// ## Partitioning Strategy -/// -/// - **Partitioned tables**: Must have the `_partition` column in the input schema (added via -/// `project_with_partition`). Uses hash partitioning if the partition spec contains Identity -/// or Bucket transforms for good data distribution. Falls back to round-robin for temporal -/// transforms (Year, Month, Day, Hour) that don't provide uniform hash distribution. -/// -/// - **Unpartitioned tables**: Always uses round-robin batch partitioning to ensure even load -/// distribution across workers. -/// -/// ## Requirements -/// -/// - **For partitioned tables**: The input MUST include the `_partition` column -/// (added via `project_with_partition()`). -/// - **For unpartitioned tables**: No special preparation needed. -/// - Returns an error if a partitioned table is missing the `_partition` column. -fn determine_partitioning_strategy( - input: &Arc, - table_metadata: &TableMetadata, - target_partitions: NonZeroUsize, -) -> DFResult { - let partition_spec = table_metadata.default_partition_spec(); - let input_schema = input.schema(); - let target_partition_count = target_partitions.get(); - - // Check if partition spec has transforms suitable for hash partitioning - let has_hash_friendly_transforms = partition_spec - .fields() - .iter() - .any(|pf| matches!(pf.transform, Transform::Identity | Transform::Bucket(_))); - - let partition_col_result = input_schema.index_of(PROJECTED_PARTITION_VALUE_COLUMN); - let is_partitioned_table = !partition_spec.is_unpartitioned(); - - match (is_partitioned_table, partition_col_result) { - // Case 1: Partitioned table with _partition column present - (true, Ok(partition_col_idx)) => { - let partition_expr = Arc::new(Column::new( - PROJECTED_PARTITION_VALUE_COLUMN, - partition_col_idx, - )) as Arc; - - if has_hash_friendly_transforms { - Ok(Partitioning::Hash( - vec![partition_expr], - target_partition_count, - )) - } else { - Ok(Partitioning::RoundRobinBatch(target_partition_count)) - } - } - - // Case 2: Partitioned table missing _partition column (normally this should not happen) - (true, Err(_)) => Err(DataFusionError::Plan(format!( - "Partitioned table input missing {PROJECTED_PARTITION_VALUE_COLUMN} column. \ - Ensure projection happens before repartitioning." - ))), - - // Case 3: Unpartitioned table, always use RoundRobinBatch - (false, _) => Ok(Partitioning::RoundRobinBatch(target_partition_count)), - } -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::datatypes::{ - DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, TimeUnit, - }; - use datafusion::execution::TaskContext; - use datafusion::physical_plan::empty::EmptyExec; - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::spec::{ - NestedField, NullOrder, PrimitiveType, Schema, SortDirection, SortField, SortOrder, - Transform, Type, - }; - use iceberg::table::Table; - use iceberg::test_utils::test_runtime; - - use super::*; - - fn create_test_table() -> Table { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "data", - Type::Primitive(PrimitiveType::String), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - - Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/metadata.json") - .runtime(test_runtime()) - .build() - .unwrap() - } - - fn create_test_arrow_schema() -> Arc { - Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", ArrowDataType::Int64, false), - ArrowField::new("data", ArrowDataType::Utf8, false), - ])) - } - - #[tokio::test] - async fn test_repartition_unpartitioned_table() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = repartition( - input.clone(), - table.metadata_ref(), - NonZeroUsize::new(4).unwrap(), - ) - .unwrap(); - - assert_ne!(input.name(), repartitioned_plan.name()); - assert_eq!(repartitioned_plan.name(), "RepartitionExec"); - } - - #[tokio::test] - async fn test_repartition_explicit_partitions() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(8).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, 8); - } - _ => panic!("Expected RoundRobinBatch partitioning"), - } - } - - #[tokio::test] - async fn test_repartition_zero_partitions_fails() { - let _table = create_test_table(); - let _input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let result = NonZeroUsize::new(0); - assert!(result.is_none(), "NonZeroUsize::new(0) should return None"); - - // Test that we can't call repartition with 0 partitions - // This is prevented at compile time by NonZeroUsize - let _ = result; // This would be None, so we can't call repartition - } - - #[tokio::test] - async fn test_partition_count_validation() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let target_partitions = 16; - let repartitioned_plan = repartition( - input, - table.metadata_ref(), - NonZeroUsize::new(target_partitions).unwrap(), - ) - .unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, target_partitions); - } - _ => panic!("Expected RoundRobinBatch partitioning"), - } - } - - #[tokio::test] - async fn test_datafusion_repartitioning_integration() { - let table = create_test_table(); - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(3).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::RoundRobinBatch(n) => { - assert_eq!(*n, 3, "Should use round-robin for unpartitioned table"); - } - _ => panic!("Expected RoundRobinBatch partitioning for unpartitioned table"), - } - - let task_ctx = Arc::new(TaskContext::default()); - let stream = repartitioned_plan.execute(0, task_ctx.clone()).unwrap(); - - assert!(!stream.schema().fields().is_empty()); - } - - #[tokio::test] - async fn test_bucket_aware_partitioning() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "category", - Type::Primitive(PrimitiveType::String), - )), - ]) - .build() - .unwrap(); - - let sort_order = SortOrder::builder() - .with_order_id(1) - .with_sort_field(SortField { - source_id: 2, - transform: Transform::Bucket(4), - direction: SortDirection::Ascending, - null_order: NullOrder::First, - }) - .build(&schema) - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/bucketed_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "bucketed_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/bucketed_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", ArrowDataType::Int64, false), - ArrowField::new("category", ArrowDataType::Utf8, false), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - // For bucketed tables without _partition column, should use round-robin - // since the new logic prioritizes _partition column when available - match partitioning { - Partitioning::Hash(_, _) => { - // This would happen if _partition column is present - } - Partitioning::RoundRobinBatch(_) => { - // This happens when _partition column is not present - } - _ => panic!("Unexpected partitioning strategy"), - } - } - - #[tokio::test] - async fn test_combined_partition_and_bucket_strategy() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 3, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder() - .with_order_id(1) - .with_sort_field(SortField { - source_id: 2, - transform: Transform::Bucket(8), - direction: SortDirection::Ascending, - null_order: NullOrder::First, - }) - .build(&schema) - .unwrap(); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/partitioned_bucketed_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "partitioned_bucketed_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/partitioned_bucketed_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::Hash(exprs, _) => { - // Should use _partition column for hash partitioning - assert_eq!( - exprs.len(), - 1, - "Should have exactly one hash column (_partition)" - ); - - let column_names: Vec = exprs - .iter() - .filter_map(|expr| { - expr.downcast_ref::() - .map(|col| col.name().to_string()) - }) - .collect(); - - assert!( - column_names.contains(&PROJECTED_PARTITION_VALUE_COLUMN.to_string()), - "Should use _partition column, got: {column_names:?}" - ); - } - _ => panic!("Expected Hash partitioning with Identity transform"), - } - } - - #[tokio::test] - async fn test_none_distribution_mode_fallback() { - let schema = Schema::builder() - .with_fields(vec![Arc::new(NestedField::required( - 1, - "id", - Type::Primitive(PrimitiveType::Long), - ))]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .build() - .unwrap(); - let sort_order = SortOrder::builder().build(&schema).unwrap(); - - let mut properties = std::collections::HashMap::new(); - properties.insert("write.distribution-mode".to_string(), "none".to_string()); - - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/none_table".to_string(), - iceberg::spec::FormatVersion::V2, - properties, - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "none_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/none_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let input = Arc::new(EmptyExec::new(create_test_arrow_schema())); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for 'none' distribution mode" - ); - } - - #[tokio::test] - async fn test_schema_ref_convenience_method() { - let table = create_test_table(); - - let schema_ref_1 = table.current_schema_ref(); - let schema_ref_2 = Arc::clone(table.metadata().current_schema()); - - assert!( - Arc::ptr_eq(&schema_ref_1, &schema_ref_2), - "current_schema_ref() should return the same Arc as manual approach" - ); - } - - #[tokio::test] - async fn test_range_only_partitions_use_round_robin() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date_day", Transform::Day) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/range_only_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "range_only_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/range_only_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for temporal transforms (Day) that don't provide good hash distribution" - ); - } - - #[tokio::test] - async fn test_mixed_transforms_use_hash_partitioning() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "date", - Type::Primitive(PrimitiveType::Date), - )), - Arc::new(NestedField::required( - 2, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 3, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("date", "date_day", Transform::Day) - .unwrap() - .add_partition_field("user_id", "user_id", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/mixed_transforms_table".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "mixed_transforms_table"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/mixed_transforms_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("date", ArrowDataType::Date32, false), - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - match partitioning { - Partitioning::Hash(exprs, _) => { - assert_eq!(exprs.len(), 1, "Should have one hash column (_partition)"); - let column_names: Vec = exprs - .iter() - .filter_map(|expr| { - expr.downcast_ref::() - .map(|col| col.name().to_string()) - }) - .collect(); - assert!( - column_names.contains(&PROJECTED_PARTITION_VALUE_COLUMN.to_string()), - "Should use _partition column for mixed transforms with Identity, got: {column_names:?}" - ); - } - _ => panic!("Expected Hash partitioning for table with identity transforms"), - } - } - - #[tokio::test] - async fn test_partition_column_with_temporal_transforms_uses_round_robin() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "event_time", - Type::Primitive(PrimitiveType::Timestamp), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("event_time", "event_month", Transform::Month) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/temporal_partition".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "temporal_partition"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/temporal_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new( - "event_time", - ArrowDataType::Timestamp(TimeUnit::Microsecond, None), - false, - ), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::RoundRobinBatch(_)), - "Should use round-robin for _partition column with temporal transforms, not Hash" - ); - } - - #[tokio::test] - async fn test_partition_column_with_identity_transforms_uses_hash() { - let schema = Schema::builder() - .with_fields(vec![ - Arc::new(NestedField::required( - 1, - "user_id", - Type::Primitive(PrimitiveType::Long), - )), - Arc::new(NestedField::required( - 2, - "amount", - Type::Primitive(PrimitiveType::Long), - )), - ]) - .build() - .unwrap(); - - let partition_spec = iceberg::spec::PartitionSpec::builder(schema.clone()) - .add_partition_field("user_id", "user_id", Transform::Identity) - .unwrap() - .build() - .unwrap(); - - let sort_order = SortOrder::builder().build(&schema).unwrap(); - let table_metadata_builder = iceberg::spec::TableMetadataBuilder::new( - schema, - partition_spec, - sort_order, - "/test/identity_partition".to_string(), - iceberg::spec::FormatVersion::V2, - std::collections::HashMap::new(), - ) - .unwrap(); - - let table_metadata = table_metadata_builder.build().unwrap(); - let table = Table::builder() - .metadata(table_metadata.metadata) - .identifier(TableIdent::from_strs(["test", "identity_partition"]).unwrap()) - .file_io(FileIO::new_with_fs()) - .metadata_location("/test/identity_metadata.json") - .runtime(test_runtime()) - .build() - .unwrap(); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("user_id", ArrowDataType::Int64, false), - ArrowField::new("amount", ArrowDataType::Int64, false), - ArrowField::new( - PROJECTED_PARTITION_VALUE_COLUMN, - ArrowDataType::Struct(Fields::empty()), - false, - ), - ])); - let input = Arc::new(EmptyExec::new(arrow_schema)); - - let repartitioned_plan = - repartition(input, table.metadata_ref(), NonZeroUsize::new(4).unwrap()).unwrap(); - - let partitioning = repartitioned_plan.properties().output_partitioning(); - assert!( - matches!(partitioning, Partitioning::Hash(_, _)), - "Should use Hash partitioning for _partition column with Identity transforms" - ); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs deleted file mode 100644 index 1796304552..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ /dev/null @@ -1,252 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::pin::Pin; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::error::Result as DFResult; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use datafusion::prelude::Expr; -use futures::{Stream, TryStreamExt}; -use iceberg::expr::Predicate; -use iceberg::table::Table; - -use super::expr_to_predicate::convert_filters_to_predicate; -use crate::to_datafusion_error; - -/// Manages the scanning process of an Iceberg [`Table`], encapsulating the -/// necessary details and computed properties required for execution planning. -#[derive(Debug)] -pub struct IcebergTableScan { - /// A table in the catalog. - table: Table, - /// Snapshot of the table to scan. - snapshot_id: Option, - /// Stores certain, often expensive to compute, - /// plan properties used in query optimization. - plan_properties: Arc, - /// Projection column names, None means all columns - projection: Option>, - /// Filters to apply to the table scan - predicates: Option, - /// Optional limit on the number of rows to return - limit: Option, -} - -impl IcebergTableScan { - /// Creates a new [`IcebergTableScan`] object. - pub(crate) fn new( - table: Table, - snapshot_id: Option, - schema: ArrowSchemaRef, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> Self { - let output_schema = match projection { - None => schema.clone(), - Some(projection) => Arc::new(schema.project(projection).unwrap()), - }; - let plan_properties = Self::compute_properties(output_schema.clone()); - let projection = get_column_names(schema.clone(), projection); - let predicates = convert_filters_to_predicate(filters); - - Self { - table, - snapshot_id, - plan_properties, - projection, - predicates, - limit, - } - } - - pub fn table(&self) -> &Table { - &self.table - } - - pub fn snapshot_id(&self) -> Option { - self.snapshot_id - } - - pub fn projection(&self) -> Option<&[String]> { - self.projection.as_deref() - } - - pub fn predicates(&self) -> Option<&Predicate> { - self.predicates.as_ref() - } - - pub fn limit(&self) -> Option { - self.limit - } - - /// Computes [`PlanProperties`] used in query optimization. - fn compute_properties(schema: ArrowSchemaRef) -> Arc { - // TODO: - // This is more or less a placeholder, to be replaced - // once we support output-partitioning - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - )) - } -} - -impl ExecutionPlan for IcebergTableScan { - fn name(&self) -> &str { - "IcebergTableScan" - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let fut = get_batch_stream( - self.table.clone(), - self.snapshot_id, - self.projection.clone(), - self.predicates.clone(), - ); - let stream = futures::stream::once(fut).try_flatten(); - - // Apply limit if specified - let limited_stream: Pin> + Send>> = - if let Some(limit) = self.limit { - let mut remaining = limit; - Box::pin(stream.try_filter_map(move |batch| { - futures::future::ready(if remaining == 0 { - Ok(None) - } else if batch.num_rows() <= remaining { - remaining -= batch.num_rows(); - Ok(Some(batch)) - } else { - let limited_batch = batch.slice(0, remaining); - remaining = 0; - Ok(Some(limited_batch)) - }) - })) - } else { - Box::pin(stream) - }; - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - limited_stream, - ))) - } -} - -impl DisplayAs for IcebergTableScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!( - f, - "IcebergTableScan projection:[{}] predicate:[{}]", - self.projection - .clone() - .map_or(String::new(), |v| v.join(",")), - self.predicates - .clone() - .map_or(String::from(""), |p| format!("{p}")) - )?; - if let Some(limit) = self.limit { - write!(f, " limit:[{limit}]")?; - } - Ok(()) - } -} - -/// Asynchronously retrieves a stream of [`RecordBatch`] instances -/// from a given table. -/// -/// This function initializes a [`TableScan`], builds it, -/// and then converts it into a stream of Arrow [`RecordBatch`]es. -async fn get_batch_stream( - table: Table, - snapshot_id: Option, - column_names: Option>, - predicates: Option, -) -> DFResult> + Send>>> { - let scan_builder = match snapshot_id { - Some(snapshot_id) => table.scan().snapshot_id(snapshot_id), - None => table.scan(), - }; - - let mut scan_builder = match column_names { - Some(column_names) => scan_builder.select(column_names), - None => scan_builder.select_all(), - }; - if let Some(pred) = predicates { - scan_builder = scan_builder.with_filter(pred); - } - let table_scan = scan_builder.build().map_err(to_datafusion_error)?; - - let stream = table_scan - .to_arrow() - .await - .map_err(to_datafusion_error)? - .map_err(to_datafusion_error); - Ok(Box::pin(stream)) -} - -fn get_column_names( - schema: ArrowSchemaRef, - projection: Option<&Vec>, -) -> Option> { - projection.map(|v| { - v.iter() - .map(|p| schema.field(*p).name().clone()) - .collect::>() - }) -} diff --git a/crates/integrations/datafusion/src/physical_plan/sort.rs b/crates/integrations/datafusion/src/physical_plan/sort.rs deleted file mode 100644 index 587ab120ca..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/sort.rs +++ /dev/null @@ -1,240 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Partition-based sorting for Iceberg tables. - -use std::sync::Arc; - -use datafusion::arrow::compute::SortOptions; -use datafusion::common::Result as DFResult; -use datafusion::error::DataFusionError; -use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::expressions::Column; -use datafusion::physical_plan::sorts::sort::SortExec; -use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; - -/// Sorts an ExecutionPlan by partition values for Iceberg tables. -/// -/// This function takes an input ExecutionPlan that has been extended with partition values -/// (via `project_with_partition`) and returns a SortExec that sorts by the partition column. -/// The partition values are expected to be in a struct column named `PROJECTED_PARTITION_VALUE_COLUMN`. -/// -/// For unpartitioned tables or plans without the partition column, returns an error. -/// -/// # Arguments -/// * `input` - The input ExecutionPlan with projected partition values -/// -/// # Returns -/// * `Ok(Arc)` - A SortExec that sorts by partition values -/// * `Err` - If the partition column is not found -pub(crate) fn sort_by_partition(input: Arc) -> DFResult> { - let schema = input.schema(); - - // Find the partition column in the schema - let (partition_column_index, _partition_field) = schema - .column_with_name(PROJECTED_PARTITION_VALUE_COLUMN) - .ok_or_else(|| { - DataFusionError::Plan(format!( - "Partition column '{PROJECTED_PARTITION_VALUE_COLUMN}' not found in schema. Ensure the plan has been extended with partition values using project_with_partition." - )) - })?; - - // Create a single sort expression for the partition column - let column_expr = Arc::new(Column::new( - PROJECTED_PARTITION_VALUE_COLUMN, - partition_column_index, - )); - - let sort_expr = PhysicalSortExpr { - expr: column_expr, - options: SortOptions::default(), // Ascending, nulls last - }; - - // Create a SortExec with preserve_partitioning=true to ensure the output partitioning - // is the same as the input partitioning, and the data is sorted within each partition - let lex_ordering = LexOrdering::new(vec![sort_expr]).ok_or_else(|| { - DataFusionError::Plan("Failed to create LexOrdering from sort expression".to_string()) - })?; - - let sort_exec = SortExec::new(lex_ordering, input).with_preserve_partitioning(true); - - Ok(Arc::new(sort_exec)) -} - -#[cfg(test)] -mod tests { - use datafusion::arrow::array::{Int32Array, RecordBatch, StringArray, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema as ArrowSchema}; - use datafusion::datasource::{MemTable, TableProvider}; - use datafusion::prelude::SessionContext; - - use super::*; - - #[tokio::test] - async fn test_sort_by_partition_basic() { - // Create a schema with a partition column - let partition_fields = - Fields::from(vec![Field::new("id_partition", DataType::Int32, false)]); - - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(partition_fields.clone()), - false, - ), - ])); - - // Create test data with partition values - let id_array = Arc::new(Int32Array::from(vec![3, 1, 2])); - let name_array = Arc::new(StringArray::from(vec!["c", "a", "b"])); - let partition_array = Arc::new(StructArray::from(vec![( - Arc::new(Field::new("id_partition", DataType::Int32, false)), - Arc::new(Int32Array::from(vec![3, 1, 2])) as _, - )])); - - let batch = - RecordBatch::try_new(schema.clone(), vec![id_array, name_array, partition_array]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - // Apply sort - let sorted_plan = sort_by_partition(input).unwrap(); - - // Execute and verify - let result = datafusion::physical_plan::collect(sorted_plan, ctx.task_ctx()) - .await - .unwrap(); - - assert_eq!(result.len(), 1); - let result_batch = &result[0]; - - let id_col = result_batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - // Verify data is sorted by partition value - assert_eq!(id_col.value(0), 1); - assert_eq!(id_col.value(1), 2); - assert_eq!(id_col.value(2), 3); - } - - #[tokio::test] - async fn test_sort_by_partition_missing_column() { - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - - let batch = RecordBatch::try_new(schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - ]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - let result = sort_by_partition(input); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("Partition column '_partition' not found") - ); - } - - #[tokio::test] - async fn test_sort_by_partition_multi_field() { - // Test with multiple partition fields in the struct - let partition_fields = Fields::from(vec![ - Field::new("year", DataType::Int32, false), - Field::new("month", DataType::Int32, false), - ]); - - let schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("data", DataType::Utf8, false), - Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(partition_fields.clone()), - false, - ), - ])); - - // Create test data with partition values (year, month) - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3, 4])); - let data_array = Arc::new(StringArray::from(vec!["a", "b", "c", "d"])); - - // Partition values: (2024, 2), (2024, 1), (2023, 12), (2024, 1) - let year_array = Arc::new(Int32Array::from(vec![2024, 2024, 2023, 2024])); - let month_array = Arc::new(Int32Array::from(vec![2, 1, 12, 1])); - - let partition_array = Arc::new(StructArray::from(vec![ - ( - Arc::new(Field::new("year", DataType::Int32, false)), - year_array as _, - ), - ( - Arc::new(Field::new("month", DataType::Int32, false)), - month_array as _, - ), - ])); - - let batch = - RecordBatch::try_new(schema.clone(), vec![id_array, data_array, partition_array]) - .unwrap(); - - let ctx = SessionContext::new(); - let mem_table = MemTable::try_new(schema.clone(), vec![vec![batch]]).unwrap(); - let input = mem_table.scan(&ctx.state(), None, &[], None).await.unwrap(); - - // Apply sort - let sorted_plan = sort_by_partition(input).unwrap(); - - // Execute and verify - let result = datafusion::physical_plan::collect(sorted_plan, ctx.task_ctx()) - .await - .unwrap(); - - assert_eq!(result.len(), 1); - let result_batch = &result[0]; - - let id_col = result_batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - - // Verify data is sorted by partition value (struct comparison) - // Expected order: (2023, 12), (2024, 1), (2024, 1), (2024, 2) - // Which corresponds to ids: 3, 2, 4, 1 - assert_eq!(id_col.value(0), 3); - assert_eq!(id_col.value(1), 2); - assert_eq!(id_col.value(2), 4); - assert_eq!(id_col.value(3), 1); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs deleted file mode 100644 index 40ad9fad00..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ /dev/null @@ -1,653 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Formatter}; -use std::str::FromStr; -use std::sync::Arc; - -use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::Result as DFResult; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::error::DataFusionError; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, - execute_input_stream, -}; -use futures::StreamExt; -use iceberg::arrow::FieldMatchMode; -use iceberg::spec::{DataFileFormat, serialize_data_file_to_json}; -use iceberg::table::Table; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; -use iceberg::{Error, ErrorKind}; -use uuid::Uuid; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::task_writer::TaskWriter; -use crate::to_datafusion_error; - -/// An execution plan node that writes data to an Iceberg table. -/// -/// This execution plan takes input data from a child execution plan and writes it to an Iceberg table. -/// It handles the creation of data files in the appropriate format and returns information about the written files as its output. -/// -/// The output of this execution plan is a record batch containing a single column with serialized -/// data file information that can be used for committing the write operation to the table. -#[derive(Debug)] -pub(crate) struct IcebergWriteExec { - table: Table, - input: Arc, - result_schema: ArrowSchemaRef, - plan_properties: Arc, -} - -impl IcebergWriteExec { - pub fn new(table: Table, input: Arc) -> Self { - let plan_properties = Self::compute_properties(&input, Self::make_result_schema()); - - Self { - table, - input, - result_schema: Self::make_result_schema(), - plan_properties, - } - } - - fn compute_properties( - input: &Arc, - schema: ArrowSchemaRef, - ) -> Arc { - Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), - EmissionType::Final, - Boundedness::Bounded, - )) - } - - // Create a record batch with serialized data files - fn make_result_batch(data_files: Vec) -> DFResult { - let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; - - RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError( - Box::new(e), - Some("Failed to make result batch".to_string()), - ) - }) - } - - fn make_result_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])) - } -} - -impl DisplayAs for IcebergWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergWriteExec: table={}, result_schema={:?}", - self.table.identifier(), - self.result_schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergWriteExec { - fn name(&self) -> &str { - "IcebergWriteExec" - } - - /// Prevents the introduction of additional `RepartitionExec` and processing input in parallel. - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - - fn maintains_input_order(&self) -> Vec { - // Maintains ordering in the sense that the written file will reflect the ordering of the input. - vec![true; self.children().len()] - } - - fn properties(&self) -> &Arc { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergWriteExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(Self::new( - self.table.clone(), - Arc::clone(&children[0]), - ))) - } - - /// Executes the write operation for the given partition. - /// - /// This function: - /// 1. Sets up a data file writer based on the table's configuration - /// 2. Processes input data from the child execution plan - /// 3. Writes the data to files using the configured writer - /// 4. Returns a stream containing information about the written data files - /// - /// The output of this function is a stream of record batches with the following structure: - /// - /// ```text - /// +------------------+ - /// | data_files | - /// +------------------+ - /// | "{"file_path":.. | <- JSON string representing a data file - /// +------------------+ - /// ``` - /// - /// Each row in the output contains a JSON string representing a data file that was written. - /// - /// This output can be used by a subsequent operation to commit the added files to the table. - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - let partition_type = self.table.metadata().default_partition_type().clone(); - let format_version = self.table.metadata().format_version(); - - // Get typed table properties - let table_props = self.table.metadata().table_properties(); - - // Check data file format - let write_format_default = table_props - .write_format_default() - .map_err(to_datafusion_error)?; - let file_format = - DataFileFormat::from_str(&write_format_default).map_err(to_datafusion_error)?; - if file_format != DataFileFormat::Parquet { - return Err(to_datafusion_error(Error::new( - ErrorKind::FeatureUnsupported, - format!("File format {file_format} is not supported for insert_into yet!"), - ))); - } - - // Build the writer from the already-parsed table properties so it honors - // `write.parquet.*` settings (e.g. CDC). Arrow batches flowing through - // DataFusion carry no field-id metadata, so match fields by name. - let mut parquet_file_writer_builder = ParquetWriterBuilder::from_table_properties( - &table_props, - self.table.metadata().current_schema().clone(), - ) - .map_err(to_datafusion_error)? - .with_match_mode(FieldMatchMode::Name); - if let Some(encryption_manager) = self.table.encryption_manager() { - parquet_file_writer_builder = - parquet_file_writer_builder.with_encryption_manager(encryption_manager.clone()); - } - let target_file_size = table_props - .write_target_file_size_bytes() - .map_err(to_datafusion_error)?; - - let file_io = self.table.file_io().clone(); - // todo location_gen and file_name_gen should be configurable - let location_generator = - DefaultLocationGenerator::new(self.table.metadata()).map_err(to_datafusion_error)?; - // todo filename prefix/suffix should be configurable - let file_name_generator = - DefaultFileNameGenerator::new(Uuid::now_v7().to_string(), None, file_format); - let rolling_writer_builder = RollingFileWriterBuilder::new( - parquet_file_writer_builder, - target_file_size, - file_io, - location_generator, - file_name_generator, - ); - let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); - - // Create TaskWriter - let fanout_enabled = table_props - .write_datafusion_fanout_enabled() - .map_err(to_datafusion_error)?; - let schema = self.table.metadata().current_schema().clone(); - let partition_spec = self.table.metadata().default_partition_spec().clone(); - let task_writer = TaskWriter::try_new( - data_file_writer_builder, - fanout_enabled, - schema.clone(), - partition_spec, - ) - .map_err(to_datafusion_error)?; - - // Get input data - let data = execute_input_stream( - Arc::clone(&self.input), - self.input.schema(), // input schema may have projected column `_partition` - partition, - Arc::clone(&context), - )?; - - // Create write stream - let stream = futures::stream::once(async move { - let mut task_writer = task_writer; - let mut input_stream = data; - - while let Some(batch) = input_stream.next().await { - let batch = batch?; - task_writer - .write(batch) - .await - .map_err(to_datafusion_error)?; - } - - let data_files = task_writer.close().await.map_err(to_datafusion_error)?; - - // Convert builders to data files and then to JSON strings - let data_files_strs: Vec = data_files - .into_iter() - .map(|data_file| { - serialize_data_file_to_json(data_file, &partition_type, format_version) - .map_err(to_datafusion_error) - }) - .collect::>>()?; - - Self::make_result_batch(data_files_strs) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.result_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::fmt::{Debug, Formatter}; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; - use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, - }; - use datafusion::common::Result as DFResult; - use datafusion::execution::{SendableRecordBatchStream, TaskContext}; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use futures::{StreamExt, stream}; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataFileFormat, NestedField, PrimitiveType, Schema, Type, deserialize_data_file_from_json, - }; - use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation}; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use tempfile::TempDir; - - use super::*; - - /// A simple execution plan that returns a predefined set of record batches - struct MockExecutionPlan { - schema: ArrowSchemaRef, - batches: Vec, - properties: Arc, - } - - impl MockExecutionPlan { - fn new(schema: ArrowSchemaRef, batches: Vec) -> Self { - let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - )); - - Self { - schema, - batches, - properties, - } - } - } - - impl Debug for MockExecutionPlan { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "MockExecutionPlan") - } - } - - impl DisplayAs for MockExecutionPlan { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockExecutionPlan") - } - } - } - } - - impl ExecutionPlan for MockExecutionPlan { - fn name(&self) -> &str { - "MockExecutionPlan" - } - - fn properties(&self) -> &Arc { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> DFResult, - ) -> DFResult { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let batches = self.batches.clone(); - let stream = stream::iter(batches.into_iter().map(Ok)); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema.clone(), - stream.boxed(), - ))) - } - } - - /// Helper function to create a temporary directory and return its path - fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() - } - - /// Helper function to create a memory catalog - async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() - } - - /// Helper function to create a test table schema - fn get_test_schema() -> Result { - Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - } - - /// Helper function to create a table creation - fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Schema, - ) -> TableCreation { - TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build() - } - - #[tokio::test] - async fn test_iceberg_write_exec() -> Result<()> { - // 1. Set up test environment - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_namespace".to_string()); - - // Create namespace - iceberg_catalog - .create_namespace(&namespace, HashMap::new()) - .await?; - - // Create schema - let schema = get_test_schema()?; - - // Create table - let table_name = "test_table"; - let table_location = temp_path(); - let creation = get_table_creation(table_location, table_name, schema); - let table = iceberg_catalog.create_table(&namespace, creation).await?; - - // 2. Create test data - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - ])); - - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; - let name_array = Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])) as ArrayRef; - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_array, name_array]) - .map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to create record batch: {e}"), - ) - })?; - - // 3. Create mock input execution plan - let input_plan = Arc::new(MockExecutionPlan::new(arrow_schema.clone(), vec![ - batch.clone(), - ])); - - // 4. Create IcebergWriteExec - let write_exec = IcebergWriteExec::new(table.clone(), input_plan); - - // 5. Execute the plan - let task_ctx = Arc::new(TaskContext::default()); - let stream = write_exec.execute(0, task_ctx).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to execute plan: {e}"), - ) - })?; - - // Collect the results - let mut results = vec![]; - let mut stream = stream; - while let Some(batch) = stream.next().await { - results.push(batch.map_err(|e| { - Error::new(ErrorKind::Unexpected, format!("Failed to get batch: {e}")) - })?); - } - - // 6. Verify the results - assert_eq!(results.len(), 1, "Expected one result batch"); - let result_batch = &results[0]; - - // Check schema - assert_eq!( - result_batch.schema().as_ref(), - &ArrowSchema::new(vec![Field::new(DATA_FILES_COL_NAME, DataType::Utf8, false)]) - ); - - // Check data - assert_eq!(result_batch.num_rows(), 1, "Expected one data file"); - - // Get the data file JSON - let data_file_json = result_batch - .column(0) - .as_any() - .downcast_ref::() - .expect("Expected StringArray") - .value(0); - - // Deserialize the data file JSON - let partition_type = table.metadata().default_partition_type(); - let spec_id = table.metadata().default_partition_spec_id(); - let schema = table.metadata().current_schema(); - - let data_file = - deserialize_data_file_from_json(data_file_json, spec_id, partition_type, schema) - .expect("Failed to deserialize data file JSON"); - - // Verify data file properties - assert_eq!( - data_file.record_count(), - 3, - "Expected 3 records in the data file" - ); - assert!( - data_file.file_size_in_bytes() > 0, - "File size should be greater than 0" - ); - assert_eq!( - data_file.file_format(), - DataFileFormat::Parquet, - "Expected Parquet file format" - ); - - // Verify column statistics - assert!( - data_file.column_sizes().get(&1).unwrap() > &0, - "Column 1 size should be greater than 0" - ); - assert!( - data_file.column_sizes().get(&2).unwrap() > &0, - "Column 2 size should be greater than 0" - ); - - assert_eq!( - *data_file.value_counts().get(&1).unwrap(), - 3, - "Expected 3 values for column 1" - ); - assert_eq!( - *data_file.value_counts().get(&2).unwrap(), - 3, - "Expected 3 values for column 2" - ); - - // Verify lower and upper bounds - assert!( - data_file.lower_bounds().contains_key(&1) || data_file.lower_bounds().contains_key(&2), - "Expected lower bounds to contain at least one column" - ); - assert!( - data_file.upper_bounds().contains_key(&1) || data_file.upper_bounds().contains_key(&2), - "Expected upper bounds to contain at least one column" - ); - - // Check that the file path exists - let file_path = data_file.file_path(); - assert!(!file_path.is_empty(), "File path should not be empty"); - - // 7. Verify the file exists - let file_io = table.file_io(); - assert!(file_io.exists(file_path).await?, "Data file should exist"); - - Ok(()) - } - - #[tokio::test] - async fn test_iceberg_write_exec_advertises_result_schema() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_namespace".to_string()); - iceberg_catalog - .create_namespace(&namespace, HashMap::new()) - .await?; - let creation = get_table_creation(temp_path(), "test_table", get_test_schema()?); - let table = iceberg_catalog.create_table(&namespace, creation).await?; - - let table_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, false), - ])); - let input = Arc::new(MockExecutionPlan::new(table_schema.clone(), vec![])); - - let write_exec = IcebergWriteExec::new(table, input); - - assert_eq!( - write_exec.schema().as_ref(), - &ArrowSchema::new(vec![Field::new(DATA_FILES_COL_NAME, DataType::Utf8, false)]), - "IcebergWriteExec should advertise the data_files schema, not the table schema" - ); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/schema.rs b/crates/integrations/datafusion/src/schema.rs deleted file mode 100644 index 545863f8c6..0000000000 --- a/crates/integrations/datafusion/src/schema.rs +++ /dev/null @@ -1,447 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use async_trait::async_trait; -use dashmap::DashMap; -use datafusion::catalog::SchemaProvider; -use datafusion::datasource::TableProvider; -use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::execution::TaskContext; -use datafusion::prelude::SessionContext; -use futures::StreamExt; -use futures::future::try_join_all; -use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids; -use iceberg::inspect::MetadataTableType; -use iceberg::spec::FormatVersion; -use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableCreation, TableIdent}; - -use crate::table::IcebergTableProvider; -use crate::to_datafusion_error; - -/// Represents a [`SchemaProvider`] for the Iceberg [`Catalog`], managing -/// access to table providers within a specific namespace. -#[derive(Debug)] -pub(crate) struct IcebergSchemaProvider { - /// Reference to the Iceberg catalog - catalog: Arc, - /// The namespace this schema represents - namespace: NamespaceIdent, - /// A concurrent map where keys are table names - /// and values are dynamic references to objects implementing the - /// [`TableProvider`] trait. - /// Wrapped in Arc to allow sharing across async boundaries in register_table. - tables: Arc>>, -} - -impl IcebergSchemaProvider { - /// Asynchronously tries to construct a new [`IcebergSchemaProvider`] - /// using the given client to fetch and initialize table providers for - /// the provided namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves a list of table names - /// attempts to create a table provider for each table name, and - /// collects these providers into a `HashMap`. - pub(crate) async fn try_new( - client: Arc, - namespace: NamespaceIdent, - ) -> Result { - // TODO: - // Tables and providers should be cached based on table_name - // if we have a cache miss; we update our internal cache & check again - // As of right now; tables might become stale. - let table_names: Vec<_> = client - .list_tables(&namespace) - .await? - .iter() - .map(|tbl| tbl.name().to_string()) - .collect(); - - let providers = try_join_all( - table_names - .iter() - .map(|name| IcebergTableProvider::try_new(client.clone(), namespace.clone(), name)) - .collect::>(), - ) - .await?; - - let tables = Arc::new(DashMap::new()); - for (name, provider) in table_names.into_iter().zip(providers) { - tables.insert(name, Arc::new(provider)); - } - - Ok(IcebergSchemaProvider { - catalog: client, - namespace, - tables, - }) - } -} - -#[async_trait] -impl SchemaProvider for IcebergSchemaProvider { - fn table_names(&self) -> Vec { - self.tables - .iter() - .flat_map(|entry| { - let table_name = entry.key().clone(); - [table_name.clone()] - .into_iter() - .chain( - MetadataTableType::all_types().map(move |metadata_table_name| { - format!("{}${}", table_name, metadata_table_name.as_str()) - }), - ) - }) - .collect() - } - - fn table_exist(&self, name: &str) -> bool { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - self.tables.contains_key(table_name) - && MetadataTableType::try_from(metadata_table_name).is_ok() - } else { - self.tables.contains_key(name) - } - } - - async fn table(&self, name: &str) -> DFResult>> { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - let metadata_table_type = - MetadataTableType::try_from(metadata_table_name).map_err(DataFusionError::Plan)?; - if let Some(table) = self.tables.get(table_name) { - let metadata_table = table - .metadata_table(metadata_table_type) - .await - .map_err(to_datafusion_error)?; - return Ok(Some(Arc::new(metadata_table))); - } else { - return Ok(None); - } - } - - Ok(self - .tables - .get(name) - .map(|entry| entry.value().clone() as Arc)) - } - - fn register_table( - &self, - name: String, - table: Arc, - ) -> DFResult>> { - // Check if table already exists - if self.table_exist(name.as_str()) { - return Err(DataFusionError::Execution(format!( - "Table {name} already exists" - ))); - } - - // Convert DataFusion schema to Iceberg schema - // DataFusion schemas don't have field IDs, so we use the function that assigns them automatically - let df_schema = table.schema(); - let iceberg_schema = arrow_schema_to_schema_auto_assign_ids(df_schema.as_ref()) - .map_err(to_datafusion_error)?; - - // Use at least V2, and upgrade to V3 if the schema requires it (e.g. timestamp_ns / variant). - let format_version = iceberg_schema - .calc_min_compatible_format() - .max(FormatVersion::V2); - - // Create the table in the Iceberg catalog - let table_creation = TableCreation::builder() - .name(name.clone()) - .schema(iceberg_schema) - .format_version(format_version) - .build(); - - let catalog = self.catalog.clone(); - let namespace = self.namespace.clone(); - let tables = self.tables.clone(); - let name_clone = name.clone(); - - // Use tokio's spawn_blocking to handle the async work on a blocking thread pool - let result = tokio::task::spawn_blocking(move || { - // Create a new runtime handle to execute the async work - let rt = tokio::runtime::Handle::current(); - rt.block_on(async move { - // Verify the input table is empty - CREATE TABLE only accepts schema definition - ensure_table_is_empty(&table) - .await - .map_err(to_datafusion_error)?; - - catalog - .create_table(&namespace, table_creation) - .await - .map_err(to_datafusion_error)?; - - // Create a new table provider using the catalog reference - let table_provider = IcebergTableProvider::try_new( - catalog.clone(), - namespace.clone(), - name_clone.clone(), - ) - .await - .map_err(to_datafusion_error)?; - - // Store the new table provider - tables.insert(name_clone, Arc::new(table_provider)); - - Ok(None) - }) - }); - - // Block on the spawned task to get the result - // This is safe because spawn_blocking moves the blocking to a dedicated thread pool - futures::executor::block_on(result).map_err(|e| { - DataFusionError::Execution(format!("Failed to create Iceberg table: {e}")) - })? - } - - fn deregister_table(&self, name: &str) -> DFResult>> { - // Check if table exists - if !self.table_exist(name) { - return Ok(None); - } - - let catalog = self.catalog.clone(); - let namespace = self.namespace.clone(); - let tables = self.tables.clone(); - let table_name = name.to_string(); - - // Use tokio's spawn_blocking to handle the async work on a blocking thread pool - let result = tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Handle::current(); - rt.block_on(async move { - let table_ident = TableIdent::new(namespace, table_name.clone()); - - // Drop the table from the Iceberg catalog - catalog - .drop_table(&table_ident) - .await - .map_err(to_datafusion_error)?; - - // Remove from local cache and return the removed provider - let removed = tables - .remove(&table_name) - .map(|(_, table)| table as Arc); - - Ok(removed) - }) - }); - - futures::executor::block_on(result) - .map_err(|e| DataFusionError::Execution(format!("Failed to drop Iceberg table: {e}")))? - } -} - -/// Verifies that a table provider contains no data by scanning with LIMIT 1. -/// Returns an error if the table has any rows. -async fn ensure_table_is_empty(table: &Arc) -> Result<()> { - let session_ctx = SessionContext::new(); - let exec_plan = table - .scan(&session_ctx.state(), None, &[], Some(1)) - .await - .map_err(|e| Error::new(ErrorKind::Unexpected, format!("Failed to scan table: {e}")))?; - - let task_ctx = Arc::new(TaskContext::default()); - let stream = exec_plan.execute(0, task_ctx).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to execute scan: {e}"), - ) - })?; - - let batches: Vec<_> = stream.collect().await; - let has_data = batches - .into_iter() - .filter_map(|r| r.ok()) - .any(|batch| batch.num_rows() > 0); - - if has_data { - return Err(Error::new( - ErrorKind::Unexpected, - "register_table does not support tables with data.", - )); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::arrow::array::{Int32Array, StringArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; - use datafusion::arrow::record_batch::RecordBatch; - use datafusion::datasource::MemTable; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent}; - use tempfile::TempDir; - - use super::*; - - async fn create_test_schema_provider() -> (IcebergSchemaProvider, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let provider = IcebergSchemaProvider::try_new(Arc::new(catalog), namespace) - .await - .unwrap(); - - (provider, temp_dir) - } - - #[tokio::test] - async fn test_register_table_with_data_fails() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create a MemTable with data - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, true), - ])); - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), - ]) - .unwrap(); - - let mem_table = MemTable::try_new(arrow_schema, vec![vec![batch]]).unwrap(); - - // Attempt to register the table with data - should fail - let result = schema_provider.register_table("test_table".to_string(), Arc::new(mem_table)); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!( - err.to_string() - .contains("register_table does not support tables with data."), - "Expected error about tables with data, got: {err}", - ); - } - - #[tokio::test] - async fn test_register_empty_table_succeeds() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create an empty MemTable (schema only, no data rows) - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("name", DataType::Utf8, true), - ])); - - // Create an empty batch (0 rows) - MemTable requires at least one partition - let empty_batch = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table = MemTable::try_new(arrow_schema, vec![vec![empty_batch]]).unwrap(); - - // Attempt to register the empty table - should succeed - let result = schema_provider.register_table("empty_table".to_string(), Arc::new(mem_table)); - - assert!(result.is_ok(), "Expected success, got: {result:?}"); - - // Verify the table was registered - assert!(schema_provider.table_exist("empty_table")); - } - - #[tokio::test] - async fn test_register_duplicate_table_fails() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create empty MemTables - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int32, - false, - )])); - - let empty_batch1 = RecordBatch::new_empty(arrow_schema.clone()); - let empty_batch2 = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table1 = MemTable::try_new(arrow_schema.clone(), vec![vec![empty_batch1]]).unwrap(); - let mem_table2 = MemTable::try_new(arrow_schema, vec![vec![empty_batch2]]).unwrap(); - - // Register first table - should succeed - let result1 = schema_provider.register_table("dup_table".to_string(), Arc::new(mem_table1)); - assert!(result1.is_ok()); - - // Register second table with same name - should fail - let result2 = schema_provider.register_table("dup_table".to_string(), Arc::new(mem_table2)); - assert!(result2.is_err()); - let err = result2.unwrap_err(); - assert!( - err.to_string().contains("already exists"), - "Expected error about table already existing, got: {err}", - ); - } - - #[tokio::test] - async fn test_deregister_table_succeeds() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Create and register an empty table - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int32, - false, - )])); - - let empty_batch = RecordBatch::new_empty(arrow_schema.clone()); - let mem_table = MemTable::try_new(arrow_schema, vec![vec![empty_batch]]).unwrap(); - - // Register the table - let result = schema_provider.register_table("drop_me".to_string(), Arc::new(mem_table)); - assert!(result.is_ok()); - assert!(schema_provider.table_exist("drop_me")); - - // Deregister the table - let result = schema_provider.deregister_table("drop_me"); - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - - // Verify the table no longer exists - assert!(!schema_provider.table_exist("drop_me")); - } - - #[tokio::test] - async fn test_deregister_nonexistent_table_returns_none() { - let (schema_provider, _temp_dir) = create_test_schema_provider().await; - - // Attempt to deregister a table that doesn't exist - let result = schema_provider.deregister_table("nonexistent"); - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - } -} diff --git a/crates/integrations/datafusion/src/table/metadata_table.rs b/crates/integrations/datafusion/src/table/metadata_table.rs deleted file mode 100644 index 56d0c10e35..0000000000 --- a/crates/integrations/datafusion/src/table/metadata_table.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use futures::TryStreamExt; -use futures::stream::BoxStream; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::table::Table; - -use crate::physical_plan::metadata_scan::IcebergMetadataScan; -use crate::to_datafusion_error; - -/// Represents a [`TableProvider`] for the Iceberg [`iceberg::Catalog`], -/// managing access to a [`iceberg::inspect::MetadataTable`]. -#[derive(Debug, Clone)] -pub struct IcebergMetadataTableProvider { - pub(crate) table: Table, - pub(crate) r#type: MetadataTableType, -} - -#[async_trait] -impl TableProvider for IcebergMetadataTableProvider { - fn schema(&self) -> ArrowSchemaRef { - let metadata_table = self.table.inspect(); - let schema = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().schema(), - MetadataTableType::Manifests => metadata_table.manifests().schema(), - MetadataTableType::History => metadata_table.history().schema(), - }; - schema_to_arrow_schema(&schema).unwrap().into() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - _projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - Ok(Arc::new(IcebergMetadataScan::new(self.clone()))) - } -} - -impl IcebergMetadataTableProvider { - pub async fn scan(self) -> DFResult>> { - let metadata_table = self.table.inspect(); - let stream = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().scan().await, - MetadataTableType::Manifests => metadata_table.manifests().scan().await, - MetadataTableType::History => metadata_table.history().scan().await, - } - .map_err(to_datafusion_error)?; - let stream = stream.map_err(to_datafusion_error); - Ok(Box::pin(stream)) - } -} diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs deleted file mode 100644 index 9de7bcb9c2..0000000000 --- a/crates/integrations/datafusion/src/table/mod.rs +++ /dev/null @@ -1,893 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Iceberg table providers for DataFusion. -//! -//! This module provides two table provider implementations: -//! -//! - [`IcebergTableProvider`]: Catalog-backed provider with automatic metadata refresh. -//! Use for write operations and when you need to see the latest table state. -//! -//! - [`IcebergStaticTableProvider`]: Static provider for read-only access to a specific -//! table snapshot. Use for consistent analytical queries or time-travel scenarios. - -pub mod metadata_table; -pub mod table_provider_factory; - -use std::num::NonZeroUsize; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::common::DataFusionError; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::spec::TableProperties; -use iceberg::table::Table; -use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableIdent}; -use metadata_table::IcebergMetadataTableProvider; - -use crate::error::to_datafusion_error; -use crate::physical_plan::commit::IcebergCommitExec; -use crate::physical_plan::project::project_with_partition; -use crate::physical_plan::repartition::repartition; -use crate::physical_plan::scan::IcebergTableScan; -use crate::physical_plan::sort::sort_by_partition; -use crate::physical_plan::write::IcebergWriteExec; - -/// Catalog-backed table provider with automatic metadata refresh. -/// -/// This provider loads fresh table metadata from the catalog on every scan and write -/// operation, ensuring you always see the latest table state. Use this when you need -/// write operations or want to see the most up-to-date data. -/// -/// For read-only access to a specific snapshot without catalog overhead, use -/// [`IcebergStaticTableProvider`] instead. -#[derive(Debug, Clone)] -pub struct IcebergTableProvider { - /// The catalog that manages this table - catalog: Arc, - /// The table identifier (namespace + name) - table_ident: TableIdent, - /// A reference-counted arrow `Schema` (cached at construction) - schema: ArrowSchemaRef, -} - -impl IcebergTableProvider { - /// Creates a new catalog-backed table provider. - /// - /// Loads the table once to get the initial schema, then stores the catalog - /// reference for future metadata refreshes on each operation. - pub(crate) async fn try_new( - catalog: Arc, - namespace: NamespaceIdent, - name: impl Into, - ) -> Result { - let table_ident = TableIdent::new(namespace, name.into()); - - // Load table once to get initial schema - let table = catalog.load_table(&table_ident).await?; - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - - Ok(IcebergTableProvider { - catalog, - table_ident, - schema, - }) - } - - pub(crate) async fn metadata_table( - &self, - r#type: MetadataTableType, - ) -> Result { - // Load fresh table metadata for metadata table access - let table = self.catalog.load_table(&self.table_ident).await?; - Ok(IcebergMetadataTableProvider { table, r#type }) - } -} - -#[async_trait] -impl TableProvider for IcebergTableProvider { - fn schema(&self) -> ArrowSchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> DFResult> { - // Load fresh table metadata from catalog - let table = self - .catalog - .load_table(&self.table_ident) - .await - .map_err(to_datafusion_error)?; - - // Create scan with fresh metadata (always use current snapshot) - Ok(Arc::new(IcebergTableScan::new( - table, - None, // Always use current snapshot for catalog-backed provider - self.schema.clone(), - projection, - filters, - limit, - ))) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - // Push down all filters, as a single source of truth, the scanner will drop the filters which couldn't be push down - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - state: &dyn Session, - input: Arc, - _insert_op: InsertOp, - ) -> DFResult> { - if _insert_op != InsertOp::Append { - return Err(DataFusionError::NotImplemented(format!( - "IcebergTableProvider supports only append inserts, got {_insert_op}" - ))); - } - - // Load fresh table metadata from catalog - let table = self - .catalog - .load_table(&self.table_ident) - .await - .map_err(to_datafusion_error)?; - - let partition_spec = table.metadata().default_partition_spec(); - - // Step 1: Project partition values for partitioned tables - let plan_with_partition = if !partition_spec.is_unpartitioned() { - project_with_partition(input, &table)? - } else { - input - }; - - // Step 2: Repartition for parallel processing - let target_partitions = - NonZeroUsize::new(state.config().target_partitions()).ok_or_else(|| { - DataFusionError::Configuration( - "target_partitions must be greater than 0".to_string(), - ) - })?; - - let repartitioned_plan = - repartition(plan_with_partition, table.metadata_ref(), target_partitions)?; - - // Apply sort node when it's not fanout mode - let fanout_enabled = table - .metadata() - .properties() - .get(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED) - .map(|value| { - value - .parse::() - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid value for {}, expected 'true' or 'false'", - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED - ), - ) - .with_source(e) - }) - .map_err(to_datafusion_error) - }) - .transpose()? - .unwrap_or(TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT); - - let write_input = if fanout_enabled { - repartitioned_plan - } else { - sort_by_partition(repartitioned_plan)? - }; - - let write_plan = Arc::new(IcebergWriteExec::new(table.clone(), write_input)); - - // Merge the outputs of write_plan into one so we can commit all files together - let coalesce_partitions = Arc::new(CoalescePartitionsExec::new(write_plan)); - - Ok(Arc::new(IcebergCommitExec::new( - table, - self.catalog.clone(), - coalesce_partitions, - self.schema.clone(), - ))) - } -} - -/// Static table provider for read-only snapshot access. -/// -/// This provider holds a cached table instance and does not refresh metadata or support -/// write operations. Use this for consistent analytical queries, time-travel scenarios, -/// or when you want to avoid catalog overhead. -/// -/// For catalog-backed tables with write support and automatic refresh, use -/// [`IcebergTableProvider`] instead. -#[derive(Debug, Clone)] -pub struct IcebergStaticTableProvider { - /// The static table instance (never refreshed) - table: Table, - /// Optional snapshot ID for this static view - snapshot_id: Option, - /// A reference-counted arrow `Schema` - schema: ArrowSchemaRef, -} - -impl IcebergStaticTableProvider { - /// Creates a static provider from a table instance. - /// - /// Uses the table's current snapshot for all queries. Does not support write operations. - pub async fn try_new_from_table(table: Table) -> Result { - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - Ok(IcebergStaticTableProvider { - table, - snapshot_id: None, - schema, - }) - } - - /// Creates a static provider for a specific table snapshot. - /// - /// Queries the specified snapshot for all operations. Useful for time-travel queries. - /// Does not support write operations. - pub async fn try_new_from_table_snapshot(table: Table, snapshot_id: i64) -> Result { - let snapshot = table - .metadata() - .snapshot_by_id(snapshot_id) - .ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - format!( - "snapshot id {snapshot_id} not found in table {}", - table.identifier().name() - ), - ) - })?; - let table_schema = snapshot.schema(table.metadata())?; - let schema = Arc::new(schema_to_arrow_schema(&table_schema)?); - Ok(IcebergStaticTableProvider { - table, - snapshot_id: Some(snapshot_id), - schema, - }) - } -} - -#[async_trait] -impl TableProvider for IcebergStaticTableProvider { - fn schema(&self) -> ArrowSchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> DFResult> { - // Use cached table (no refresh) - Ok(Arc::new(IcebergTableScan::new( - self.table.clone(), - self.snapshot_id, - self.schema.clone(), - projection, - filters, - limit, - ))) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - // Push down all filters, as a single source of truth, the scanner will drop the filters which couldn't be push down - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - _state: &dyn Session, - _input: Arc, - _insert_op: InsertOp, - ) -> DFResult> { - Err(to_datafusion_error(Error::new( - ErrorKind::FeatureUnsupported, - "Write operations are not supported on IcebergStaticTableProvider. \ - Use IcebergTableProvider with a catalog for write support." - .to_string(), - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::common::Column; - use datafusion::physical_plan::ExecutionPlan; - use datafusion::prelude::SessionContext; - use iceberg::io::FileIO; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; - use iceberg::table::{StaticTable, Table}; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; - use tempfile::TempDir; - - use super::*; - - async fn get_test_table_from_metadata_file() -> Table { - let metadata_file_name = "TableMetadataV2Valid.json"; - let metadata_file_path = format!( - "{}/tests/test_data/{}", - env!("CARGO_MANIFEST_DIR"), - metadata_file_name - ); - let file_io = FileIO::new_with_fs(); - let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); - let static_table = - StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io) - .await - .unwrap(); - static_table.into_table() - } - - async fn get_test_catalog_and_table() -> (Arc, NamespaceIdent, String, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let table_creation = TableCreation::builder() - .name("test_table".to_string()) - .location(format!("{warehouse_path}/test_table")) - .schema(schema) - .properties(HashMap::new()) - .build(); - - catalog - .create_table(&namespace, table_creation) - .await - .unwrap(); - - ( - Arc::new(catalog), - namespace, - "test_table".to_string(), - temp_dir, - ) - } - - // Tests for IcebergStaticTableProvider - - #[tokio::test] - async fn test_static_provider_from_table() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_static_provider_from_snapshot() { - let table = get_test_table_from_metadata_file().await; - let snapshot_id = table.metadata().snapshots().next().unwrap().snapshot_id(); - let table_provider = - IcebergStaticTableProvider::try_new_from_table_snapshot(table.clone(), snapshot_id) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_static_provider_rejects_writes() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - - // Attempt to insert into the static provider should fail - let result = ctx.sql("INSERT INTO mytable VALUES (1, 2, 3)").await; - - // The error should occur during planning or execution - // We expect an error indicating write operations are not supported - assert!( - result.is_err() || { - let df = result.unwrap(); - df.collect().await.is_err() - } - ); - } - - #[tokio::test] - async fn test_static_provider_scan() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - - // Test that scan operations work correctly - let df = ctx.sql("SELECT count(*) FROM mytable").await.unwrap(); - let physical_plan = df.create_physical_plan().await; - assert!(physical_plan.is_ok()); - } - - // Tests for IcebergTableProvider - - #[tokio::test] - async fn test_catalog_backed_provider_creation() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - // Test creating a catalog-backed provider - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - // Verify the schema is loaded correctly - let schema = provider.schema(); - assert_eq!(schema.fields().len(), 2); - assert_eq!(schema.field(0).name(), "id"); - assert_eq!(schema.field(1).name(), "name"); - } - - #[tokio::test] - async fn test_catalog_backed_provider_scan() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Test that scan operations work correctly - let df = ctx.sql("SELECT * FROM test_table").await.unwrap(); - - // Verify the schema in the query result - let df_schema = df.schema(); - assert_eq!(df_schema.fields().len(), 2); - assert_eq!(df_schema.field(0).name(), "id"); - assert_eq!(df_schema.field(1).name(), "name"); - - let physical_plan = df.create_physical_plan().await; - assert!(physical_plan.is_ok()); - } - - #[tokio::test] - async fn test_catalog_backed_provider_insert() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Test that insert operations work correctly - let result = ctx.sql("INSERT INTO test_table VALUES (1, 'test')").await; - - // Insert should succeed (or at least not fail during planning) - assert!(result.is_ok()); - - // Try to execute the insert plan - let df = result.unwrap(); - let execution_result = df.collect().await; - - // The execution should succeed - assert!(execution_result.is_ok()); - } - - #[tokio::test] - async fn test_physical_input_schema_consistent_with_logical_input_schema() { - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - ctx.register_table("test_table", Arc::new(provider)) - .unwrap(); - - // Create a query plan - let df = ctx.sql("SELECT id, name FROM test_table").await.unwrap(); - - // Get logical schema before consuming df - let logical_schema = df.schema().clone(); - - // Get physical plan (this consumes df) - let physical_plan = df.create_physical_plan().await.unwrap(); - let physical_schema = physical_plan.schema(); - - // Verify that logical and physical schemas are consistent - assert_eq!( - logical_schema.fields().len(), - physical_schema.fields().len() - ); - - for (logical_field, physical_field) in logical_schema - .fields() - .iter() - .zip(physical_schema.fields().iter()) - { - assert_eq!(logical_field.name(), physical_field.name()); - assert_eq!(logical_field.data_type(), physical_field.data_type()); - } - } - - async fn get_partitioned_test_catalog_and_table( - fanout_enabled: Option, - ) -> (Arc, NamespaceIdent, String, TempDir) { - use iceberg::spec::{Transform, UnboundPartitionSpec}; - - let temp_dir = TempDir::new().unwrap(); - let warehouse_path = temp_dir.path().to_str().unwrap().to_string(); - - let catalog = MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_path.clone())]), - ) - .await - .unwrap(); - - let namespace = NamespaceIdent::new("test_ns".to_string()); - catalog - .create_namespace(&namespace, HashMap::new()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(); - - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity) - .unwrap() - .build(); - - let mut properties = HashMap::new(); - if let Some(enabled) = fanout_enabled { - properties.insert( - TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED.to_string(), - enabled.to_string(), - ); - } - - let table_creation = TableCreation::builder() - .name("partitioned_table".to_string()) - .location(format!("{warehouse_path}/partitioned_table")) - .schema(schema) - .partition_spec(partition_spec) - .properties(properties) - .build(); - - catalog - .create_table(&namespace, table_creation) - .await - .unwrap(); - - ( - Arc::new(catalog), - namespace, - "partitioned_table".to_string(), - temp_dir, - ) - } - - /// Helper to check if a plan contains a SortExec node - fn plan_contains_sort(plan: &Arc) -> bool { - if plan.name() == "SortExec" { - return true; - } - for child in plan.children() { - if plan_contains_sort(child) { - return true; - } - } - false - } - - #[tokio::test] - async fn test_catalog_backed_provider_rejects_non_append_op() { - use datafusion::physical_plan::empty::EmptyExec; - - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - let provider = IcebergTableProvider::try_new(catalog, namespace, table_name) - .await - .unwrap(); - let ctx = SessionContext::new(); - - for (insert_op, expected_message) in [ - ( - InsertOp::Overwrite, - "IcebergTableProvider supports only append inserts, got Insert Overwrite", - ), - ( - InsertOp::Replace, - "IcebergTableProvider supports only append inserts, got Replace Into", - ), - ] { - let input = Arc::new(EmptyExec::new(provider.schema())) as Arc; - let error = provider - .insert_into(&ctx.state(), input, insert_op) - .await - .expect_err("non-append inserts should be rejected"); - - assert!( - matches!( - error, - DataFusionError::NotImplemented(ref message) if message == expected_message - ), - "unexpected error: {error}" - ); - } - } - - #[tokio::test] - async fn test_insert_plan_fanout_enabled_no_sort() { - use datafusion::datasource::TableProvider; - use datafusion::logical_expr::dml::InsertOp; - use datafusion::physical_plan::empty::EmptyExec; - - // When fanout is enabled (default), no sort node should be added - let (catalog, namespace, table_name, _temp_dir) = - get_partitioned_test_catalog_and_table(Some(true)).await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let input_schema = provider.schema(); - let input = Arc::new(EmptyExec::new(input_schema)) as Arc; - - let state = ctx.state(); - let insert_plan = provider - .insert_into(&state, input, InsertOp::Append) - .await - .unwrap(); - - // With fanout enabled, there should be no SortExec in the plan - assert!( - !plan_contains_sort(&insert_plan), - "Plan should NOT contain SortExec when fanout is enabled" - ); - } - - #[tokio::test] - async fn test_insert_plan_fanout_disabled_has_sort() { - use datafusion::datasource::TableProvider; - use datafusion::logical_expr::dml::InsertOp; - use datafusion::physical_plan::empty::EmptyExec; - - // When fanout is disabled, a sort node should be added - let (catalog, namespace, table_name, _temp_dir) = - get_partitioned_test_catalog_and_table(Some(false)).await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let input_schema = provider.schema(); - let input = Arc::new(EmptyExec::new(input_schema)) as Arc; - - let state = ctx.state(); - let insert_plan = provider - .insert_into(&state, input, InsertOp::Append) - .await - .unwrap(); - - // With fanout disabled, there should be a SortExec in the plan - assert!( - plan_contains_sort(&insert_plan), - "Plan should contain SortExec when fanout is disabled" - ); - } - - #[tokio::test] - async fn test_limit_pushdown_static_provider() { - use datafusion::datasource::TableProvider; - - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan with limit - let scan_plan = table_provider - .scan(&state, None, &[], Some(10)) - .await - .unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is set - assert_eq!( - iceberg_scan.limit(), - Some(10), - "Limit should be set to 10 in the scan plan" - ); - } - - #[tokio::test] - async fn test_limit_pushdown_catalog_backed_provider() { - use datafusion::datasource::TableProvider; - - let (catalog, namespace, table_name, _temp_dir) = get_test_catalog_and_table().await; - - let provider = - IcebergTableProvider::try_new(catalog.clone(), namespace.clone(), table_name.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan with limit - let scan_plan = provider.scan(&state, None, &[], Some(5)).await.unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is set - assert_eq!( - iceberg_scan.limit(), - Some(5), - "Limit should be set to 5 in the scan plan" - ); - } - - #[tokio::test] - async fn test_no_limit_pushdown() { - use datafusion::datasource::TableProvider; - - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergStaticTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - let state = ctx.state(); - - // Test scan without limit - let scan_plan = table_provider.scan(&state, None, &[], None).await.unwrap(); - - // Verify that the scan plan is an IcebergTableScan - let iceberg_scan = scan_plan - .downcast_ref::() - .expect("Expected IcebergTableScan"); - - // Verify the limit is None - assert_eq!( - iceberg_scan.limit(), - None, - "Limit should be None when not specified" - ); - } -} diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs deleted file mode 100644 index a3b293f33a..0000000000 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ /dev/null @@ -1,331 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::borrow::Cow; -use std::collections::HashMap; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::catalog::{Session, TableProvider, TableProviderFactory}; -use datafusion::common::TableReference; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::CreateExternalTable; -use iceberg::io::{FileIOBuilder, LocalFsStorageFactory, StorageFactory}; -use iceberg::table::StaticTable; -use iceberg::{Error, ErrorKind, Result, TableIdent}; - -use super::IcebergStaticTableProvider; -use crate::to_datafusion_error; - -/// A factory that implements DataFusion's `TableProviderFactory` to create `IcebergTableProvider` instances. -/// -/// # Example -/// -/// The following example demonstrates how to create an Iceberg external table using SQL in -/// a DataFusion session with `IcebergTableProviderFactory`: -/// -/// ``` -/// use std::sync::Arc; -/// -/// use datafusion::common::TableReference; -/// use datafusion::execution::session_state::SessionStateBuilder; -/// use datafusion::prelude::*; -/// use iceberg_datafusion::IcebergTableProviderFactory; -/// -/// #[tokio::main] -/// async fn main() { -/// // Create a new session context -/// let mut state = SessionStateBuilder::new().with_default_features().build(); -/// -/// // Register the IcebergTableProviderFactory in the session -/// state.table_factories_mut().insert( -/// "ICEBERG".to_string(), -/// Arc::new(IcebergTableProviderFactory::new()), -/// ); -/// -/// let ctx = SessionContext::new_with_state(state); -/// -/// // Define the table reference and the location of the Iceberg metadata file -/// let table_ref = TableReference::bare("my_iceberg_table"); -/// // /path/to/iceberg/metadata -/// let metadata_file_path = format!( -/// "{}/testdata/table_metadata/{}", -/// env!("CARGO_MANIFEST_DIR"), -/// "TableMetadataV2.json" -/// ); -/// -/// // SQL command to create the Iceberg external table -/// let sql = format!( -/// "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", -/// table_ref, metadata_file_path -/// ); -/// -/// // Execute the SQL to create the external table -/// ctx.sql(&sql).await.expect("Failed to create table"); -/// -/// // Verify the table was created by retrieving the table provider -/// let table_provider = ctx -/// .table_provider(table_ref) -/// .await -/// .expect("Table not found"); -/// -/// println!("Iceberg external table created successfully."); -/// } -/// ``` -/// -/// # Note -/// This factory is designed to work with the DataFusion query engine, -/// specifically for handling Iceberg tables in external table commands. -/// Currently, this implementation supports only reading Iceberg tables, with -/// the creation of new tables not yet available. -/// -/// # Errors -/// An error will be returned if any unsupported feature, such as partition columns, -/// order expressions, constraints, or column defaults, is detected in the table creation command. -#[derive(Debug, Default)] -pub struct IcebergTableProviderFactory { - storage_factory: Option>, -} - -impl IcebergTableProviderFactory { - pub fn new() -> Self { - Self { - storage_factory: None, - } - } - - /// Create a new factory with a custom storage factory for creating FileIO instances. - pub fn new_with_storage_factory(storage_factory: Arc) -> Self { - Self { - storage_factory: Some(storage_factory), - } - } -} - -#[async_trait] -impl TableProviderFactory for IcebergTableProviderFactory { - async fn create( - &self, - _state: &dyn Session, - cmd: &CreateExternalTable, - ) -> DFResult> { - let metadata_file_path = check_cmd(cmd).map_err(to_datafusion_error)?; - - let table_name = &cmd.name; - let options = &cmd.options; - - let table_name_with_ns = complement_namespace_if_necessary(table_name); - - let storage_factory = self - .storage_factory - .clone() - .unwrap_or_else(|| Arc::new(LocalFsStorageFactory)); - - let table = create_static_table( - table_name_with_ns, - metadata_file_path, - options, - storage_factory, - ) - .await - .map_err(to_datafusion_error)? - .into_table(); - - let provider = IcebergStaticTableProvider::try_new_from_table(table) - .await - .map_err(to_datafusion_error)?; - - Ok(Arc::new(provider)) - } -} - -fn check_cmd(cmd: &CreateExternalTable) -> Result<&str> { - let CreateExternalTable { - schema, - table_partition_cols, - order_exprs, - constraints, - column_defaults, - .. - } = cmd; - - // Check if any of the fields violate the constraints in a single condition - let is_invalid = !schema.fields().is_empty() - || !table_partition_cols.is_empty() - || !order_exprs.is_empty() - || !constraints.is_empty() - || !column_defaults.is_empty(); - - if is_invalid { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Currently we only support reading existing icebergs tables in external table command. To create new table, please use catalog provider.", - )); - } - - match cmd.locations.as_slice() { - [location] => Ok(location), - _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - "Iceberg external tables require exactly one metadata location.", - )), - } -} - -/// Complements the namespace of a table name if necessary. -/// -/// # Note -/// If the table name is a bare name, it will be complemented with the 'default' namespace. -/// Otherwise, it will be returned as is. Because Iceberg tables are always namespaced, but DataFusion -/// external table commands maybe not include the namespace, this function ensures that the namespace is always present. -/// -/// # See also -/// - [`iceberg::NamespaceIdent`] -/// - [`datafusion::sql::planner::SqlToRel::external_table_to_plan`] -fn complement_namespace_if_necessary(table_name: &TableReference) -> Cow<'_, TableReference> { - match table_name { - TableReference::Bare { table } => { - Cow::Owned(TableReference::partial("default", table.as_ref())) - } - other => Cow::Borrowed(other), - } -} - -async fn create_static_table( - table_name: Cow<'_, TableReference>, - metadata_file_path: &str, - props: &HashMap, - storage_factory: Arc, -) -> Result { - let table_ident = TableIdent::from_strs(table_name.to_vec())?; - let file_io = FileIOBuilder::new(storage_factory) - .with_props(props) - .build(); - StaticTable::from_metadata_file(metadata_file_path, table_ident, file_io).await -} - -#[cfg(test)] -mod tests { - - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::catalog::TableProviderFactory; - use datafusion::common::{Constraints, DFSchema, TableReference}; - use datafusion::execution::session_state::SessionStateBuilder; - use datafusion::logical_expr::CreateExternalTable; - use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use datafusion::prelude::SessionContext; - - use super::*; - - fn table_metadata_v2_schema() -> Schema { - Schema::new(vec![ - Field::new("x", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("y", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("z", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ]) - } - - fn table_metadata_location() -> String { - format!( - "{}/testdata/table_metadata/{}", - env!("CARGO_MANIFEST_DIR"), - "TableMetadataV2.json" - ) - } - - fn create_external_table_cmd() -> CreateExternalTable { - let metadata_file_path = table_metadata_location(); - - CreateExternalTable { - name: TableReference::partial("static_ns", "static_table"), - locations: vec![metadata_file_path], - schema: Arc::new(DFSchema::empty()), - file_type: "iceberg".to_string(), - options: Default::default(), - table_partition_cols: Default::default(), - order_exprs: Default::default(), - constraints: Constraints::default(), - column_defaults: Default::default(), - if_not_exists: Default::default(), - or_replace: false, - temporary: false, - definition: Default::default(), - unbounded: Default::default(), - } - } - - #[tokio::test] - async fn test_schema_of_created_table() { - let factory = IcebergTableProviderFactory::new(); - - let state = SessionStateBuilder::new().build(); - let cmd = create_external_table_cmd(); - - let table_provider = factory - .create(&state, &cmd) - .await - .expect("create table failed"); - - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } - - #[tokio::test] - async fn test_schema_of_created_external_table_sql() { - let mut state = SessionStateBuilder::new().with_default_features().build(); - state.table_factories_mut().insert( - "ICEBERG".to_string(), - Arc::new(IcebergTableProviderFactory::new()), - ); - let ctx = SessionContext::new_with_state(state); - - // All external tables in DataFusion use bare names. - // See https://github.com/apache/datafusion/blob/main/datafusion/sql/src/statement.rs#L1038-#L1039 - let table_ref = TableReference::bare("static_table"); - - // Create the external table - let sql = format!( - "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", - table_ref, - table_metadata_location() - ); - let _df = ctx.sql(&sql).await.expect("create table failed"); - - // Get the created external table - let table_provider = ctx - .table_provider(table_ref) - .await - .expect("table not found"); - - // Check the schema of the created table - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } -} diff --git a/crates/integrations/datafusion/src/task_writer.rs b/crates/integrations/datafusion/src/task_writer.rs deleted file mode 100644 index cf44f60220..0000000000 --- a/crates/integrations/datafusion/src/task_writer.rs +++ /dev/null @@ -1,517 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! TaskWriter for DataFusion integration. -//! -//! This module provides a high-level writer that handles partitioning and routing -//! of RecordBatch data to Iceberg tables. - -use datafusion::arrow::array::RecordBatch; -use iceberg::Result; -use iceberg::arrow::RecordBatchPartitionSplitter; -use iceberg::spec::{DataFile, PartitionSpecRef, SchemaRef}; -use iceberg::writer::IcebergWriterBuilder; -use iceberg::writer::partitioning::PartitioningWriter; -use iceberg::writer::partitioning::clustered_writer::ClusteredWriter; -use iceberg::writer::partitioning::fanout_writer::FanoutWriter; -use iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter; - -/// High-level writer for DataFusion that handles partitioning and routing of RecordBatch data. -/// -/// TaskWriter coordinates writing data to Iceberg tables by: -/// - Selecting the appropriate partitioning strategy (unpartitioned, fanout, or clustered) -/// - Initializing the partition splitter in the constructor for partitioned tables -/// - Routing data to the underlying writer -/// - Collecting all written data files -/// -/// # Type Parameters -/// -/// * `B` - The IcebergWriterBuilder type used to create underlying writers -/// -/// # Example -/// -/// ```rust,ignore -/// use iceberg::spec::{PartitionSpec, Schema}; -/// use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -/// use iceberg_datafusion::writer::task_writer::TaskWriter; -/// -/// // Create a TaskWriter for an unpartitioned table -/// let task_writer = TaskWriter::new( -/// data_file_writer_builder, -/// false, // fanout_enabled -/// schema, -/// partition_spec, -/// ); -/// -/// // Write data -/// task_writer.write(record_batch).await?; -/// -/// // Close and get data files -/// let data_files = task_writer.close().await?; -/// ``` -pub(crate) struct TaskWriter { - /// The underlying writer (UnpartitionedWriter, FanoutWriter, or ClusteredWriter) - writer: SupportedWriter, - /// Partition splitter for partitioned tables (initialized in constructor) - partition_splitter: Option, -} - -/// Internal enum to hold the different writer types. -/// -/// This enum allows TaskWriter to work with different partitioning strategies -/// while maintaining a unified interface. -enum SupportedWriter { - /// Writer for unpartitioned tables - Unpartitioned(UnpartitionedWriter), - /// Writer for partitioned tables with unsorted data (maintains multiple active writers) - Fanout(FanoutWriter), - /// Writer for partitioned tables with sorted data (maintains single active writer) - Clustered(ClusteredWriter), -} - -impl TaskWriter { - /// Create a new TaskWriter. - /// - /// # Parameters - /// - /// * `writer_builder` - The IcebergWriterBuilder to use for creating underlying writers - /// * `fanout_enabled` - If true, use FanoutWriter for partitioned tables; otherwise use ClusteredWriter - /// * `schema` - The Iceberg schema reference - /// * `partition_spec` - The partition specification reference - /// - /// # Returns - /// - /// Returns a new TaskWriter instance. - /// - /// # Writer Selection Logic - /// - /// - If partition_spec is unpartitioned: creates UnpartitionedWriter - /// - If partition_spec is partitioned AND fanout_enabled is true: creates FanoutWriter - /// - If partition_spec is partitioned AND fanout_enabled is false: creates ClusteredWriter - /// - /// # Example - /// - /// ```rust,ignore - /// use iceberg::spec::{PartitionSpec, Schema}; - /// use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Create a TaskWriter for an unpartitioned table - /// let task_writer = TaskWriter::new( - /// data_file_writer_builder, - /// false, // fanout_enabled - /// schema, - /// partition_spec, - /// ); - /// ``` - pub fn try_new( - writer_builder: B, - fanout_enabled: bool, - schema: SchemaRef, - partition_spec: PartitionSpecRef, - ) -> Result { - let writer = if partition_spec.is_unpartitioned() { - SupportedWriter::Unpartitioned(UnpartitionedWriter::new(writer_builder)) - } else if fanout_enabled { - SupportedWriter::Fanout(FanoutWriter::new(writer_builder)) - } else { - SupportedWriter::Clustered(ClusteredWriter::new(writer_builder)) - }; - - // Initialize partition splitter in constructor for partitioned tables - let partition_splitter = if !partition_spec.is_unpartitioned() { - Some( - RecordBatchPartitionSplitter::try_new_with_precomputed_values( - schema.clone(), - partition_spec.clone(), - )?, - ) - } else { - None - }; - - Ok(Self { - writer, - partition_splitter, - }) - } - - /// Write a RecordBatch to the TaskWriter. - /// - /// For partitioned tables, uses the partition splitter to split - /// the batch by partition key and route each partition to the underlying writer. - /// For unpartitioned tables, data is written directly without splitting. - /// - /// # Parameters - /// - /// * `batch` - The RecordBatch to write - /// - /// # Returns - /// - /// Returns `Ok(())` on success, or an error if the write fails. - /// - /// # Errors - /// - /// This method will return an error if: - /// - Splitting the batch by partition fails - /// - Writing to the underlying writer fails - /// - /// # Example - /// - /// ```rust,ignore - /// use arrow_array::RecordBatch; - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Write a RecordBatch - /// task_writer.write(record_batch).await?; - /// ``` - pub async fn write(&mut self, batch: RecordBatch) -> Result<()> { - match &mut self.writer { - SupportedWriter::Unpartitioned(writer) => { - // Unpartitioned: write directly without splitting - writer.write(batch).await - } - SupportedWriter::Fanout(writer) => { - Self::write_partitioned_batches(writer, &self.partition_splitter, &batch).await - } - SupportedWriter::Clustered(writer) => { - Self::write_partitioned_batches(writer, &self.partition_splitter, &batch).await - } - } - } - - /// Helper method to split and write partitioned data. - /// - /// This method handles the common logic for both FanoutWriter and ClusteredWriter: - /// - Splits the batch by partition key using the partition splitter - /// - Writes each partition to the underlying writer with its corresponding partition key - /// - /// # Parameters - /// - /// * `writer` - The underlying PartitioningWriter (FanoutWriter or ClusteredWriter) - /// * `partition_splitter` - The partition splitter - /// * `batch` - The RecordBatch to write - /// - /// # Returns - /// - /// Returns `Ok(())` on success, or an error if the operation fails. - async fn write_partitioned_batches( - writer: &mut W, - partition_splitter: &Option, - batch: &RecordBatch, - ) -> Result<()> { - // Split batch by partition - let splitter = partition_splitter - .as_ref() - .expect("Partition splitter should be initialized"); - let partitioned_batches = splitter.split(batch)?; - - // Write each partition - for (partition_key, partition_batch) in partitioned_batches { - writer.write(partition_key, partition_batch).await?; - } - - Ok(()) - } - - /// Close the TaskWriter and return all written data files. - /// - /// This method consumes the TaskWriter to prevent further use. - /// - /// # Returns - /// - /// Returns a `Vec` containing all written files, or an error if closing fails. - /// - /// # Errors - /// - /// This method will return an error if: - /// - Closing the underlying writer fails - /// - Any I/O operation fails during the close process - /// - /// # Example - /// - /// ```rust,ignore - /// use iceberg_datafusion::writer::task_writer::TaskWriter; - /// - /// // Close the writer and get all data files - /// let data_files = task_writer.close().await?; - /// ``` - pub async fn close(self) -> Result> { - match self.writer { - SupportedWriter::Unpartitioned(writer) => writer.close().await, - SupportedWriter::Fanout(writer) => writer.close().await, - SupportedWriter::Clustered(writer) => writer.close().await, - } - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN; - use iceberg::io::FileIO; - use iceberg::spec::{DataFileFormat, NestedField, PartitionSpec, PrimitiveType, Type}; - use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; - use iceberg::writer::file_writer::ParquetWriterBuilder; - use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, - }; - use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use parquet::file::properties::WriterProperties; - use tempfile::TempDir; - - use super::*; - - fn create_test_schema() -> Result> { - Ok(Arc::new( - iceberg::spec::Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) - .into(), - ]) - .build()?, - )) - } - - fn create_arrow_schema() -> Arc { - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ])) - } - - fn create_arrow_schema_with_partition() -> Arc { - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_struct_field = Field::new( - PROJECTED_PARTITION_VALUE_COLUMN, - DataType::Struct(vec![partition_field].into()), - false, - ); - - Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - partition_struct_field, - ])) - } - - fn create_writer_builder( - temp_dir: &TempDir, - schema: Arc, - ) -> Result< - DataFileWriterBuilder< - ParquetWriterBuilder, - DefaultLocationGenerator, - DefaultFileNameGenerator, - >, - > { - let file_io = FileIO::new_with_fs(); - let location_gen = DefaultLocationGenerator::with_data_location( - temp_dir.path().to_str().unwrap().to_string(), - ); - let file_name_gen = - DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); - let parquet_writer_builder = - ParquetWriterBuilder::new(WriterProperties::builder().build(), schema); - let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( - parquet_writer_builder, - file_io, - location_gen, - file_name_gen, - ); - Ok(DataFileWriterBuilder::new(rolling_writer_builder)) - } - - #[tokio::test] - async fn test_task_writer_unpartitioned() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema(); - - // Create unpartitioned spec - let partition_spec = Arc::new(PartitionSpec::builder(schema.clone()).build()?); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, false, schema, partition_spec)?; - - // Write data - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), - Arc::new(StringArray::from(vec!["US", "EU", "US"])), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - // Verify results - assert!(!data_files.is_empty()); - assert_eq!(data_files[0].record_count(), 3); - - Ok(()) - } - - /// Helper to verify partition data files - fn verify_partition_files( - data_files: &[DataFile], - expected_total: u64, - ) -> HashMap { - let total_records: u64 = data_files.iter().map(|f| f.record_count()).sum(); - assert_eq!(total_records, expected_total, "Total record count mismatch"); - - let mut partition_counts = HashMap::new(); - for data_file in data_files { - let partition_value = data_file.partition(); - let region_literal = partition_value.fields()[0] - .as_ref() - .expect("Partition value should not be null"); - let region = match region_literal - .as_primitive_literal() - .expect("Should be primitive literal") - { - iceberg::spec::PrimitiveLiteral::String(s) => s.clone(), - _ => panic!("Expected string partition value"), - }; - - *partition_counts.entry(region.clone()).or_insert(0) += data_file.record_count(); - - // Verify file path contains partition information - assert!( - data_file.file_path().contains("region="), - "File path should contain partition info" - ); - } - partition_counts - } - - #[tokio::test] - async fn test_task_writer_partitioned_fanout() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema_with_partition(); - - let partition_spec = Arc::new( - PartitionSpec::builder(schema.clone()) - .with_spec_id(1) - .add_partition_field("region", "region", iceberg::spec::Transform::Identity)? - .build()?, - ); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, true, schema, partition_spec)?; - - // Create partition column - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_values = StringArray::from(vec!["US", "EU", "US", "EU"]); - let partition_struct = StructArray::from(vec![( - Arc::new(partition_field), - Arc::new(partition_values) as ArrayRef, - )]); - - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3, 4])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Dave"])), - Arc::new(StringArray::from(vec!["US", "EU", "US", "EU"])), - Arc::new(partition_struct), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - let partition_counts = verify_partition_files(&data_files, 4); - assert_eq!(partition_counts.get("US"), Some(&2)); - assert_eq!(partition_counts.get("EU"), Some(&2)); - - Ok(()) - } - - #[tokio::test] - async fn test_task_writer_partitioned_clustered() -> Result<()> { - let temp_dir = TempDir::new()?; - let schema = create_test_schema()?; - let arrow_schema = create_arrow_schema_with_partition(); - - let partition_spec = Arc::new( - PartitionSpec::builder(schema.clone()) - .with_spec_id(1) - .add_partition_field("region", "region", iceberg::spec::Transform::Identity)? - .build()?, - ); - - let writer_builder = create_writer_builder(&temp_dir, schema.clone())?; - let mut task_writer = TaskWriter::try_new(writer_builder, false, schema, partition_spec)?; - - // Create partition column - let partition_field = Field::new("region", DataType::Utf8, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), - ); - let partition_values = StringArray::from(vec!["ASIA", "ASIA", "EU", "EU"]); - let partition_struct = StructArray::from(vec![( - Arc::new(partition_field), - Arc::new(partition_values) as ArrayRef, - )]); - - // ClusteredWriter expects data to be pre-sorted by partition - let batch = RecordBatch::try_new(arrow_schema, vec![ - Arc::new(Int32Array::from(vec![1, 2, 3, 4])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Dave"])), - Arc::new(StringArray::from(vec!["ASIA", "ASIA", "EU", "EU"])), - Arc::new(partition_struct), - ])?; - - task_writer.write(batch).await?; - let data_files = task_writer.close().await?; - - let partition_counts = verify_partition_files(&data_files, 4); - assert_eq!(partition_counts.get("ASIA"), Some(&2)); - assert_eq!(partition_counts.get("EU"), Some(&2)); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json b/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json deleted file mode 100644 index a7b47217f5..0000000000 --- a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "default-spec-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "name": "x", - "transform": "identity", - "source-id": 1, - "field-id": 1000 - } - ] - } - ], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - { - "transform": "identity", - "source-id": 2, - "direction": "asc", - "null-order": "nulls-first" - }, - { - "transform": "bucket[4]", - "source-id": 3, - "direction": "desc", - "null-order": "nulls-last" - } - ] - } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] -} \ No newline at end of file diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs deleted file mode 100644 index 9aa6a1a501..0000000000 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ /dev/null @@ -1,948 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for Iceberg Datafusion with Hive Metastore. - -use std::collections::HashMap; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::{Array, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; -use datafusion::execution::context::SessionContext; -use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use expect_test::expect; -use iceberg::io::LocalFsStorageFactory; -use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; -use iceberg::spec::{ - NestedField, PrimitiveType, Schema, StructType, Transform, Type, UnboundPartitionSpec, -}; -use iceberg::test_utils::check_record_batches; -use iceberg::{ - Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation, TableIdent, -}; -use iceberg_datafusion::IcebergCatalogProvider; -use tempfile::TempDir; - -fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() -} - -async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .with_storage_factory(Arc::new(LocalFsStorageFactory)) - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() -} - -fn get_struct_type() -> StructType { - StructType::new(vec![ - NestedField::required(4, "s_foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(5, "s_foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) -} - -async fn set_test_namespace(catalog: &MemoryCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Option, -) -> Result { - let schema = match schema { - None => Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?, - Some(schema) => schema, - }; - - let creation = TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build(); - - Ok(creation) -} - -#[tokio::test] -async fn test_provider_plan_stream_schema() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_get_table_schema".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_get_table_schema").unwrap(); - - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - let df = ctx - .sql("select foo2 from catalog.test_provider_get_table_schema.my_table") - .await - .unwrap(); - - let task_ctx = Arc::new(df.task_ctx()); - let plan = df.create_physical_plan().await.unwrap(); - let stream = plan.execute(1, task_ctx).unwrap(); - - // Ensure both the plan and the stream conform to the same schema - assert_eq!(plan.schema(), stream.schema()); - assert_eq!( - stream.schema().as_ref(), - &ArrowSchema::new(vec![ - Field::new("foo2", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])) - ]), - ); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_table_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_table_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_list_table_names").unwrap(); - - let result = schema.table_names(); - - expect![[r#" - [ - "my_table", - "my_table$snapshots", - "my_table$manifests", - "my_table$history", - ] - "#]] - .assert_debug_eq(&result); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_schema_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_schema_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - - let expected = ["test_provider_list_schema_names"]; - let result = provider.schema_names(); - - assert!( - expected - .iter() - .all(|item| result.contains(&item.to_string())) - ); - Ok(()) -} - -#[tokio::test] -async fn test_table_projection() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "foo3", Type::Struct(get_struct_type())).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let table_df = ctx.table("catalog.ns.t1").await.unwrap(); - - let records = table_df - .clone() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - assert!(s.value(1).contains("projection:[foo1,foo2,foo3]")); - - // datafusion doesn't support query foo3.s_foo1, use foo3 instead - let records = table_df - .select_columns(&["foo1", "foo3"]) - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - assert!( - s.value(1) - .contains("IcebergTableScan projection:[foo1,foo3]") - ); - - Ok(()) -} - -#[tokio::test] -async fn test_table_predict_pushdown() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let records = ctx - .sql("select * from catalog.ns.t1 where (foo > 1 and length(bar) = 1 ) or bar is null") - .await - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - let expected = "predicate:[(foo > 1) OR (bar IS NULL)]"; - assert!(s.value(1).trim().contains(expected)); - Ok(()) -} - -#[tokio::test] -async fn test_metadata_table() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let snapshots = ctx - .sql("select * from catalog.ns.t1$snapshots") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - snapshots, - expect![[r#" - Field { "committed_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, - Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, - Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, - Field { "operation": nullable Utf8, metadata: {"PARQUET:field_id": "4"} }, - Field { "manifest_list": nullable Utf8, metadata: {"PARQUET:field_id": "5"} }, - Field { "summary": nullable Map("key_value": non-null Struct("key": non-null Utf8, metadata: {"PARQUET:field_id": "7"}, "value": Utf8, metadata: {"PARQUET:field_id": "8"}), unsorted), metadata: {"PARQUET:field_id": "6"} }"#]], - expect![[r#" - committed_at: PrimitiveArray - [ - ], - snapshot_id: PrimitiveArray - [ - ], - parent_id: PrimitiveArray - [ - ], - operation: StringArray - [ - ], - manifest_list: StringArray - [ - ], - summary: MapArray - [ - ]"#]], - &[], - None, - ); - - let manifests = ctx - .sql("select * from catalog.ns.t1$manifests") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - manifests, - expect![[r#" - Field { "content": Int32, metadata: {"PARQUET:field_id": "14"} }, - Field { "path": Utf8, metadata: {"PARQUET:field_id": "1"} }, - Field { "length": Int64, metadata: {"PARQUET:field_id": "2"} }, - Field { "partition_spec_id": Int32, metadata: {"PARQUET:field_id": "3"} }, - Field { "added_snapshot_id": Int64, metadata: {"PARQUET:field_id": "4"} }, - Field { "added_data_files_count": Int32, metadata: {"PARQUET:field_id": "5"} }, - Field { "existing_data_files_count": Int32, metadata: {"PARQUET:field_id": "6"} }, - Field { "deleted_data_files_count": Int32, metadata: {"PARQUET:field_id": "7"} }, - Field { "added_delete_files_count": Int32, metadata: {"PARQUET:field_id": "15"} }, - Field { "existing_delete_files_count": Int32, metadata: {"PARQUET:field_id": "16"} }, - Field { "deleted_delete_files_count": Int32, metadata: {"PARQUET:field_id": "17"} }, - Field { "partition_summaries": List(non-null Struct("contains_null": non-null Boolean, metadata: {"PARQUET:field_id": "10"}, "contains_nan": Boolean, metadata: {"PARQUET:field_id": "11"}, "lower_bound": Utf8, metadata: {"PARQUET:field_id": "12"}, "upper_bound": Utf8, metadata: {"PARQUET:field_id": "13"}), metadata: {"PARQUET:field_id": "9"}), metadata: {"PARQUET:field_id": "8"} }"#]], - expect![[r#" - content: PrimitiveArray - [ - ], - path: StringArray - [ - ], - length: PrimitiveArray - [ - ], - partition_spec_id: PrimitiveArray - [ - ], - added_snapshot_id: PrimitiveArray - [ - ], - added_data_files_count: PrimitiveArray - [ - ], - existing_data_files_count: PrimitiveArray - [ - ], - deleted_data_files_count: PrimitiveArray - [ - ], - added_delete_files_count: PrimitiveArray - [ - ], - existing_delete_files_count: PrimitiveArray - [ - ], - deleted_delete_files_count: PrimitiveArray - [ - ], - partition_summaries: ListArray - [ - ]"#]], - &[], - None, - ); - - Ok(()) -} - -#[tokio::test] -async fn test_insert_into() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_into".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_into").unwrap(); - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - // Insert data into the table - let df = ctx - .sql("INSERT INTO catalog.test_insert_into.my_table VALUES (1, 'alan'), (2, 'turing')") - .await - .unwrap(); - - // Verify the insert operation result - let batches = df.collect().await.unwrap(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!( - batch.num_rows() == 1 && batch.num_columns() == 1, - "Results should only have one row and one column that has the number of rows inserted" - ); - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_into.my_table") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { "foo1": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "foo2": Utf8, metadata: {"PARQUET:field_id": "2"} }"#]], - expect![[r#" - foo1: PrimitiveArray - [ - 1, - 2, - ], - foo2: StringArray - [ - "alan", - "turing", - ]"#]], - &[], - Some("foo1"), - ); - - Ok(()) -} - -fn get_nested_struct_type() -> StructType { - // Create a nested struct type with: - // - address: STRUCT - // - contact: STRUCT - StructType::new(vec![ - NestedField::optional( - 10, - "address", - Type::Struct(StructType::new(vec![ - NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(12, "city", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int)).into(), - ])), - ) - .into(), - NestedField::optional( - 20, - "contact", - Type::Struct(StructType::new(vec![ - NestedField::optional(21, "email", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(22, "phone", Type::Primitive(PrimitiveType::String)).into(), - ])), - ) - .into(), - ]) -} - -#[tokio::test] -async fn test_insert_into_nested() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_nested".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - let table_name = "nested_table"; - - // Create a schema with nested fields - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "profile", Type::Struct(get_nested_struct_type())).into(), - ]) - .build()?; - - // Create the table with the nested schema - let creation = get_table_creation(temp_path(), table_name, Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_nested").unwrap(); - let table = schema.table("nested_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - // Verify the schema has the expected structure - assert_eq!(table_schema.fields().len(), 3); - assert_eq!(table_schema.field(0).name(), "id"); - assert_eq!(table_schema.field(1).name(), "name"); - assert_eq!(table_schema.field(2).name(), "profile"); - assert!(matches!( - table_schema.field(2).data_type(), - DataType::Struct(_) - )); - - // In DataFusion, we need to use named_struct to create struct values - // Insert data with nested structs - let insert_sql = r#" - INSERT INTO catalog.test_insert_nested.nested_table - SELECT - 1 as id, - 'Alice' as name, - named_struct( - 'address', named_struct( - 'street', '123 Main St', - 'city', 'San Francisco', - 'zip', 94105 - ), - 'contact', named_struct( - 'email', 'alice@example.com', - 'phone', '555-1234' - ) - ) as profile - UNION ALL - SELECT - 2 as id, - 'Bob' as name, - named_struct( - 'address', named_struct( - 'street', '456 Market St', - 'city', 'San Jose', - 'zip', 95113 - ), - 'contact', named_struct( - 'email', 'bob@example.com', - 'phone', NULL - ) - ) as profile - "#; - - // Execute the insert - let df = ctx.sql(insert_sql).await.unwrap(); - let batches = df.collect().await.unwrap(); - - // Verify the insert operation result - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!(batch.num_rows() == 1 && batch.num_columns() == 1); - - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_nested.nested_table ORDER BY id") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "name": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "profile": nullable Struct("address": Struct("street": Utf8, metadata: {"PARQUET:field_id": "6"}, "city": Utf8, metadata: {"PARQUET:field_id": "7"}, "zip": Int32, metadata: {"PARQUET:field_id": "8"}), metadata: {"PARQUET:field_id": "4"}, "contact": Struct("email": Utf8, metadata: {"PARQUET:field_id": "9"}, "phone": Utf8, metadata: {"PARQUET:field_id": "10"}), metadata: {"PARQUET:field_id": "5"}), metadata: {"PARQUET:field_id": "3"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - profile: StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "address" (Struct([Field { name: "street", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, nullable: true, metadata: {"PARQUET:field_id": "8"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "street" (Utf8) - StringArray - [ - "123 Main St", - "456 Market St", - ] - -- child 1: "city" (Utf8) - StringArray - [ - "San Francisco", - "San Jose", - ] - -- child 2: "zip" (Int32) - PrimitiveArray - [ - 94105, - 95113, - ] - ] - -- child 1: "contact" (Struct([Field { name: "email", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "9"} }, Field { name: "phone", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "10"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "email" (Utf8) - StringArray - [ - "alice@example.com", - "bob@example.com", - ] - -- child 1: "phone" (Utf8) - StringArray - [ - "555-1234", - null, - ] - ] - ]"#]], - &[], - Some("id"), - ); - - // Query with explicit field access to verify nested data - let df = ctx - .sql( - r#" - SELECT - id, - name, - profile.address.street, - profile.address.city, - profile.address.zip, - profile.contact.email, - profile.contact.phone - FROM catalog.test_insert_nested.nested_table - ORDER BY id - "#, - ) - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the flattened data - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "name": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][street]": nullable Utf8, metadata: {"PARQUET:field_id": "6"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][city]": nullable Utf8, metadata: {"PARQUET:field_id": "7"} }, - Field { "catalog.test_insert_nested.nested_table.profile[address][zip]": nullable Int32, metadata: {"PARQUET:field_id": "8"} }, - Field { "catalog.test_insert_nested.nested_table.profile[contact][email]": nullable Utf8, metadata: {"PARQUET:field_id": "9"} }, - Field { "catalog.test_insert_nested.nested_table.profile[contact][phone]": nullable Utf8, metadata: {"PARQUET:field_id": "10"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - catalog.test_insert_nested.nested_table.profile[address][street]: StringArray - [ - "123 Main St", - "456 Market St", - ], - catalog.test_insert_nested.nested_table.profile[address][city]: StringArray - [ - "San Francisco", - "San Jose", - ], - catalog.test_insert_nested.nested_table.profile[address][zip]: PrimitiveArray - [ - 94105, - 95113, - ], - catalog.test_insert_nested.nested_table.profile[contact][email]: StringArray - [ - "alice@example.com", - "bob@example.com", - ], - catalog.test_insert_nested.nested_table.profile[contact][phone]: StringArray - [ - "555-1234", - null, - ]"#]], - &[], - Some("id"), - ); - - Ok(()) -} - -#[tokio::test] -async fn test_insert_into_partitioned() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_partitioned_write".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - // Create a schema with a partition column - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(3, "value", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - // Create partition spec with identity transform on category - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity)? - .build(); - - // Create the partitioned table - let creation = TableCreation::builder() - .name("partitioned_table".to_string()) - .location(temp_path()) - .schema(schema) - .partition_spec(partition_spec) - .properties(HashMap::new()) - .build(); - - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Insert data with multiple partition values in a single batch - let df = ctx - .sql( - r#" - INSERT INTO catalog.test_partitioned_write.partitioned_table - VALUES - (1, 'electronics', 'laptop'), - (2, 'electronics', 'phone'), - (3, 'books', 'novel'), - (4, 'books', 'textbook'), - (5, 'clothing', 'shirt') - "#, - ) - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 5); - - // Query the table to verify data - let df = ctx - .sql("SELECT * FROM catalog.test_partitioned_write.partitioned_table ORDER BY id") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Verify the data - note that _partition column should NOT be present - check_record_batches( - batches, - expect![[r#" - Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, - Field { "category": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "value": Utf8, metadata: {"PARQUET:field_id": "3"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - 3, - 4, - 5, - ], - category: StringArray - [ - "electronics", - "electronics", - "books", - "books", - "clothing", - ], - value: StringArray - [ - "laptop", - "phone", - "novel", - "textbook", - "shirt", - ]"#]], - &[], - Some("id"), - ); - - // Verify that data files exist under correct partition paths - let table_ident = TableIdent::new(namespace.clone(), "partitioned_table".to_string()); - let table = client.load_table(&table_ident).await?; - let table_location = table.metadata().location(); - let file_io = table.file_io(); - - // List files under each expected partition path - let electronics_path = format!("{table_location}/data/category=electronics"); - let books_path = format!("{table_location}/data/category=books"); - let clothing_path = format!("{table_location}/data/category=clothing"); - - // Verify partition directories exist and contain data files - assert!( - file_io.exists(&electronics_path).await?, - "Expected partition directory: {electronics_path}" - ); - assert!( - file_io.exists(&books_path).await?, - "Expected partition directory: {books_path}" - ); - assert!( - file_io.exists(&clothing_path).await?, - "Expected partition directory: {clothing_path}" - ); - - Ok(()) -} diff --git a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json b/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json deleted file mode 100644 index 0dc89de58c..0000000000 --- a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long", - "doc": "comment" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "default-spec-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "name": "x", - "transform": "identity", - "source-id": 1, - "field-id": 1000 - } - ] - } - ], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - { - "transform": "identity", - "source-id": 2, - "direction": "asc", - "null-order": "nulls-first" - }, - { - "transform": "bucket[4]", - "source-id": 3, - "direction": "desc", - "null-order": "nulls-last" - } - ] - } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] -} \ No newline at end of file diff --git a/crates/integrations/playground/Cargo.toml b/crates/integrations/playground/Cargo.toml deleted file mode 100644 index 60137e2ff8..0000000000 --- a/crates/integrations/playground/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -description = "Apache iceberg client" -edition.workspace = true -homepage.workspace = true -license.workspace = true -name = "iceberg-playground" -publish = false -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -datafusion = { workspace = true } -datafusion-cli = { workspace = true } -dirs = { workspace = true } -fs-err = { workspace = true } -home = { workspace = true } -iceberg = { workspace = true } -iceberg-catalog-rest = { workspace = true } -iceberg-datafusion = { workspace = true } -mimalloc = { workspace = true } -stacker = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["stacker", "mimalloc", "home"] - -[lints] -workspace = true diff --git a/crates/integrations/playground/DEPENDENCIES.rust.tsv b/crates/integrations/playground/DEPENDENCIES.rust.tsv deleted file mode 100644 index 49af43d19f..0000000000 --- a/crates/integrations/playground/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,517 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.21 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@0.2.7 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.15 X -aws-credential-types@1.2.14 X -aws-lc-rs@1.16.2 X X -aws-lc-sys@0.39.0 X X X X X -aws-runtime@1.7.2 X -aws-sdk-sso@1.97.0 X -aws-sdk-ssooidc@1.99.0 X -aws-sdk-sts@1.101.0 X -aws-sigv4@1.4.2 X -aws-smithy-async@1.2.14 X -aws-smithy-http@0.63.6 X -aws-smithy-http-client@1.1.12 X -aws-smithy-json@0.62.5 X -aws-smithy-observability@0.2.6 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.10.3 X -aws-smithy-runtime-api@1.11.6 X -aws-smithy-types@1.4.7 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.14 X -backon@1.6.0 X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -cfg_aliases@0.2.1 X -chacha20@0.10.0 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -clipboard-win@5.4.1 X -cmake@0.1.57 X X -colorchoice@1.0.5 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -cpufeatures@0.3.0 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-cli@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-avro@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-sql@53.1.0 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -dirs@6.0.0 X X -dirs-sys@0.5.0 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dunce@1.0.5 X X X -either@1.15.0 X X -endian-type@0.1.2 X -env_filter@1.0.0 X X -env_logger@0.11.9 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -error-code@3.3.2 X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -fd-lock@4.0.4 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -fs-err@3.3.0 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -h2@0.4.13 X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.4.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-catalog-rest@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg-playground@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -libmimalloc-sys@0.1.44 X -libredox@0.1.14 X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -mimalloc@0.1.48 X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nibble_vec@0.1.0 X -nix@0.30.1 X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.2.0 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -opaque-debug@0.3.1 X X -openssl-probe@0.2.1 X X -option-ext@0.2.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -psm@0.1.30 X X -quad-rand@0.2.3 X -quick-xml@0.39.4 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -radix_trie@0.2.1 X -rand@0.10.1 X X -rand@0.9.4 X X -rand_chacha@0.9.0 X X -rand_core@0.10.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -redox_users@0.5.2 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustls@0.23.37 X X X -rustls-native-certs@0.8.3 X X X -rustls-pki-types@1.14.0 X X -rustls-webpki@0.103.13 X -rustversion@1.0.22 X X -rustyline@17.0.2 X -ryu@1.0.23 X X -same-file@1.0.6 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_spanned@0.6.9 X X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -time@0.3.47 X X -time-core@0.1.8 X X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.2 X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -wasm-streams@0.4.2 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@0.7.15 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -xmlparser@0.13.6 X X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/playground/README.md b/crates/integrations/playground/README.md deleted file mode 100644 index c471724598..0000000000 --- a/crates/integrations/playground/README.md +++ /dev/null @@ -1,25 +0,0 @@ - - - -# Introduction - -Iceberg Playground (`iceberg-playground`) is a small command line utility that runs SQL queries against tables, -which is backed by the DataFusion engine. - diff --git a/crates/integrations/playground/src/catalog.rs b/crates/integrations/playground/src/catalog.rs deleted file mode 100644 index e815cf720b..0000000000 --- a/crates/integrations/playground/src/catalog.rs +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; - -use anyhow::anyhow; -use datafusion::catalog::{CatalogProvider, CatalogProviderList}; -use fs_err::read_to_string; -use iceberg::CatalogBuilder; -use iceberg::memory::MemoryCatalogBuilder; -use iceberg_catalog_rest::RestCatalogBuilder; -use iceberg_datafusion::IcebergCatalogProvider; -use toml::{Table as TomlTable, Value}; - -const CONFIG_NAME_CATALOGS: &str = "catalogs"; - -#[derive(Debug)] -pub struct IcebergCatalogList { - catalogs: HashMap>, -} - -impl IcebergCatalogList { - pub async fn parse(path: &Path) -> anyhow::Result { - let toml_table: TomlTable = toml::from_str(&read_to_string(path)?)?; - Self::parse_table(&toml_table).await - } - - pub async fn parse_table(configs: &TomlTable) -> anyhow::Result { - if let Value::Array(catalogs_config) = - configs.get(CONFIG_NAME_CATALOGS).ok_or_else(|| { - anyhow::Error::msg(format!("{CONFIG_NAME_CATALOGS} entry not found in config")) - })? - { - let mut catalogs = HashMap::with_capacity(catalogs_config.len()); - for config in catalogs_config { - if let Value::Table(table_config) = config { - let (name, catalog_provider) = - IcebergCatalogList::parse_one(table_config).await?; - catalogs.insert(name, catalog_provider); - } else { - return Err(anyhow!("{CONFIG_NAME_CATALOGS} entry must be a table")); - } - } - Ok(Self { catalogs }) - } else { - Err(anyhow!("{CONFIG_NAME_CATALOGS} must be an array of table!")) - } - } - - async fn parse_one( - config: &TomlTable, - ) -> anyhow::Result<(String, Arc)> { - let name = config - .get("name") - .ok_or_else(|| anyhow::anyhow!("name not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("name is not string"))?; - - let r#type = config - .get("type") - .ok_or_else(|| anyhow::anyhow!("type not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("type is not string"))?; - - let catalog_config = config - .get("config") - .ok_or_else(|| anyhow::anyhow!("config not found for catalog {name}"))? - .as_table() - .ok_or_else(|| anyhow::anyhow!("config is not table for catalog {name}"))?; - - // parse all config into props - let mut props = HashMap::with_capacity(catalog_config.len()); - for (key, value) in catalog_config { - let value_str = value - .as_str() - .ok_or_else(|| anyhow::anyhow!("props {key} is not string"))?; - props.insert(key.to_string(), value_str.to_string()); - } - - // Create catalog based on type using the appropriate builder - let catalog: Arc = match r#type { - "rest" => Arc::new(RestCatalogBuilder::default().load(name, props).await?), - "memory" => Arc::new(MemoryCatalogBuilder::default().load(name, props).await?), - _ => { - return Err(anyhow::anyhow!( - "Unsupported catalog type: '{type}'. Supported types: rest, memory" - )); - } - }; - - Ok(( - name.to_string(), - Arc::new(IcebergCatalogProvider::try_new(catalog).await?), - )) - } -} - -impl CatalogProviderList for IcebergCatalogList { - fn register_catalog( - &self, - _name: String, - _catalog: Arc, - ) -> Option> { - tracing::error!("Registering catalog is not supported yet"); - None - } - - fn catalog_names(&self) -> Vec { - self.catalogs.keys().cloned().collect() - } - - fn catalog(&self, name: &str) -> Option> { - self.catalogs - .get(name) - .map(|c| c.clone() as Arc) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_parse_memory_catalog() { - let config = r#" - [[catalogs]] - name = "test_memory" - type = "memory" - [catalogs.config] - warehouse = "/tmp/test-warehouse" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let catalog_list = IcebergCatalogList::parse_table(&toml_table).await.unwrap(); - - assert!( - catalog_list - .catalog_names() - .contains(&"test_memory".to_string()) - ); - assert!(catalog_list.catalog("test_memory").is_some()); - } - - #[tokio::test] - async fn test_parse_unsupported_catalog_type() { - let config = r#" - [[catalogs]] - name = "test_hive" - type = "hive" - [catalogs.config] - uri = "thrift://localhost:9083" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let result = IcebergCatalogList::parse_table(&toml_table).await; - - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!(err_msg.contains("Unsupported catalog type")); - assert!(err_msg.contains("hive")); - assert!(err_msg.contains("rest, memory")); - } - - #[tokio::test] - async fn test_catalog_names() { - let config = r#" - [[catalogs]] - name = "catalog_one" - type = "memory" - [catalogs.config] - warehouse = "/tmp/warehouse1" - - [[catalogs]] - name = "catalog_two" - type = "memory" - [catalogs.config] - warehouse = "/tmp/warehouse2" - "#; - - let toml_table: TomlTable = toml::from_str(config).unwrap(); - let catalog_list = IcebergCatalogList::parse_table(&toml_table).await.unwrap(); - - let names = catalog_list.catalog_names(); - assert_eq!(names.len(), 2); - assert!(names.contains(&"catalog_one".to_string())); - assert!(names.contains(&"catalog_two".to_string())); - } -} diff --git a/crates/integrations/playground/src/lib.rs b/crates/integrations/playground/src/lib.rs deleted file mode 100644 index de62f4b143..0000000000 --- a/crates/integrations/playground/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![doc = include_str!("../README.md")] -pub const ICEBERG_PLAYGROUND_VERSION: &str = env!("CARGO_PKG_VERSION"); - -mod catalog; -pub use catalog::*; diff --git a/crates/integrations/playground/src/main.rs b/crates/integrations/playground/src/main.rs deleted file mode 100644 index 94068bb558..0000000000 --- a/crates/integrations/playground/src/main.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::path::PathBuf; -use std::process::ExitCode; -use std::str::FromStr; -use std::sync::Arc; - -use clap::Parser; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_cli::exec; -use datafusion_cli::object_storage::instrumented::InstrumentedObjectStoreRegistry; -use datafusion_cli::print_format::PrintFormat; -use datafusion_cli::print_options::{MaxRows, PrintOptions}; -use iceberg_playground::{ICEBERG_PLAYGROUND_VERSION, IcebergCatalogList}; - -#[derive(Debug, Parser, PartialEq)] -#[clap(author, version, about, long_about= None)] -struct Args { - #[clap( - short = 'r', - long, - help = "Parse catalog config instead of using ~/.icebergrc" - )] - rc: Option, - - #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] - format: PrintFormat, - - #[clap( - short, - long, - help = "Reduce printing other than the results and work quietly" - )] - quiet: bool, - - #[clap( - long, - help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", - default_value = "40" - )] - maxrows: MaxRows, - - #[clap(long, help = "Enables console syntax highlighting")] - color: bool, -} - -#[tokio::main] -/// Calls [`main_inner`], then handles printing errors and returning the correct exit code -pub async fn main() -> ExitCode { - tracing_subscriber::fmt::init(); - - if let Err(e) = main_inner().await { - println!("Error: {e}"); - return ExitCode::FAILURE; - } - - ExitCode::SUCCESS -} - -async fn main_inner() -> anyhow::Result<()> { - let args = Args::parse(); - - if !args.quiet { - println!("ICEBERG PLAYGROUND v{ICEBERG_PLAYGROUND_VERSION}"); - } - - let session_config = SessionConfig::from_env()?.with_information_schema(true); - - let rt_builder = RuntimeEnvBuilder::new(); - - let runtime_env = rt_builder.build_arc()?; - - // enable dynamic file query - let ctx = SessionContext::new_with_config_rt(session_config, runtime_env).enable_url_table(); - ctx.refresh_catalogs().await?; - - let mut print_options = PrintOptions { - format: args.format, - quiet: args.quiet, - maxrows: args.maxrows, - color: args.color, - instrumented_registry: Arc::new(InstrumentedObjectStoreRegistry::new()), - }; - - let rc = match args.rc { - Some(file) => PathBuf::from_str(&file)?, - None => dirs::home_dir() - .map(|h| h.join(".icebergrc")) - .ok_or_else(|| anyhow::anyhow!("cannot find home directory"))?, - }; - - let catalogs = Arc::new(IcebergCatalogList::parse(&rc).await?); - ctx.register_catalog_list(catalogs); - - Ok(exec::exec_from_repl(&ctx, &mut print_options).await?) -} diff --git a/crates/sqllogictest/Cargo.toml b/crates/sqllogictest/Cargo.toml deleted file mode 100644 index 37f6a663f2..0000000000 --- a/crates/sqllogictest/Cargo.toml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -name = "iceberg-sqllogictest" -publish = false -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -datafusion = { workspace = true } -datafusion-sqllogictest = { workspace = true } -enum-ordinalize = { workspace = true } -env_logger = { workspace = true } -iceberg = { workspace = true } -iceberg-datafusion = { workspace = true } -indicatif = { workspace = true } -log = { workspace = true } -serde = { workspace = true } -sqllogictest = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -libtest-mimic = { workspace = true } - -[[test]] -harness = false -name = "sqllogictests" -path = "tests/sqllogictests.rs" - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["enum-ordinalize"] - -[lints] -workspace = true diff --git a/crates/sqllogictest/DEPENDENCIES.rust.tsv b/crates/sqllogictest/DEPENDENCIES.rust.tsv deleted file mode 100644 index d50ca9d2df..0000000000 --- a/crates/sqllogictest/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,481 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.21 X X -anstream@1.0.0 X X -anstyle@1.0.14 X X -anstyle-parse@0.2.7 X X -anstyle-parse@1.0.0 X X -anstyle-query@1.1.5 X X -anstyle-wincon@3.0.11 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -ar_archive_writer@0.5.1 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@58.1.0 X -arrow-arith@58.3.0 X -arrow-array@58.3.0 X X -arrow-buffer@58.3.0 X -arrow-cast@58.3.0 X -arrow-csv@58.1.0 X -arrow-data@58.3.0 X -arrow-ipc@58.1.0 X -arrow-json@58.1.0 X -arrow-ord@58.3.0 X -arrow-row@58.1.0 X -arrow-schema@58.3.0 X -arrow-select@58.3.0 X -arrow-string@58.3.0 X -as-any@0.3.2 X X -async-compression@0.4.41 X X -async-lock@3.4.2 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.6.0 X -base64@0.22.1 X X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.11.0 X X -blake2@0.10.6 X X -blake3@1.8.3 X X X -block-buffer@0.10.4 X X -bnum@0.12.1 X X -bon@3.9.1 X X -bon-macros@3.9.1 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.20.2 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bzip2@0.6.1 X X -cc@1.2.57 X X -cfg-if@1.0.4 X X -chrono@0.4.44 X X -chrono-tz@0.10.4 X X -cipher@0.4.4 X X -clap@4.6.0 X X -clap_builder@4.6.0 X X -clap_derive@4.6.0 X X -clap_lex@1.1.0 X X -colorchoice@1.0.5 X X -comfy-table@7.2.2 X -compression-codecs@0.4.37 X X -compression-core@0.4.31 X X -concurrent-queue@2.5.0 X X -console@0.16.3 X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.4.2 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -csv@1.4.0 X X -csv-core@0.1.13 X X -ctr@0.9.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -dashmap@6.2.1 X -datafusion@53.1.0 X -datafusion-catalog@53.1.0 X -datafusion-catalog-listing@53.1.0 X -datafusion-common@53.1.0 X -datafusion-common-runtime@53.1.0 X -datafusion-datasource@53.1.0 X -datafusion-datasource-arrow@53.1.0 X -datafusion-datasource-avro@53.1.0 X -datafusion-datasource-csv@53.1.0 X -datafusion-datasource-json@53.1.0 X -datafusion-datasource-parquet@53.1.0 X -datafusion-doc@53.1.0 X -datafusion-execution@53.1.0 X -datafusion-expr@53.1.0 X -datafusion-expr-common@53.1.0 X -datafusion-functions@53.1.0 X -datafusion-functions-aggregate@53.1.0 X -datafusion-functions-aggregate-common@53.1.0 X -datafusion-functions-nested@53.1.0 X -datafusion-functions-table@53.1.0 X -datafusion-functions-window@53.1.0 X -datafusion-functions-window-common@53.1.0 X -datafusion-macros@53.1.0 X -datafusion-optimizer@53.1.0 X -datafusion-physical-expr@53.1.0 X -datafusion-physical-expr-adapter@53.1.0 X -datafusion-physical-expr-common@53.1.0 X -datafusion-physical-optimizer@53.1.0 X -datafusion-physical-plan@53.1.0 X -datafusion-pruning@53.1.0 X -datafusion-session@53.1.0 X -datafusion-spark@53.1.0 X -datafusion-sql@53.1.0 X -datafusion-sqllogictest@53.1.0 X -datafusion-substrait@53.1.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.11 X -dyn-clone@1.0.20 X X -educe@0.6.0 X -either@1.15.0 X X -encode_unicode@1.0.0 X X -enum-ordinalize@4.4.1 X -enum-ordinalize-derive@4.4.1 X -env_filter@1.0.0 X X -env_logger@0.11.9 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -escape8259@0.5.3 X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.4 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.9 X X -fixedbitset@0.5.7 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -foldhash@0.2.0 X -form_urlencoded@1.2.2 X X -fs-err@3.3.0 X X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.2 X X -ghash@0.5.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.7.1 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -hashbrown@0.16.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -http@1.4.0 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.3.0 X X -hyper@1.8.1 X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.0 X -iceberg-datafusion@0.10.0 X -iceberg-sqllogictest@0.10.0 X -iceberg_test_utils@0.10.0 X -icu_collections@2.1.1 X -icu_locale_core@2.1.1 X -icu_normalizer@2.1.1 X -icu_normalizer_data@2.1.1 X -icu_properties@2.1.2 X -icu_properties_data@2.1.2 X -icu_provider@2.1.1 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.13.0 X X -indicatif@0.18.4 X -inout@0.1.4 X X -integer-encoding@3.0.4 X -inventory@0.3.22 X X -ipnet@2.12.0 X X -iri-string@0.7.11 X X -is_terminal_polyfill@1.70.2 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.18 X X -jiff@0.2.23 X X -jobserver@0.1.34 X X -js-sys@0.3.91 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libbz2-rs-sys@0.2.2 X -libc@0.2.183 X X -liblzma@0.4.6 X X -liblzma-sys@0.4.5 X X -libm@0.2.16 X -libtest-mimic@0.8.2 X X -linux-raw-sys@0.12.1 X X X -litemap@0.8.1 X -lock_api@0.4.14 X X -log@0.4.29 X X -lz4_flex@0.13.0 X -md-5@0.10.6 X X -memchr@2.8.0 X X -miniz_oxide@0.8.9 X X X -mio@1.2.0 X -moka@0.12.15 X X -multimap@0.10.1 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.3 X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-traits@0.2.19 X X -object@0.37.3 X X -object_store@0.13.2 X X -once_cell@1.21.4 X X -once_cell_polyfill@1.70.2 X X -opaque-debug@0.3.1 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -owo-colors@4.3.0 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@58.1.0 X -paste@1.0.15 X X -pbjson@0.8.0 X -pbjson-build@0.8.0 X -pbjson-types@0.8.0 X -percent-encoding@2.3.2 X X -petgraph@0.8.3 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -portable-atomic-util@0.2.6 X X -potential_utf@0.1.4 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -prost@0.14.3 X -prost-build@0.14.3 X -prost-derive@0.14.3 X -prost-types@0.14.3 X -psm@0.1.30 X X -quad-rand@0.2.3 X -quote@1.0.45 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.8.5 X X -rand@0.9.4 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.18 X -regex@1.12.3 X X -regex-automata@0.4.14 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.10 X X -regress@0.10.5 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.3 X X -rustc_version@0.4.1 X X -rustix@1.1.4 X X X -rustversion@1.0.22 X X -ryu@1.0.23 X X -same-file@1.0.6 X X -schemars@0.8.22 X -schemars_derive@0.8.22 X -scopeguard@1.2.0 X X -semver@1.0.27 X X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_derive_internals@0.29.1 X X -serde_json@1.0.149 X X -serde_repr@0.1.20 X X -serde_spanned@0.6.9 X X -serde_tokenstream@0.2.3 X -serde_urlencoded@0.7.1 X X -serde_with@3.21.0 X X -serde_with_macros@3.21.0 X X -serde_yaml@0.9.34+deprecated X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simd-adler32@0.3.8 X -simdutf8@0.1.5 X X -similar@2.7.0 X -siphasher@1.0.2 X X -slab@0.4.12 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.3 X X -sqllogictest@0.29.1 X X -sqlparser@0.61.0 X -sqlparser_derive@0.5.0 X -stable_deref_trait@1.2.1 X X -stacker@0.1.23 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subst@0.3.8 X X -substrait@0.62.2 X -subtle@2.6.1 X -syn@2.0.117 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.27.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.2 X -tokio@1.52.1 X -tokio-macros@2.7.0 X -tokio-stream@0.1.18 X -tokio-util@0.7.18 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tower@0.5.3 X -tower-http@0.6.8 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.21 X X -typetag-impl@0.2.21 X X -typify@0.5.0 X -typify-impl@0.5.0 X -typify-macro@0.5.0 X -unicode-ident@1.0.24 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.1.14 X X -unicode-width@0.2.2 X X -unit-prefix@0.5.2 X -universal-hash@0.5.1 X X -unsafe-libyaml@0.2.11 X -untrusted@0.9.0 X -url@2.5.8 X X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.23.0 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.2+wasi-0.2.9 X X X -wasip3@0.4.0+wasi-0.3.0-rc-2026-01-06 X X X -wasm-bindgen@0.2.114 X X -wasm-bindgen-futures@0.4.64 X X -wasm-bindgen-macro@0.2.114 X X -wasm-bindgen-macro-support@0.2.114 X X -wasm-bindgen-shared@0.2.114 X X -web-sys@0.3.91 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.48.0 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.2 X X -windows-targets@0.48.5 X X -windows-targets@0.52.6 X X -windows-targets@0.53.5 X X -windows_aarch64_gnullvm@0.48.5 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.1 X X -windows_aarch64_msvc@0.48.5 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.1 X X -windows_i686_gnu@0.48.5 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.1 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.1 X X -windows_i686_msvc@0.48.5 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.1 X X -windows_x86_64_gnu@0.48.5 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.1 X X -windows_x86_64_gnullvm@0.48.5 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.1 X X -windows_x86_64_msvc@0.48.5 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.1 X X -winnow@0.7.15 X -wit-bindgen@0.51.0 X X X -writeable@0.6.2 X -yoke@0.8.1 X -yoke-derive@0.8.1 X -zerocopy@0.8.47 X X X -zerocopy-derive@0.8.47 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.9.0 X X -zerotrie@0.2.3 X -zerovec@0.11.5 X -zerovec-derive@0.11.2 X -zlib-rs@0.6.3 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/sqllogictest/README.md b/crates/sqllogictest/README.md deleted file mode 100644 index ddcfe851c5..0000000000 --- a/crates/sqllogictest/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -This crate contains a suite of [sqllogictest](https://crates.io/crates/sqllogictest) tests that are used to validate [iceberg-rust](https://github.com/apache/iceberg-rust). - -## Running the tests - -Just run the following command: - -```bash -cargo test -``` - -## Sql Engines - -The tests are run against the following sql engines: - -* [Apache datafusion](https://crates.io/crates/datafusion) -* [Apache spark](https://github.com/apache/spark) \ No newline at end of file diff --git a/crates/sqllogictest/src/engine/datafusion.rs b/crates/sqllogictest/src/engine/datafusion.rs deleted file mode 100644 index 0b52d50e53..0000000000 --- a/crates/sqllogictest/src/engine/datafusion.rs +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use datafusion::catalog::CatalogProvider; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_sqllogictest::DataFusion; -use iceberg::encryption::kms::MemoryKmsClientFactory; -use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; -use iceberg::spec::{ - NestedField, PrimitiveType, Schema, TableProperties, Transform, Type, UnboundPartitionSpec, -}; -use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; -use iceberg_datafusion::IcebergCatalogProvider; -use indicatif::ProgressBar; - -use crate::engine::{DatafusionCatalogConfig, EngineRunner, run_slt_with_runner}; -use crate::error::Result; - -const TEST_MASTER_KEY_ID: &str = "test-master-key"; - -pub struct DataFusionEngine { - test_data_path: PathBuf, - session_context: SessionContext, -} - -#[async_trait::async_trait] -impl EngineRunner for DataFusionEngine { - async fn run_slt_file(&mut self, path: &Path) -> Result<()> { - let ctx = self.session_context.clone(); - let testdata = self.test_data_path.clone(); - - let runner = sqllogictest::Runner::new({ - move || { - let ctx = ctx.clone(); - let testdata = testdata.clone(); - async move { - // Everything here is owned; no `self` capture. - Ok(DataFusion::new(ctx, testdata, ProgressBar::new(100))) - } - } - }); - - run_slt_with_runner(runner, path).await - } -} - -impl DataFusionEngine { - pub async fn new(catalog_config: Option) -> Result { - let session_config = SessionConfig::new() - .with_target_partitions(4) - .with_information_schema(true); - let ctx = SessionContext::new_with_config(session_config); - ctx.register_catalog( - "default", - Self::create_catalog(catalog_config.as_ref()).await?, - ); - - Ok(Self { - test_data_path: PathBuf::from("testdata"), - session_context: ctx, - }) - } - - async fn create_catalog( - _catalog_config: Option<&DatafusionCatalogConfig>, - ) -> anyhow::Result> { - // TODO: Use catalog_config to load different catalog types via iceberg-catalog-loader - // See: https://github.com/apache/iceberg-rust/issues/1780 - let kms_factory = MemoryKmsClientFactory::new(); - kms_factory.add_master_key(TEST_MASTER_KEY_ID)?; - - let catalog = MemoryCatalogBuilder::default() - .with_kms_client_factory(Arc::new(kms_factory)) - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://".to_string(), - )]), - ) - .await?; - - // Create a test namespace for INSERT INTO tests - let namespace = NamespaceIdent::new("default".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - // Create partitioned test table (unpartitioned tables are now created via SQL) - Self::create_partitioned_table(&catalog, &namespace).await?; - Self::create_binary_table(&catalog, &namespace).await?; - Self::create_encrypted_table(&catalog, &namespace).await?; - - Ok(Arc::new( - IcebergCatalogProvider::try_new(Arc::new(catalog)).await?, - )) - } - - /// Create a partitioned test table with id, category, and value columns - /// Partitioned by category using identity transform - /// TODO: this can be removed when we support CREATE EXTERNAL TABLE - async fn create_partitioned_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "category", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "value", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let partition_spec = UnboundPartitionSpec::builder() - .with_spec_id(0) - .add_partition_field(2, "category", Transform::Identity)? - .build(); - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_partitioned_table".to_string()) - .schema(schema) - .partition_spec(partition_spec) - .build(), - ) - .await?; - - Ok(()) - } - - /// Create a test table with binary type column - /// Used for testing binary predicate pushdown - /// TODO: this can be removed when we support CREATE TABLE - async fn create_binary_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Binary)).into(), - ]) - .build()?; - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_binary_table".to_string()) - .schema(schema) - .build(), - ) - .await?; - - Ok(()) - } - - async fn create_encrypted_table( - catalog: &impl Catalog, - namespace: &NamespaceIdent, - ) -> anyhow::Result<()> { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - catalog - .create_table( - namespace, - TableCreation::builder() - .name("test_encrypted_round_trip".to_string()) - .schema(schema) - .format_version(iceberg::spec::FormatVersion::V3) - .properties([( - TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), - TEST_MASTER_KEY_ID.to_string(), - )]) - .build(), - ) - .await?; - - Ok(()) - } -} diff --git a/crates/sqllogictest/src/engine/mod.rs b/crates/sqllogictest/src/engine/mod.rs deleted file mode 100644 index a276671401..0000000000 --- a/crates/sqllogictest/src/engine/mod.rs +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod datafusion; - -use std::collections::HashMap; -use std::path::Path; - -use anyhow::anyhow; -use serde::Deserialize; -use sqllogictest::{AsyncDB, MakeConnection, Runner, parse_file}; - -use crate::engine::datafusion::DataFusionEngine; -use crate::error::{Error, Result}; - -/// Configuration for the catalog used by the DataFusion engine -#[derive(Debug, Clone, Deserialize)] -pub struct DatafusionCatalogConfig { - /// Catalog type: "memory", "rest", "glue", "hms", "s3tables", "sql" - #[serde(rename = "type")] - pub catalog_type: String, - /// Catalog properties passed to the catalog loader - #[serde(default)] - pub props: HashMap, -} - -/// Engine configuration as a tagged enum -#[derive(Debug, Clone, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum EngineConfig { - Datafusion { - #[serde(default)] - catalog: Option, - }, -} - -#[async_trait::async_trait] -pub trait EngineRunner: Send { - async fn run_slt_file(&mut self, path: &Path) -> Result<()>; -} - -pub async fn load_engine_runner(config: EngineConfig) -> Result> { - match config { - EngineConfig::Datafusion { catalog } => Ok(Box::new(DataFusionEngine::new(catalog).await?)), - } -} - -pub async fn run_slt_with_runner( - mut runner: Runner, - step_slt_file: impl AsRef, -) -> Result<()> -where - D: AsyncDB + Send + 'static, - M: MakeConnection + Send + 'static, -{ - let path = step_slt_file.as_ref().canonicalize()?; - let records = parse_file(&path).map_err(|e| Error(anyhow!("parsing slt file failed: {e}")))?; - - for record in records { - if let Err(err) = runner.run_async(record).await { - return Err(Error(anyhow!("SLT record execution failed: {err}"))); - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use crate::engine::{DatafusionCatalogConfig, EngineConfig, load_engine_runner}; - - #[test] - fn test_deserialize_engine_config() { - let input = r#"type = "datafusion""#; - - let config: EngineConfig = toml::from_str(input).unwrap(); - assert!(matches!(config, EngineConfig::Datafusion { catalog: None })); - } - - #[test] - fn test_deserialize_engine_config_with_catalog() { - let input = r#" - type = "datafusion" - - [catalog] - type = "rest" - - [catalog.props] - uri = "http://localhost:8181" - "#; - - let config: EngineConfig = toml::from_str(input).unwrap(); - match config { - EngineConfig::Datafusion { catalog: Some(cat) } => { - assert_eq!(cat.catalog_type, "rest"); - assert_eq!( - cat.props.get("uri"), - Some(&"http://localhost:8181".to_string()) - ); - } - _ => panic!("Expected Datafusion with catalog"), - } - } - - #[test] - fn test_deserialize_catalog_config() { - let input = r#" - type = "memory" - - [props] - warehouse = "file:///tmp/warehouse" - "#; - - let config: DatafusionCatalogConfig = toml::from_str(input).unwrap(); - assert_eq!(config.catalog_type, "memory"); - assert_eq!( - config.props.get("warehouse"), - Some(&"file:///tmp/warehouse".to_string()) - ); - } - - #[tokio::test] - async fn test_load_datafusion() { - let config = EngineConfig::Datafusion { catalog: None }; - - let result = load_engine_runner(config).await; - assert!(result.is_ok()); - } -} diff --git a/crates/sqllogictest/src/error.rs b/crates/sqllogictest/src/error.rs deleted file mode 100644 index 2bf5a09d6b..0000000000 --- a/crates/sqllogictest/src/error.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fmt::{Debug, Display, Formatter}; - -pub struct Error(pub anyhow::Error); - -pub type Result = std::result::Result; - -impl Debug for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl Display for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.0.source() - } -} - -impl From for Error { - fn from(value: anyhow::Error) -> Self { - Self(value) - } -} - -impl From for Error { - fn from(value: std::io::Error) -> Self { - Self(value.into()) - } -} diff --git a/crates/sqllogictest/src/lib.rs b/crates/sqllogictest/src/lib.rs deleted file mode 100644 index 34a3c654aa..0000000000 --- a/crates/sqllogictest/src/lib.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This lib contains codes copied from -// [Apache Datafusion](https://github.com/apache/datafusion/tree/main/datafusion/sqllogictest) - -mod engine; -pub mod error; -pub mod schedule; diff --git a/crates/sqllogictest/src/schedule.rs b/crates/sqllogictest/src/schedule.rs deleted file mode 100644 index 29e37c5c72..0000000000 --- a/crates/sqllogictest/src/schedule.rs +++ /dev/null @@ -1,255 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::fs::read_to_string; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, anyhow}; -use serde::{Deserialize, Serialize}; -use tracing::info; - -use crate::engine::{EngineConfig, EngineRunner, load_engine_runner}; - -/// Raw configuration parsed from the schedule TOML file -#[derive(Debug, Clone, Deserialize)] -pub struct ScheduleConfig { - /// Engine name to engine configuration - pub engines: HashMap, - /// List of test steps to run - pub steps: Vec, -} - -pub struct Schedule { - /// Engine names to engine instances - engines: HashMap>, - /// List of test steps to run - steps: Vec, - /// Path of the schedule file - schedule_file: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Step { - /// Engine name - engine: String, - /// Stl file path - slt: String, -} - -impl Schedule { - pub fn new( - engines: HashMap>, - steps: Vec, - schedule_file: String, - ) -> Self { - Self { - engines, - steps, - schedule_file, - } - } - - pub async fn from_file>(path: P) -> anyhow::Result { - let path_str = path.as_ref().to_string_lossy().to_string(); - let content = read_to_string(path)?; - - let config: ScheduleConfig = toml::from_str(&content) - .with_context(|| format!("Failed to parse schedule file: {path_str}"))?; - - let engines = Self::instantiate_engines(config.engines).await?; - - Ok(Self::new(engines, config.steps, path_str)) - } - - /// Instantiate engine runners from their configurations - async fn instantiate_engines( - configs: HashMap, - ) -> anyhow::Result>> { - let mut engines = HashMap::new(); - - for (name, config) in configs { - let engine = load_engine_runner(config).await?; - engines.insert(name, engine); - } - - Ok(engines) - } - - pub async fn run(mut self) -> anyhow::Result<()> { - info!("Starting test run with schedule: {}", self.schedule_file); - - for (idx, step) in self.steps.iter().enumerate() { - info!( - "Running step {}/{}, using engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - - let engine = self - .engines - .get_mut(&step.engine) - .ok_or_else(|| anyhow!("Engine {} not found", step.engine))?; - - let step_sql_path = PathBuf::from(format!( - "{}/testdata/slts/{}", - env!("CARGO_MANIFEST_DIR"), - step.slt - )); - - engine.run_slt_file(&step_sql_path).await?; - - info!( - "Completed step {}/{}, engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::engine::EngineConfig; - use crate::schedule::ScheduleConfig; - - #[test] - fn test_deserialize_schedule_config() { - let input = r#" - [engines] - df = { type = "datafusion" } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - assert_eq!(config.engines.len(), 1); - assert!(config.engines.contains_key("df")); - assert!(matches!(config.engines["df"], EngineConfig::Datafusion { - catalog: None - })); - assert_eq!(config.steps.len(), 1); - assert_eq!(config.steps[0].engine, "df"); - assert_eq!(config.steps[0].slt, "test.slt"); - } - - #[test] - fn test_deserialize_multiple_steps() { - let input = r#" - [engines] - datafusion = { type = "datafusion" } - - [[steps]] - engine = "datafusion" - slt = "test.slt" - - [[steps]] - engine = "datafusion" - slt = "test2.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - assert_eq!(config.steps.len(), 2); - assert_eq!(config.steps[0].engine, "datafusion"); - assert_eq!(config.steps[0].slt, "test.slt"); - assert_eq!(config.steps[1].engine, "datafusion"); - assert_eq!(config.steps[1].slt, "test2.slt"); - } - - #[test] - fn test_deserialize_with_catalog_config() { - let input = r#" - [engines.df] - type = "datafusion" - - [engines.df.catalog] - type = "rest" - - [engines.df.catalog.props] - uri = "http://localhost:8181" - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let config: ScheduleConfig = toml::from_str(input).unwrap(); - - match &config.engines["df"] { - EngineConfig::Datafusion { catalog: Some(cat) } => { - assert_eq!(cat.catalog_type, "rest"); - assert_eq!( - cat.props.get("uri"), - Some(&"http://localhost:8181".to_string()) - ); - } - _ => panic!("Expected Datafusion with catalog config"), - } - } - - #[test] - fn test_deserialize_missing_engine_type() { - let input = r#" - [engines] - df = { } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } - - #[test] - fn test_deserialize_invalid_engine_type() { - let input = r#" - [engines] - df = { type = "unknown_engine" } - - [[steps]] - engine = "df" - slt = "test.slt" - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } - - #[test] - fn test_deserialize_missing_step_fields() { - let input = r#" - [engines] - df = { type = "datafusion" } - - [[steps]] - "#; - - let result: Result = toml::from_str(input); - assert!(result.is_err()); - } -} diff --git a/crates/sqllogictest/testdata/schedules/df_basic_queries.toml b/crates/sqllogictest/testdata/schedules/df_basic_queries.toml deleted file mode 100644 index c0d5439d30..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_basic_queries.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/basic_queries.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml deleted file mode 100644 index de0093acb2..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/binary_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml deleted file mode 100644 index 5b4f63dc7c..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/boolean_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_create_table.toml b/crates/sqllogictest/testdata/schedules/df_create_table.toml deleted file mode 100644 index 1935c9fbd7..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_create_table.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/create_table.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_drop_table.toml b/crates/sqllogictest/testdata/schedules/df_drop_table.toml deleted file mode 100644 index 8a6804dff7..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_drop_table.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/drop_table.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml b/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml deleted file mode 100644 index 231649ef3b..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/encrypted_insert_round_trip.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml b/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml deleted file mode 100644 index 0c89a0426a..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_insert_and_like.toml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/insert_into.slt" - -[[steps]] -engine = "df" -slt = "df_test/like_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml b/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml deleted file mode 100644 index cb97fec641..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_scan_all_types.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/scan_all_types.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_show_tables.toml b/crates/sqllogictest/testdata/schedules/df_show_tables.toml deleted file mode 100644 index 5fdd8eec93..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_show_tables.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/show_tables.slt" diff --git a/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml b/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml deleted file mode 100644 index 849bba50a2..0000000000 --- a/crates/sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[engines] -df = { type = "datafusion" } - -[[steps]] -engine = "df" -slt = "df_test/timestamp_predicate_pushdown.slt" diff --git a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt b/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt deleted file mode 100644 index a5ca4de46a..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/basic_queries.slt +++ /dev/null @@ -1,224 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test basic SELECT queries with ORDER BY, LIMIT, and various clauses -# This covers the functionality from the original datafusion.rs integration test - -# Create a test table with multiple columns for query testing -statement ok -CREATE TABLE default.default.query_test_table ( - id INT NOT NULL, - name STRING, - score DOUBLE, - category STRING -) - -# Insert test data -query I -INSERT INTO default.default.query_test_table VALUES - (1, 'Alice', 95.5, 'A'), - (2, 'Bob', 87.0, 'B'), - (3, 'Charlie', 92.5, 'A'), - (4, 'Diana', 78.5, 'C'), - (5, 'Eve', 88.0, 'B'), - (6, 'Frank', 91.0, 'A'), - (7, 'Grace', 85.5, 'B'), - (8, 'Henry', 79.0, 'C'), - (9, 'Ivy', 96.5, 'A'), - (10, 'Jack', 82.0, 'C') ----- -10 - -# Verify EXPLAIN shows limit is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.query_test_table LIMIT 3 ----- -logical_plan -01)Limit: skip=0, fetch=3 -02)--TableScan: default.default.query_test_table projection=[id, name, score, category], fetch=3 -physical_plan -01)GlobalLimitExec: skip=0, fetch=3 -02)--CooperativeExec -03)----IcebergTableScan projection:[id,name,score,category] predicate:[] limit:[3] - -# Test SELECT * with ORDER BY and LIMIT -query ITRT -SELECT * FROM default.default.query_test_table ORDER BY id LIMIT 3 ----- -1 Alice 95.5 A -2 Bob 87 B -3 Charlie 92.5 A - -# Test ORDER BY DESC with LIMIT -query ITRT -SELECT * FROM default.default.query_test_table ORDER BY score DESC LIMIT 5 ----- -9 Ivy 96.5 A -1 Alice 95.5 A -3 Charlie 92.5 A -6 Frank 91 A -5 Eve 88 B - -# Test ORDER BY with multiple columns -query ITR -SELECT id, name, score FROM default.default.query_test_table ORDER BY category, score DESC ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 -5 Eve 88 -2 Bob 87 -7 Grace 85.5 -10 Jack 82 -8 Henry 79 -4 Diana 78.5 - -# Test LIMIT with OFFSET -query IT -SELECT id, name FROM default.default.query_test_table ORDER BY id LIMIT 3 OFFSET 5 ----- -6 Frank -7 Grace -8 Henry - -# Test WHERE clause with ORDER BY -query ITR -SELECT id, name, score FROM default.default.query_test_table WHERE score > 90 ORDER BY score DESC ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 - -# Test WHERE clause with string comparison -query ITT rowsort -SELECT id, name, category FROM default.default.query_test_table WHERE category = 'A' ----- -1 Alice A -3 Charlie A -6 Frank A -9 Ivy A - -# Test aggregation with GROUP BY -query TIR rowsort -SELECT category, COUNT(*), AVG(score) FROM default.default.query_test_table GROUP BY category ----- -A 4 93.875 -B 3 86.833333333333 -C 3 79.833333333333 - -# Test aggregation with HAVING clause -query TI rowsort -SELECT category, COUNT(*) as cnt FROM default.default.query_test_table GROUP BY category HAVING COUNT(*) >= 4 ----- -A 4 - -# Test DISTINCT -query T rowsort -SELECT DISTINCT category FROM default.default.query_test_table ----- -A -B -C - -# Test COUNT with DISTINCT -query I -SELECT COUNT(DISTINCT category) FROM default.default.query_test_table ----- -3 - -# Test MIN and MAX -query RR -SELECT MIN(score), MAX(score) FROM default.default.query_test_table ----- -78.5 96.5 - -# Test SUM -query R -SELECT SUM(score) FROM default.default.query_test_table ----- -875.5 - -# Test combined WHERE, ORDER BY, and LIMIT -query ITR -SELECT id, name, score FROM default.default.query_test_table -WHERE category IN ('A', 'B') AND score > 85 -ORDER BY score DESC -LIMIT 4 ----- -9 Ivy 96.5 -1 Alice 95.5 -3 Charlie 92.5 -6 Frank 91 - -# Test aliasing -query IT -SELECT id AS user_id, name AS user_name FROM default.default.query_test_table ORDER BY user_id LIMIT 2 ----- -1 Alice -2 Bob - -# Test expression in SELECT -query ITR -SELECT id, name, score * 1.1 AS adjusted_score FROM default.default.query_test_table ORDER BY id LIMIT 3 ----- -1 Alice 105.05 -2 Bob 95.7 -3 Charlie 101.75 - -# Test BETWEEN -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE score BETWEEN 85 AND 92 ----- -2 Bob -5 Eve -6 Frank -7 Grace - -# Test NOT IN -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE category NOT IN ('A') ----- -10 Jack -2 Bob -4 Diana -5 Eve -7 Grace -8 Henry - -# Test LIKE pattern matching -query IT rowsort -SELECT id, name FROM default.default.query_test_table WHERE name LIKE 'A%' ----- -1 Alice - -# Test OR condition -query ITT rowsort -SELECT id, name, category FROM default.default.query_test_table WHERE category = 'A' OR score > 90 ----- -1 Alice A -3 Charlie A -6 Frank A -9 Ivy A - -# Test AND condition -query ITR rowsort -SELECT id, name, score FROM default.default.query_test_table WHERE category = 'A' AND score > 93 ----- -1 Alice 95.5 -9 Ivy 96.5 diff --git a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt deleted file mode 100644 index aa68ab2762..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test that binary predicates are pushed down to IcebergTableScan -# This validates that ScalarValue::Binary is correctly converted to Datum::binary - -# Verify EXPLAIN shows binary predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -logical_plan -01)Filter: default.default.test_binary_table.data = LargeBinary("1,2") -02)--TableScan: default.default.test_binary_table projection=[id, data], partial_filters=[default.default.test_binary_table.data = LargeBinary("1,2")] -physical_plan -01)FilterExec: data@1 = 0102 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,data] predicate:[data = 0102] - -# Verify empty result from empty table -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- - -# Insert test data -query I -INSERT INTO default.default.test_binary_table VALUES (1, X'0102') ----- -1 - -# Verify binary predicate filtering works -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -1 0102 - -# Insert more rows with different binary values -query I -INSERT INTO default.default.test_binary_table VALUES (2, X'0304'), (3, X'0102'), (4, X'0506') ----- -3 - -# Verify binary predicate filters correctly -query I? rowsort -SELECT * FROM default.default.test_binary_table WHERE data = X'0102' ----- -1 0102 -3 0102 - -# Verify different binary value -query I? -SELECT * FROM default.default.test_binary_table WHERE data = X'0506' ----- -4 0506 diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt deleted file mode 100644 index 496f719261..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Create test table with boolean column -statement ok -CREATE TABLE default.default.test_boolean_table (id INT NOT NULL, is_active BOOLEAN, description STRING) - -# Insert test data into test_boolean_table -statement ok -INSERT INTO default.default.test_boolean_table VALUES - (1, true, 'Active user'), - (2, false, 'Inactive user'), - (3, true, 'Premium member'), - (4, false, 'Trial expired'), - (5, true, 'Verified account'), - (6, NULL, 'Pending verification') - -# Verify boolean equality predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active = true ----- -logical_plan -01)Filter: default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] - -# Query with is_active = true -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true ----- -1 true Active user -3 true Premium member -5 true Verified account - -# Verify boolean false predicate is pushed down to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active = false ----- -logical_plan -01)Filter: NOT default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] - -# Query with is_active = false -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = false ----- -2 false Inactive user -4 false Trial expired - -# Verify boolean NOT EQUAL predicate is pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_boolean_table WHERE is_active != true ----- -logical_plan -01)Filter: NOT default.default.test_boolean_table.is_active -02)--TableScan: default.default.test_boolean_table projection=[id, is_active, description], partial_filters=[NOT default.default.test_boolean_table.is_active] -physical_plan -01)FilterExec: NOT is_active@1 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] - -# Query with is_active != true (includes false and NULL) -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active != true ----- -2 false Inactive user -4 false Trial expired - -# Test combined boolean predicates with AND -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true AND id > 2 ----- -3 true Premium member -5 true Verified account - -# Test combined boolean predicates with OR -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active = true OR id = 2 ----- -1 true Active user -2 false Inactive user -3 true Premium member -5 true Verified account - -# Test IS NULL on boolean column -query ITT -SELECT * FROM default.default.test_boolean_table WHERE is_active IS NULL ----- -6 NULL Pending verification - -# Test IS NOT NULL on boolean column -query ITT rowsort -SELECT * FROM default.default.test_boolean_table WHERE is_active IS NOT NULL ----- -1 true Active user -2 false Inactive user -3 true Premium member -4 false Trial expired -5 true Verified account - -# Clean up: Drop the test table -statement ok -DROP TABLE default.default.test_boolean_table diff --git a/crates/sqllogictest/testdata/slts/df_test/create_table.slt b/crates/sqllogictest/testdata/slts/df_test/create_table.slt deleted file mode 100644 index 2eab1b6bab..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/create_table.slt +++ /dev/null @@ -1,90 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test CREATE TABLE with explicit schema -statement ok -CREATE TABLE default.default.empty_table (id INT NOT NULL, name STRING) - -# Verify the empty table exists and has correct schema -query IT rowsort -SELECT * FROM default.default.empty_table ----- - -# Insert data into the created table -query I -INSERT INTO default.default.empty_table VALUES (1, 'Alice') ----- -1 - -# Verify the inserted data -query IT rowsort -SELECT * FROM default.default.empty_table ----- -1 Alice - -# Insert multiple rows -query I -INSERT INTO default.default.empty_table VALUES (2, 'Bob'), (3, 'Charlie') ----- -2 - -# Verify all rows -query IT rowsort -SELECT * FROM default.default.empty_table ----- -1 Alice -2 Bob -3 Charlie - -# Test CREATE TABLE with different column types -statement ok -CREATE TABLE default.default.typed_table (id BIGINT NOT NULL, value DOUBLE, flag BOOLEAN) - -# Verify the typed table exists -query IDT rowsort -SELECT * FROM default.default.typed_table ----- - -# Insert data with different types -query I -INSERT INTO default.default.typed_table VALUES (100, 3.14, true), (200, 2.71, false) ----- -2 - -# Verify typed data -query IDT rowsort -SELECT * FROM default.default.typed_table ----- -100 3.14 true -200 2.71 false - -# Test CREATE TABLE with nullable columns -statement ok -CREATE TABLE default.default.nullable_table (id INT NOT NULL, optional_name STRING) - -# Insert with NULL value -query I -INSERT INTO default.default.nullable_table VALUES (1, 'Value'), (2, NULL) ----- -2 - -# Verify NULL handling -query IT rowsort -SELECT * FROM default.default.nullable_table ----- -1 Value -2 NULL diff --git a/crates/sqllogictest/testdata/slts/df_test/drop_table.slt b/crates/sqllogictest/testdata/slts/df_test/drop_table.slt deleted file mode 100644 index c721c84e11..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/drop_table.slt +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test DROP TABLE functionality - -# Create a table to drop -statement ok -CREATE TABLE default.default.table_to_drop (id INT NOT NULL, name STRING) - -# Verify the table exists -query IT rowsort -SELECT * FROM default.default.table_to_drop ----- - -# Insert some data -query I -INSERT INTO default.default.table_to_drop VALUES (1, 'Test') ----- -1 - -# Verify data exists -query IT rowsort -SELECT * FROM default.default.table_to_drop ----- -1 Test - -# Drop the table -statement ok -DROP TABLE default.default.table_to_drop - -# Verify the table no longer exists (should error) -statement error -SELECT * FROM default.default.table_to_drop - -# Test DROP TABLE IF EXISTS on non-existent table (should not error) -statement ok -DROP TABLE IF EXISTS default.default.nonexistent_table - -# Create another table for IF EXISTS test -statement ok -CREATE TABLE default.default.another_table (id INT NOT NULL) - -# Drop with IF EXISTS -statement ok -DROP TABLE IF EXISTS default.default.another_table - -# Verify it's gone -statement error -SELECT * FROM default.default.another_table diff --git a/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt b/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt deleted file mode 100644 index f1760242f0..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -query II rowsort -SELECT * FROM default.default.test_encrypted_round_trip ----- - -query I -INSERT INTO default.default.test_encrypted_round_trip VALUES (1, 'Alice'), (2, 'Bob') ----- -2 - -query II rowsort -SELECT * FROM default.default.test_encrypted_round_trip ----- -1 Alice -2 Bob diff --git a/crates/sqllogictest/testdata/slts/df_test/insert_into.slt b/crates/sqllogictest/testdata/slts/df_test/insert_into.slt deleted file mode 100644 index 56fc1fefb1..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/insert_into.slt +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Create unpartitioned test table -statement ok -CREATE TABLE default.default.test_unpartitioned_table (id INT NOT NULL, name STRING) - -# Verify the table is initially empty -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- - -# Insert zero rows and verify it returns count 0 without creating a snapshot -query I -INSERT INTO default.default.test_unpartitioned_table SELECT * FROM (VALUES (1, 'Nobody')) AS t(id, name) WHERE false ----- -0 - -query I -SELECT COUNT(*) FROM default.default.test_unpartitioned_table ----- -0 - -query ?????? -SELECT * FROM default.default.test_unpartitioned_table$snapshots ----- - -# Insert a single row and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (1, 'Alice') ----- -1 - -# Verify the inserted row -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice - -# Insert multiple rows and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (2, 'Bob'), (3, 'Charlie') ----- -2 - -# Verify all rows -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice -2 Bob -3 Charlie - -# Insert with NULL value and verify the count -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (4, NULL) ----- -1 - -# Verify NULL handling -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table ----- -1 Alice -2 Bob -3 Charlie -4 NULL - -# Test partitioned table - verify initially empty -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- - -# Insert zero rows into a partitioned table and verify it returns count 0 without creating a snapshot -query I -INSERT INTO default.default.test_partitioned_table SELECT * FROM (VALUES (1, 'electronics', 'laptop')) AS t(id, category, value) WHERE false ----- -0 - -query I -SELECT COUNT(*) FROM default.default.test_partitioned_table ----- -0 - -query ?????? -SELECT * FROM default.default.test_partitioned_table$snapshots ----- - -# Insert single row into partitioned table -query I -INSERT INTO default.default.test_partitioned_table VALUES (1, 'electronics', 'laptop') ----- -1 - -# Verify the inserted row in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop - -# Insert multiple rows with different partition values -query I -INSERT INTO default.default.test_partitioned_table VALUES (2, 'electronics', 'phone'), (3, 'books', 'novel'), (4, 'books', 'textbook'), (5, 'clothing', 'shirt') ----- -4 - -# Verify all rows in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop -2 electronics phone -3 books novel -4 books textbook -5 clothing shirt - -# Insert with NULL value in optional column -query I -INSERT INTO default.default.test_partitioned_table VALUES (6, 'electronics', NULL) ----- -1 - -# Verify NULL handling in partitioned table -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table ----- -1 electronics laptop -2 electronics phone -3 books novel -4 books textbook -5 clothing shirt -6 electronics NULL - -# Verify partition filtering works -query ITT rowsort -SELECT * FROM default.default.test_partitioned_table WHERE category = 'books' ----- -3 books novel -4 books textbook diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt deleted file mode 100644 index 3d8b151aa9..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ /dev/null @@ -1,109 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test LIKE predicate pushdown for StartsWith patterns -# This validates that LIKE 'prefix%' patterns are converted to StartsWith operator -# Note: Iceberg's StartsWith operator is case-sensitive - -# Note: test_unpartitioned_table already contains data from insert_into.slt: -# (1, 'Alice'), (2, 'Bob'), (3, 'Charlie'), (4, NULL) -# We'll insert additional test data for LIKE testing -query I -INSERT INTO default.default.test_unpartitioned_table VALUES (5, 'alice'), (6, 'Albert'), (7, 'Bobby'), (8, 'Carol') ----- -4 - -# Test basic LIKE prefix pattern pushdown - verify predicate is pushed to IcebergTableScan -query TT -EXPLAIN SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' ----- -logical_plan -01)Filter: default.default.test_unpartitioned_table.name LIKE Utf8("Al%") -02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name LIKE Utf8("Al%")] -physical_plan -01)FilterExec: name@1 LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] - -# Test LIKE filtering with case-sensitive match -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' ----- -1 Alice -6 Albert - -# Test NOT LIKE prefix pattern pushdown -query TT -EXPLAIN SELECT * FROM default.default.test_unpartitioned_table WHERE name NOT LIKE 'Al%' ----- -logical_plan -01)Filter: default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%") -02)--TableScan: default.default.test_unpartitioned_table projection=[id, name], partial_filters=[default.default.test_unpartitioned_table.name NOT LIKE Utf8("Al%")] -physical_plan -01)FilterExec: name@1 NOT LIKE Al% -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] - -# Test NOT LIKE filtering -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name NOT LIKE 'Al%' ----- -2 Bob -3 Charlie -5 alice -7 Bobby -8 Carol - -# Test case sensitivity - lowercase 'al%' should not match 'Alice' or 'Albert' -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'al%' ----- -5 alice - -# Test LIKE with empty prefix (matches everything except NULL) -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE '%' ----- -1 Alice -2 Bob -3 Charlie -5 alice -6 Albert -7 Bobby -8 Carol - -# Test LIKE with single character prefix -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'B%' ----- -2 Bob -7 Bobby - -# Test LIKE combined with other predicates -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' AND id > 1 ----- -6 Albert - -# Test LIKE in OR expression -query IT rowsort -SELECT * FROM default.default.test_unpartitioned_table WHERE name LIKE 'Al%' OR name LIKE 'Bo%' ----- -1 Alice -2 Bob -6 Albert -7 Bobby diff --git a/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt b/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt deleted file mode 100644 index d93e461333..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/scan_all_types.slt +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test scanning all primitive types supported by Iceberg -# This tests INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, BOOLEAN, DATE, TIMESTAMP - -# Create a table with all primitive types -statement ok -CREATE TABLE default.default.all_primitive_types ( - col_int INT NOT NULL, - col_long BIGINT NOT NULL, - col_float FLOAT NOT NULL, - col_double DOUBLE NOT NULL, - col_decimal DECIMAL(10, 2), - col_string STRING, - col_boolean BOOLEAN -) - -# Verify the table is initially empty -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types ----- - -# Insert test data with various primitive types -query I -INSERT INTO default.default.all_primitive_types VALUES - (1, 100, 1.5, 2.5, 123.45, 'hello', true), - (2, 200, 2.5, 3.5, 234.56, 'world', false), - (3, 300, 3.5, 4.5, 345.67, NULL, true), - (-1, -100, -1.5, -2.5, -123.45, 'negative', false), - (0, 0, 0.0, 0.0, 0.00, 'zero', NULL) ----- -5 - -# Verify all inserted rows -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types ----- --1 -100 -1.5 -2.5 -123.45 negative false -0 0 0 0 0 zero NULL -1 100 1.5 2.5 123.45 hello true -2 200 2.5 3.5 234.56 world false -3 300 3.5 4.5 345.67 NULL true - -# Test selecting specific columns with string filter -query IT rowsort -SELECT col_int, col_string FROM default.default.all_primitive_types WHERE col_string = 'hello' ----- -1 hello - -# Test numeric comparisons -query IR rowsort -SELECT col_int, col_decimal FROM default.default.all_primitive_types WHERE col_decimal > 200.00 ----- -2 234.56 -3 345.67 - -# Test filtering on int column -query IIRRTTT rowsort -SELECT * FROM default.default.all_primitive_types WHERE col_int >= 0 ORDER BY col_int ----- -0 0 0 0 0 zero NULL -1 100 1.5 2.5 123.45 hello true -2 200 2.5 3.5 234.56 world false -3 300 3.5 4.5 345.67 NULL true - -# Test filtering with NULL -query IT rowsort -SELECT col_int, col_string FROM default.default.all_primitive_types WHERE col_string IS NOT NULL ----- --1 negative -0 zero -1 hello -2 world - -# Test aggregations on numeric types -query IIRR -SELECT COUNT(*), SUM(col_int), AVG(col_float), MAX(col_double) FROM default.default.all_primitive_types ----- -5 5 1.2 4.5 - -# Create a table with date and timestamp types -statement ok -CREATE TABLE default.default.temporal_types ( - id INT NOT NULL, - col_date DATE, - col_timestamp TIMESTAMP -) - -# Insert temporal data -query I -INSERT INTO default.default.temporal_types VALUES - (1, '2024-01-15', '2024-01-15 10:30:00'), - (2, '2024-06-20', '2024-06-20 14:45:30'), - (3, '2023-12-31', '2023-12-31 23:59:59'), - (4, NULL, NULL) ----- -4 - -# Verify temporal data -query IDP rowsort -SELECT * FROM default.default.temporal_types ----- -1 2024-01-15 2024-01-15T10:30:00 -2 2024-06-20 2024-06-20T14:45:30 -3 2023-12-31 2023-12-31T23:59:59 -4 NULL NULL - -# Test date comparisons -query ID rowsort -SELECT id, col_date FROM default.default.temporal_types WHERE col_date >= '2024-01-01' ----- -1 2024-01-15 -2 2024-06-20 diff --git a/crates/sqllogictest/testdata/slts/df_test/show_tables.slt b/crates/sqllogictest/testdata/slts/df_test/show_tables.slt deleted file mode 100644 index 29cc3cd7e6..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/show_tables.slt +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -query TTTT rowsort -SHOW TABLES ----- -datafusion information_schema columns VIEW -datafusion information_schema df_settings VIEW -datafusion information_schema parameters VIEW -datafusion information_schema routines VIEW -datafusion information_schema schemata VIEW -datafusion information_schema tables VIEW -datafusion information_schema views VIEW -default default test_binary_table BASE TABLE -default default test_binary_table$history BASE TABLE -default default test_binary_table$manifests BASE TABLE -default default test_binary_table$snapshots BASE TABLE -default default test_encrypted_round_trip BASE TABLE -default default test_encrypted_round_trip$history BASE TABLE -default default test_encrypted_round_trip$manifests BASE TABLE -default default test_encrypted_round_trip$snapshots BASE TABLE -default default test_partitioned_table BASE TABLE -default default test_partitioned_table$history BASE TABLE -default default test_partitioned_table$manifests BASE TABLE -default default test_partitioned_table$snapshots BASE TABLE -default information_schema columns VIEW -default information_schema df_settings VIEW -default information_schema parameters VIEW -default information_schema routines VIEW -default information_schema schemata VIEW -default information_schema tables VIEW -default information_schema views VIEW diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt deleted file mode 100644 index ffa74173dc..0000000000 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ /dev/null @@ -1,183 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Test timestamp predicate pushdown behavior -# -# When you CREATE TABLE with TIMESTAMP in DataFusion, it creates a TimestampNs column -# (nanosecond precision) in Iceberg, since DataFusion's default TIMESTAMP type is nanoseconds. -# -# We use Datum::timestamp_nanos() for nanosecond timestamp predicates to preserve -# full precision. This allows predicates to be correctly pushed down to Iceberg. - -# Create test table with timestamp column -statement ok -CREATE TABLE default.default.test_timestamp_table (id INT NOT NULL, ts TIMESTAMP) - -# Insert test data with timestamps -# We use CAST to convert string timestamps to proper timestamp values -query I -INSERT INTO default.default.test_timestamp_table -VALUES - (1, CAST('2023-01-01 00:00:00' AS TIMESTAMP)), - (2, CAST('2023-01-05 12:30:00' AS TIMESTAMP)), - (3, CAST('2023-01-10 15:45:30' AS TIMESTAMP)), - (4, CAST('2023-01-15 09:00:00' AS TIMESTAMP)), - (5, CAST('2023-01-20 18:20:10' AS TIMESTAMP)) ----- -5 - -# Verify timestamp equality predicate IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table WHERE ts = CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts = TimestampNanosecond(1672921800000000000, None)] -physical_plan -01)FilterExec: ts@1 = 1672921800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] - -# Verify timestamp equality filtering works -query I? -SELECT * FROM default.default.test_timestamp_table WHERE ts = CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 - -# Verify timestamp greater than predicate IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table WHERE ts > CAST('2023-01-10 00:00:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts > TimestampNanosecond(1673308800000000000, None)] -physical_plan -01)FilterExec: ts@1 > 1673308800000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] - -# Verify timestamp greater than filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts > CAST('2023-01-10 00:00:00' AS TIMESTAMP) ----- -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 -5 2023-01-20T18:20:10 - -# Test timestamp less than or equal filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts <= CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 -2 2023-01-05T12:30:00 - -# Verify timestamp range predicate (AND of two comparisons) IS pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-05 00:00:00' AS TIMESTAMP) - AND ts <= CAST('2023-01-15 23:59:59' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None) AND default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None) -02)--TableScan: default.default.test_timestamp_table projection=[id, ts], partial_filters=[default.default.test_timestamp_table.ts >= TimestampNanosecond(1672876800000000000, None), default.default.test_timestamp_table.ts <= TimestampNanosecond(1673827199000000000, None)] -physical_plan -01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] - -# Test timestamp range predicate filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-05 00:00:00' AS TIMESTAMP) - AND ts <= CAST('2023-01-15 23:59:59' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 - -# Test timestamp predicate combined with other predicates -query I? rowsort -SELECT * FROM default.default.test_timestamp_table -WHERE ts >= CAST('2023-01-10 00:00:00' AS TIMESTAMP) AND id < 5 ----- -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 - -# Test timestamp NOT EQUAL predicate -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts != CAST('2023-01-05 12:30:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 -3 2023-01-10T15:45:30 -4 2023-01-15T09:00:00 -5 2023-01-20T18:20:10 - -# Test timestamp less than filtering -query I? rowsort -SELECT * FROM default.default.test_timestamp_table WHERE ts < CAST('2023-01-05 00:00:00' AS TIMESTAMP) ----- -1 2023-01-01T00:00:00 - -# Clean up: Drop the test table -statement ok -DROP TABLE default.default.test_timestamp_table - -# ============================================================================ -# Test timestamp predicate pushdown with different precisions -# ============================================================================ - -# Test with TIMESTAMP(6) - microsecond precision -statement ok -CREATE TABLE default.default.test_timestamp_micros (id INT NOT NULL, ts TIMESTAMP(6)) - -query I -INSERT INTO default.default.test_timestamp_micros -VALUES - (1, CAST('2023-01-01 00:00:00' AS TIMESTAMP)), - (2, CAST('2023-01-05 12:30:00' AS TIMESTAMP)) ----- -2 - -# Verify microsecond timestamp predicate is pushed down -query TT -EXPLAIN SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP) ----- -logical_plan -01)Filter: default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None) -02)--TableScan: default.default.test_timestamp_micros projection=[id, ts], partial_filters=[default.default.test_timestamp_micros.ts > TimestampMicrosecond(1672531200000000, None)] -physical_plan -01)FilterExec: ts@1 > 1672531200000000 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] - -query I? -SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP) ----- -2 2023-01-05T12:30:00 - -statement ok -DROP TABLE default.default.test_timestamp_micros - -# Test with TIMESTAMP(3) - millisecond precision -# This should fail because Iceberg doesn't support millisecond precision -statement error DataFusion error: External error: DataInvalid => Unsupported Arrow data type: Timestamp\(ms\) -CREATE TABLE default.default.test_timestamp_millis (id INT NOT NULL, ts TIMESTAMP(3)) - -# Test with TIMESTAMP(0) - second precision -# This should fail because Iceberg doesn't support second precision -statement error DataFusion error: External error: DataInvalid => Unsupported Arrow data type: Timestamp\(s\) -CREATE TABLE default.default.test_timestamp_seconds (id INT NOT NULL, ts TIMESTAMP(0)) diff --git a/crates/sqllogictest/tests/sqllogictests.rs b/crates/sqllogictest/tests/sqllogictests.rs deleted file mode 100644 index 434d9d78cc..0000000000 --- a/crates/sqllogictest/tests/sqllogictests.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fs; -use std::path::PathBuf; - -use iceberg_sqllogictest::schedule::Schedule; -use libtest_mimic::{Arguments, Trial}; -use tokio::runtime::Handle; - -pub fn main() { - env_logger::init(); - - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); - - // Parse command line arguments - let args = Arguments::from_args(); - - let tests = collect_trials(rt.handle().clone()).unwrap(); - let result = libtest_mimic::run(&args, tests); - - drop(rt); - - result.exit(); -} - -pub(crate) fn collect_trials(handle: Handle) -> anyhow::Result> { - let schedule_files = collect_schedule_files()?; - log::debug!( - "Found {} schedules files: {:?}", - schedule_files.len(), - &schedule_files - ); - let mut trials = Vec::with_capacity(schedule_files.len()); - for schedule_file in schedule_files { - let h = handle.clone(); - let trial_name = format!( - "schedule: {}", - schedule_file - .file_name() - .expect("Schedule file should have a name") - .to_string_lossy() - ); - let trial = Trial::test(trial_name, move || { - Ok(h.block_on(run_schedule(schedule_file.clone()))?) - }); - trials.push(trial); - } - Ok(trials) -} - -pub(crate) fn collect_schedule_files() -> anyhow::Result> { - let dir = PathBuf::from(format!("{}/testdata/schedules", env!("CARGO_MANIFEST_DIR"))); - let mut schedule_files = Vec::with_capacity(32); - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - schedule_files.push(fs::canonicalize(dir.join(path))?); - } - } - Ok(schedule_files) -} - -pub(crate) async fn run_schedule(schedule_file: PathBuf) -> anyhow::Result<()> { - let schedules = Schedule::from_file(schedule_file).await?; - schedules.run().await?; - - Ok(()) -}