diff --git a/datafusion-partitioned/install b/datafusion-partitioned/install index 20f376193e..fc09c96d8e 100755 --- a/datafusion-partitioned/install +++ b/datafusion-partitioned/install @@ -26,7 +26,7 @@ if [ ! -x arrow-datafusion/target/release/datafusion-cli ]; then git clone https://github.com/apache/arrow-datafusion.git fi cd arrow-datafusion - git checkout 53.1.0 + git checkout 54.0.0 CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" \ cargo build --release --package datafusion-cli --bin datafusion-cli fi diff --git a/datafusion-partitioned/results/20260619/c6a.metal.json b/datafusion-partitioned/results/20260619/c6a.metal.json new file mode 100644 index 0000000000..1ac7072820 --- /dev/null +++ b/datafusion-partitioned/results/20260619/c6a.metal.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, partitioned)", + "date": "2026-06-19", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 49, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.064, 0.002, 0.002], + [0.121, 0.061, 0.063], + [0.205, 0.061, 0.054], + [0.413, 0.049, 0.059], + [0.64, 0.18, 0.172], + [0.845, 0.215, 0.217], + [0.065, 0.002, 0.003], + [0.138, 0.064, 0.064], + [0.613, 0.212, 0.228], + [1.366, 0.344, 0.376], + [0.474, 0.111, 0.115], + [0.599, 0.136, 0.111], + [0.922, 0.24, 0.253], + [2.17, 0.416, 0.37], + [0.877, 0.222, 0.233], + [0.408, 0.203, 0.195], + [2.08, 0.391, 0.382], + [2.061, 0.367, 0.401], + [4.17, 0.697, 0.706], + [0.227, 0.067, 0.063], + [9.685, 0.301, 0.276], + [11.135, 0.312, 0.316], + [21.804, 0.521, 0.552], + [51.443, 1.712, 1.816], + [1.49, 0.078, 0.061], + [1.162, 0.087, 0.101], + [1.725, 0.059, 0.058], + [10.112, 0.326, 0.326], + [8.148, 0.646, 0.613], + [0.16, 0.073, 0.066], + [2.798, 0.248, 0.234], + [6.406, 0.318, 0.332], + [4.481, 0.935, 0.883], + [9.523, 0.733, 0.719], + [9.528, 0.701, 0.696], + [0.331, 0.152, 0.165], + [0.256, 0.105, 0.104], + [0.195, 0.074, 0.067], + [0.242, 0.065, 0.072], + [0.388, 0.17, 0.181], + [0.163, 0.054, 0.051], + [0.153, 0.056, 0.048], + [0.158, 0.054, 0.04] + ] +} diff --git a/datafusion-partitioned/results/20260619/c7a.metal-48xl.json b/datafusion-partitioned/results/20260619/c7a.metal-48xl.json new file mode 100644 index 0000000000..5438d0dcff --- /dev/null +++ b/datafusion-partitioned/results/20260619/c7a.metal-48xl.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, partitioned)", + "date": "2026-06-19", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 55, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.045, 0.003, 0.003], + [0.099, 0.054, 0.067], + [0.129, 0.047, 0.049], + [0.451, 0.051, 0.052], + [0.902, 0.118, 0.121], + [1.116, 0.158, 0.156], + [0.047, 0.003, 0.003], + [0.109, 0.057, 0.061], + [0.661, 0.16, 0.145], + [1.601, 0.336, 0.277], + [0.46, 0.09, 0.095], + [0.855, 0.096, 0.09], + [1.186, 0.165, 0.165], + [2.283, 0.244, 0.251], + [0.906, 0.154, 0.163], + [0.582, 0.127, 0.128], + [2.385, 0.256, 0.246], + [2.141, 0.269, 0.262], + [4.222, 0.377, 0.383], + [0.167, 0.061, 0.067], + [9.968, 0.167, 0.18], + [11.246, 0.226, 0.232], + [21.818, 0.302, 0.318], + [49.199, 3.118, 0.863], + [1.388, 0.06, 0.056], + [1.247, 0.061, 0.067], + [1.993, 0.054, 0.054], + [10.235, 0.2, 0.202], + [8.159, 0.466, 0.451], + [0.116, 0.056, 0.059], + [2.828, 0.153, 0.15], + [6.563, 0.214, 0.208], + [4.515, 0.512, 0.507], + [9.581, 0.375, 0.357], + [9.581, 0.368, 0.356], + [0.19, 0.108, 0.108], + [0.222, 0.096, 0.091], + [0.154, 0.069, 0.079], + [0.186, 0.061, 0.063], + [0.32, 0.173, 0.174], + [0.127, 0.045, 0.055], + [0.12, 0.048, 0.05], + [0.107, 0.045, 0.056] + ] +} diff --git a/datafusion-partitioned/results/20260619/c8g.4xlarge.json b/datafusion-partitioned/results/20260619/c8g.4xlarge.json new file mode 100644 index 0000000000..37e07b70f1 --- /dev/null +++ b/datafusion-partitioned/results/20260619/c8g.4xlarge.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, partitioned)", + "date": "2026-06-19", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 17, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.038, 0.002, 0.002], + [0.077, 0.015, 0.014], + [0.264, 0.039, 0.042], + [0.482, 0.034, 0.033], + [1.11, 0.203, 0.204], + [1.043, 0.239, 0.241], + [0.044, 0.002, 0.002], + [0.09, 0.017, 0.018], + [0.826, 0.306, 0.283], + [1.477, 0.371, 0.368], + [0.555, 0.074, 0.079], + [1.047, 0.09, 0.089], + [1.313, 0.254, 0.254], + [2.373, 0.328, 0.335], + [0.951, 0.245, 0.245], + [0.638, 0.257, 0.256], + [2.355, 0.501, 0.5], + [2.345, 0.496, 0.504], + [4.632, 1.012, 0.997], + [0.242, 0.032, 0.033], + [10.202, 0.429, 0.428], + [11.39, 0.506, 0.511], + [22.29, 0.907, 0.898], + [54.509, 2.776, 2.881], + [0.248, 0.046, 0.047], + [1.098, 0.107, 0.108], + [0.365, 0.044, 0.045], + [10.413, 0.594, 0.596], + [8.29, 2.426, 2.429], + [0.124, 0.043, 0.045], + [3.156, 0.265, 0.263], + [7.015, 0.256, 0.262], + [4.936, 0.764, 0.764], + [9.831, 1.288, 1.271], + [9.832, 1.273, 1.303], + [0.369, 0.247, 0.244], + [0.192, 0.072, 0.072], + [0.14, 0.04, 0.048], + [0.183, 0.042, 0.042], + [0.313, 0.145, 0.143], + [0.112, 0.017, 0.016], + [0.099, 0.016, 0.016], + [0.101, 0.016, 0.015] + ] +} diff --git a/datafusion-partitioned/results/20260619/c8g.metal-48xl.json b/datafusion-partitioned/results/20260619/c8g.metal-48xl.json new file mode 100644 index 0000000000..b0d6270421 --- /dev/null +++ b/datafusion-partitioned/results/20260619/c8g.metal-48xl.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, partitioned)", + "date": "2026-06-19", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 55, + "data_size": 14737666736, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.028, 0.002, 0.002], + [0.059, 0.034, 0.033], + [0.154, 0.081, 0.06], + [0.402, 0.033, 0.056], + [0.954, 0.111, 0.098], + [1.214, 0.14, 0.136], + [0.032, 0.002, 0.002], + [0.073, 0.043, 0.043], + [0.622, 0.144, 0.15], + [1.56, 0.259, 0.235], + [0.446, 0.076, 0.072], + [0.962, 0.076, 0.097], + [1.252, 0.143, 0.128], + [2.388, 0.227, 0.242], + [0.874, 0.154, 0.158], + [0.638, 0.115, 0.109], + [2.516, 0.221, 0.229], + [2.228, 0.223, 0.22], + [4.271, 0.412, 0.382], + [0.174, 0.035, 0.035], + [10.086, 0.185, 0.183], + [11.305, 0.216, 0.234], + [21.868, 0.375, 0.372], + [51.578, 1.286, 0.807], + [1.603, 0.052, 0.04], + [1.291, 0.065, 0.051], + [1.869, 0.056, 0.049], + [10.309, 0.224, 0.215], + [8.24, 0.544, 0.522], + [0.116, 0.048, 0.045], + [2.807, 0.132, 0.142], + [6.725, 0.181, 0.176], + [4.644, 0.443, 0.436], + [9.554, 0.498, 0.487], + [9.576, 0.507, 0.547], + [0.232, 0.094, 0.095], + [0.178, 0.087, 0.088], + [0.128, 0.056, 0.055], + [0.163, 0.057, 0.057], + [0.298, 0.156, 0.157], + [0.096, 0.036, 0.034], + [0.087, 0.034, 0.034], + [0.086, 0.033, 0.032] + ] +} diff --git a/datafusion/README.md b/datafusion/README.md index 3a4de18ef9..c870d84be1 100644 --- a/datafusion/README.md +++ b/datafusion/README.md @@ -28,7 +28,7 @@ All with no EBS optimization and no instance store. 5. `vi install` and modify the following line to target the DataFusion version ```bash - git checkout 53.1.0 + git checkout 54.0.0 ``` 6. `bash benchmark.sh` diff --git a/datafusion/install b/datafusion/install index 8f4cee6f1e..7de35c4566 100755 --- a/datafusion/install +++ b/datafusion/install @@ -31,7 +31,7 @@ if [ ! -x arrow-datafusion/target/release/datafusion-cli ]; then git clone https://github.com/apache/arrow-datafusion.git fi cd arrow-datafusion - git checkout 53.1.0 + git checkout 54.0.0 CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" \ cargo build --release --package datafusion-cli --bin datafusion-cli fi diff --git a/datafusion/results/20260619/c6a.metal.json b/datafusion/results/20260619/c6a.metal.json new file mode 100644 index 0000000000..9182a1a5db --- /dev/null +++ b/datafusion/results/20260619/c6a.metal.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, single)", + "date": "2026-06-19", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 6, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.07, 0.001, 0.002], + [0.191, 0.137, 0.122], + [0.162, 0.072, 0.067], + [0.282, 0.069, 0.071], + [0.655, 0.164, 0.166], + [0.819, 0.245, 0.231], + [0.055, 0.001, 0.002], + [0.198, 0.139, 0.143], + [0.552, 0.204, 0.191], + [1.15, 0.311, 0.339], + [0.379, 0.181, 0.178], + [0.413, 0.18, 0.184], + [0.784, 0.285, 0.289], + [2.138, 0.451, 0.444], + [0.852, 0.299, 0.287], + [0.318, 0.184, 0.192], + [2.072, 0.401, 0.393], + [1.966, 0.366, 0.392], + [4.153, 0.702, 0.723], + [0.244, 0.135, 0.144], + [9.546, 0.315, 0.32], + [11.146, 0.401, 0.428], + [21.861, 0.71, 0.774], + [55.936, 2.428, 2.557], + [2.432, 0.194, 0.199], + [0.719, 0.186, 0.187], + [2.419, 0.194, 0.176], + [9.697, 0.418, 0.436], + [8.122, 0.911, 0.937], + [0.149, 0.063, 0.061], + [2.622, 0.313, 0.304], + [5.915, 0.375, 0.36], + [4.218, 0.917, 0.985], + [9.568, 0.704, 0.689], + [9.574, 0.705, 0.704], + [0.365, 0.172, 0.158], + [0.321, 0.204, 0.165], + [0.267, 0.184, 0.186], + [0.285, 0.129, 0.151], + [0.506, 0.264, 0.315], + [0.25, 0.12, 0.133], + [0.223, 0.119, 0.143], + [0.235, 0.128, 0.12] + ] +} diff --git a/datafusion/results/20260619/c7a.metal-48xl.json b/datafusion/results/20260619/c7a.metal-48xl.json new file mode 100644 index 0000000000..6e9683e49d --- /dev/null +++ b/datafusion/results/20260619/c7a.metal-48xl.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, single)", + "date": "2026-06-19", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 6, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.07, 0.001, 0.002], + [0.184, 0.12, 0.095], + [0.164, 0.056, 0.064], + [0.267, 0.07, 0.074], + [0.72, 0.108, 0.119], + [0.914, 0.167, 0.178], + [0.069, 0.001, 0.002], + [0.184, 0.108, 0.111], + [0.562, 0.139, 0.14], + [1.194, 0.254, 0.307], + [0.377, 0.135, 0.147], + [0.436, 0.146, 0.149], + [0.814, 0.195, 0.205], + [2.06, 0.294, 0.316], + [1.016, 0.187, 0.184], + [0.317, 0.113, 0.128], + [2.07, 0.26, 0.268], + [1.987, 0.239, 0.256], + [3.889, 0.424, 0.427], + [0.246, 0.106, 0.097], + [9.579, 0.203, 0.23], + [11.116, 0.254, 0.271], + [21.869, 0.501, 0.508], + [55.918, 1.603, 1.527], + [2.461, 0.15, 0.155], + [0.777, 0.133, 0.128], + [2.513, 0.136, 0.138], + [9.718, 0.286, 0.273], + [8.137, 0.69, 0.688], + [0.157, 0.078, 0.043], + [2.574, 0.215, 0.205], + [5.944, 0.24, 0.242], + [4.179, 0.541, 0.543], + [9.594, 0.417, 0.407], + [9.624, 0.4, 0.393], + [0.276, 0.114, 0.112], + [0.339, 0.161, 0.145], + [0.286, 0.126, 0.129], + [0.281, 0.099, 0.111], + [0.452, 0.255, 0.245], + [0.203, 0.092, 0.097], + [0.188, 0.094, 0.099], + [0.189, 0.09, 0.1] + ] +} diff --git a/datafusion/results/20260619/c8g.4xlarge.json b/datafusion/results/20260619/c8g.4xlarge.json new file mode 100644 index 0000000000..c22e359811 --- /dev/null +++ b/datafusion/results/20260619/c8g.4xlarge.json @@ -0,0 +1,59 @@ +{ + "system": "DataFusion (Parquet, single)", + "date": "2026-06-19", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","column-oriented","embedded","stateless"], + "load_time": 10, + "data_size": 14779976446, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.04, 0.001, 0.001], + [0.081, 0.019, 0.019], + [0.135, 0.043, 0.041], + [0.376, 0.039, 0.038], + [0.925, 0.218, 0.231], + [0.873, 0.301, 0.293], + [0.045, 0.001, 0.001], + [0.09, 0.02, 0.021], + [0.674, 0.314, 0.304], + [1.163, 0.439, 0.449], + [0.45, 0.093, 0.096], + [0.929, 0.103, 0.104], + [1.206, 0.302, 0.301], + [2.227, 0.382, 0.359], + [0.927, 0.31, 0.307], + [0.48, 0.241, 0.241], + [2.25, 0.518, 0.509], + [2.226, 0.512, 0.513], + [4.254, 0.956, 0.958], + [0.271, 0.041, 0.04], + [10.018, 0.576, 0.576], + [11.292, 0.641, 0.658], + [22.28, 1.171, 1.157], + [55.698, 2.845, 2.912], + [2.67, 0.186, 0.184], + [0.911, 0.147, 0.142], + [2.904, 0.178, 0.187], + [9.903, 0.705, 0.692], + [8.957, 2.793, 2.791], + [0.135, 0.049, 0.045], + [2.835, 0.298, 0.293], + [6.354, 0.281, 0.29], + [4.519, 0.702, 0.716], + [9.822, 1.203, 1.185], + [9.819, 1.18, 1.167], + [0.422, 0.254, 0.249], + [0.315, 0.125, 0.125], + [0.162, 0.058, 0.058], + [0.267, 0.086, 0.086], + [0.498, 0.232, 0.232], + [0.135, 0.025, 0.026], + [0.113, 0.023, 0.024], + [0.123, 0.024, 0.024] + ] +}