diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql new file mode 100644 index 00000000000..9e6603d46b4 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql @@ -0,0 +1,61 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.0 +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.NextDay.allowIncompatible=false +-- Config: spark.comet.expression.TruncDate.allowIncompatible=false +-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=false +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false +-- Config: spark.comet.expression.MakeTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.ToUnixTimestamp.allowIncompatible=false + +statement +CREATE TABLE routing_datetime_collation(ts TIMESTAMP, d DATE, fmt STRING, day STRING, zone STRING, str_ts STRING, seconds BIGINT) USING parquet + +statement +INSERT INTO routing_datetime_collation VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 'MON', 'UTC', '2024-06-15', 0), (NULL, NULL, NULL, NULL, NULL, NULL, NULL) + +query expect_native(next_day) +SELECT next_day(d, day) FROM routing_datetime_collation + +query expect_fallback(next_day: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT next_day(d, day COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT trunc(d, fmt COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_trunc(fmt COLLATE UTF8_LCASE, ts) FROM routing_datetime_collation + +query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_format(ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT from_unixtime(seconds, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_fallback(convert_timezone: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT convert_timezone(zone COLLATE UTF8_LCASE, 'America/Los_Angeles', CAST(ts AS TIMESTAMP_NTZ)) FROM routing_datetime_collation + +query expect_fallback(make_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT make_timestamp(2024, 6, 15, 12, 0, 0, zone COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_fallback(to_unix_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT to_unix_timestamp(str_ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql new file mode 100644 index 00000000000..ec4be718c31 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql @@ -0,0 +1,61 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.0 +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.NextDay.allowIncompatible=false +-- Config: spark.comet.expression.TruncDate.allowIncompatible=false +-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=false +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false +-- Config: spark.comet.expression.MakeTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.ToUnixTimestamp.allowIncompatible=false + +statement +CREATE TABLE routing_datetime_collation(ts TIMESTAMP, d DATE, fmt STRING, day STRING, zone STRING, str_ts STRING, seconds BIGINT) USING parquet + +statement +INSERT INTO routing_datetime_collation VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 'MON', 'UTC', '2024-06-15', 0), (NULL, NULL, NULL, NULL, NULL, NULL, NULL) + +query expect_native(next_day) +SELECT next_day(d, day) FROM routing_datetime_collation + +query expect_dispatch(next_day) +SELECT next_day(d, day COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_dispatch(trunc) +SELECT trunc(d, fmt COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_dispatch(date_trunc) +SELECT date_trunc(fmt COLLATE UTF8_LCASE, ts) FROM routing_datetime_collation + +query expect_dispatch(date_format) +SELECT date_format(ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_dispatch(from_unixtime) +SELECT from_unixtime(seconds, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_dispatch(convert_timezone) +SELECT convert_timezone(zone COLLATE UTF8_LCASE, 'America/Los_Angeles', CAST(ts AS TIMESTAMP_NTZ)) FROM routing_datetime_collation + +query expect_dispatch(make_timestamp) +SELECT make_timestamp(2024, 6, 15, 12, 0, 0, zone COLLATE UTF8_LCASE) FROM routing_datetime_collation + +query expect_dispatch(to_unix_timestamp) +SELECT to_unix_timestamp(str_ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql new file mode 100644 index 00000000000..003d531179b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql @@ -0,0 +1,66 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false +-- Config: spark.comet.expression.TruncDate.allowIncompatible=false +-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false + +statement +CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet + +statement +INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL) + +query expect_native(trunc) +SELECT trunc(d, 'YEAR') FROM routing_temporal + +query expect_native(date_trunc) +SELECT date_trunc('YEAR', ts) FROM routing_temporal + +query expect_native(date_format) +SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal + +query expect_fallback(from_utc_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_fallback(to_utc_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT from_unixtime(seconds) FROM routing_temporal + +query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal + +query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT trunc(d, fmt) FROM routing_temporal + +query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT trunc(d, 'day') FROM routing_temporal + +query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_trunc(fmt, ts) FROM routing_temporal + +query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_trunc('not_a_unit', ts) FROM routing_temporal + +query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql new file mode 100644 index 00000000000..00e8b7ffa53 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql @@ -0,0 +1,66 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=false +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false +-- Config: spark.comet.expression.TruncDate.allowIncompatible=false +-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false + +statement +CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet + +statement +INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL) + +query expect_native(trunc) +SELECT trunc(d, 'YEAR') FROM routing_temporal + +query expect_native(date_trunc) +SELECT date_trunc('YEAR', ts) FROM routing_temporal + +query expect_native(date_format) +SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal + +query expect_dispatch(from_utc_timestamp) +SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_dispatch(to_utc_timestamp) +SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_dispatch(from_unixtime) +SELECT from_unixtime(seconds) FROM routing_temporal + +query expect_dispatch(from_unixtime) +SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal + +query expect_dispatch(trunc) +SELECT trunc(d, fmt) FROM routing_temporal + +query expect_dispatch(trunc) +SELECT trunc(d, 'day') FROM routing_temporal + +query expect_dispatch(date_trunc) +SELECT date_trunc(fmt, ts) FROM routing_temporal + +query expect_dispatch(date_trunc) +SELECT date_trunc('not_a_unit', ts) FROM routing_temporal + +query expect_dispatch(date_format) +SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql new file mode 100644 index 00000000000..d006b417b09 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql @@ -0,0 +1,67 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=true +-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=true +-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=true +-- Config: spark.comet.expression.TruncDate.allowIncompatible=true +-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=true + +statement +CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet + +statement +INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL) + +query expect_native(trunc) +SELECT trunc(d, 'YEAR') FROM routing_temporal + +query expect_native(date_trunc) +SELECT date_trunc('YEAR', ts) FROM routing_temporal + +query expect_native(date_format) +SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal + +query expect_native(from_utc_timestamp) +SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_native(to_utc_timestamp) +SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal + +query expect_native(from_unixtime) +SELECT from_unixtime(seconds) FROM routing_temporal + +query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal + +-- Use non-null formats for native opt-in; the dispatcher cases retain null-format coverage. +query expect_native(trunc) +SELECT trunc(d, fmt) FROM routing_temporal WHERE fmt IS NOT NULL + +query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT trunc(d, 'day') FROM routing_temporal + +query expect_native(date_trunc) +SELECT date_trunc(fmt, ts) FROM routing_temporal WHERE fmt IS NOT NULL + +query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_trunc('not_a_unit', ts) FROM routing_temporal + +query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql new file mode 100644 index 00000000000..caa4d87ce63 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 3.5 +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false + +statement +CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet + +statement +INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL) + +query expect_fallback(convert_timezone: spark.comet.exec.scalaUDF.codegen.enabled=false) +SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql new file mode 100644 index 00000000000..ed0ed79f21b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 3.5 +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false + +statement +CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet + +statement +INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL) + +query expect_dispatch(convert_timezone) +SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql new file mode 100644 index 00000000000..f61c34ddcb5 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 3.5 +-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false +-- Config: spark.sql.session.timeZone=UTC +-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=true + +statement +CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet + +statement +INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL) + +query expect_native(convert_timezone) +SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone diff --git a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala index fce36496aa4..9b2b73f97e3 100644 --- a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala @@ -176,8 +176,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { for (format <- CometTruncTimestamp.supportedFormats) { - checkSparkAnswerAndOperator( - s"SELECT c0, date_trunc('$format', c0) from tbl order by c0") + checkSparkAnswerAndImpl( + s"SELECT c0, date_trunc('$format', c0) from tbl order by c0", + native = Seq.empty, + dispatched = Seq("date_trunc")) } } } @@ -213,10 +215,13 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH for (tz <- nonUtcTimezones) { withSQLConf( SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, - "spark.comet.expression.TruncTimestamp.allowIncompatible" -> "true") { + "spark.comet.expression.TruncTimestamp.allowIncompatible" -> "true", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") { for (format <- CometTruncTimestamp.supportedFormats) { - checkSparkAnswerAndOperator( - s"SELECT c0, date_trunc('$format', c0) from tbl order by c0") + checkSparkAnswerAndImpl( + s"SELECT c0, date_trunc('$format', c0) from tbl order by c0", + native = Seq("date_trunc"), + dispatched = Seq.empty) } } } @@ -602,8 +607,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH withSQLConf( SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC", CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { - checkSparkAnswerAndOperator( - "SELECT c0, date_format(c0, 'yyyy-MM-dd EEEE') from tbl order by c0") + checkSparkAnswerAndImpl( + "SELECT c0, date_format(c0, 'yyyy-MM-dd EEEE') from tbl order by c0", + native = Seq.empty, + dispatched = Seq("date_format")) } } @@ -629,8 +636,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH withSQLConf( SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { - checkSparkAnswerAndOperator( - "SELECT c0, date_format(c0, 'yyyy-MM-dd HH:mm:ss') from tbl order by c0") + checkSparkAnswerAndImpl( + "SELECT c0, date_format(c0, 'yyyy-MM-dd HH:mm:ss') from tbl order by c0", + native = Seq.empty, + dispatched = Seq("date_format")) } } } @@ -659,13 +668,15 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH for (tz <- nonUtcTimezones) { withSQLConf( SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, - "spark.comet.expression.DateFormatClass.allowIncompatible" -> "true") { + "spark.comet.expression.DateFormatClass.allowIncompatible" -> "true", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") { // Native to_char results may diverge from Spark for non-UTC timezones (the reason the // JVM UDF is the default), so we only check that execution stays inside Comet. ORDER BY // is omitted to keep the plan free of AQEShuffleRead. val df = sql("SELECT c0, date_format(c0, 'yyyy-MM-dd') from tbl") df.collect() checkCometOperators(stripAQEPlan(df.queryExecution.executedPlan)) + assertExpressionImpl(df.queryExecution.executedPlan, Seq("date_format"), Seq.empty) } } }