diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql deleted file mode 100755 index 0e8c60d4de5628..00000000000000 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run25.hql +++ /dev/null @@ -1,118 +0,0 @@ -SET hive.support.concurrency=true; -SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; - -drop table if exists orc_full_acid_empty; - -create table orc_full_acid_empty (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -drop table if exists orc_full_acid_par_empty; - -create table orc_full_acid_par_empty (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -drop table if exists orc_full_acid; - -create table orc_full_acid (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -insert into orc_full_acid values -(1, 'A'), -(2, 'B'), -(3, 'C'); - -update orc_full_acid set value = 'CC' where id = 3; - -drop table if exists orc_full_acid_par; - -create table orc_full_acid_par (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); - -insert into orc_full_acid_par PARTITION(part_col=20230101) values -(1, 'A'), -(2, 'B'), -(3, 'C'); - -insert into orc_full_acid_par PARTITION(part_col=20230102) values -(4, 'D'), -(5, 'E'), -(6, 'F'); - -update orc_full_acid_par set value = 'BB' where id = 2; - - - - -drop table if exists orc_to_acid_tb; - - - - -create table orc_to_acid_tb (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC; -INSERT INTO TABLE orc_to_acid_tb PARTITION (part_col=101) VALUES (1, 'A'), (3, 'C'); -INSERT INTO TABLE orc_to_acid_tb PARTITION (part_col=102) VALUES (2, 'B'); -ALTER TABLE orc_to_acid_tb SET TBLPROPERTIES ('transactional'='true'); - - -drop table if exists orc_to_acid_compacted_tb; - - -create table orc_to_acid_compacted_tb (id INT, value STRING) -PARTITIONED BY (part_col INT) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC; -INSERT INTO TABLE orc_to_acid_compacted_tb PARTITION (part_col=101) VALUES (1, 'A'), (3, 'C'); -INSERT INTO TABLE orc_to_acid_compacted_tb PARTITION (part_col=102) VALUES (2, 'B'); -ALTER TABLE orc_to_acid_compacted_tb SET TBLPROPERTIES ('transactional'='true'); -ALTER TABLE orc_to_acid_compacted_tb partition(part_col='101') COMPACT 'major' and wait; -ALTER TABLE orc_to_acid_compacted_tb partition(part_col='102') COMPACT 'major' and wait; -INSERT INTO TABLE orc_to_acid_compacted_tb PARTITION (part_col=102) VALUES (4, 'D'); -update orc_to_acid_compacted_tb set value = "CC" where id = 3; -update orc_to_acid_compacted_tb set value = "BB" where id = 2; - - -drop table if exists orc_acid_minor; - - -create table orc_acid_minor (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); -insert into orc_acid_minor values (1, 'A'); -insert into orc_acid_minor values (2, 'B'); -insert into orc_acid_minor values (3, 'C'); -update orc_acid_minor set value = "BB" where id = 2; -ALTER TABLE orc_acid_minor COMPACT 'minor' and wait; -insert into orc_acid_minor values (4, 'D'); -update orc_acid_minor set value = "DD" where id = 4; -DELETE FROM orc_acid_minor WHERE id = 3; - - -drop table if exists orc_acid_major; - - -create table orc_acid_major (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS ORC -TBLPROPERTIES ('transactional' = 'true'); -insert into orc_acid_major values (1, 'A'); -insert into orc_acid_major values (2, 'B'); -insert into orc_acid_major values (3, 'C'); -update orc_acid_major set value = "BB" where id = 2; -ALTER TABLE orc_acid_major COMPACT 'minor' and wait; -insert into orc_acid_major values (4, 'D'); -update orc_acid_major set value = "DD" where id = 4; -DELETE FROM orc_acid_major WHERE id = 3; diff --git a/regression-test/data/external_table_p0/hive/test_hive_orc_predicate.out b/regression-test/data/external_table_p0/hive/test_hive_orc_predicate.out index 7d21967daade2d..9f5c0e0b310da1 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_orc_predicate.out +++ b/regression-test/data/external_table_p0/hive/test_hive_orc_predicate.out @@ -17,8 +17,6 @@ 1 \N 1 \N 3 \N 1 \N --- !predicate_full_acid_push_down -- - -- !lazy_materialization_for_list_type -- ["c", "a"] ["b", "c"] @@ -79,9 +77,6 @@ 1 \N 1 \N 3 \N 1 \N --- !predicate_full_acid_push_down -- -2 BB 20230101 - -- !lazy_materialization_for_list_type -- ["c", "a"] ["b", "c"] @@ -123,4 +118,3 @@ {"struct_field":["1", "2", "3"]} {"struct_field":["1", "2", "3"]} {"struct_field":["1", "2", "3"]} - diff --git a/regression-test/data/external_table_p0/hive/test_transactional_hive.out b/regression-test/data/external_table_p0/hive/test_transactional_hive.out deleted file mode 100644 index d17ad0b7cd5124..00000000000000 --- a/regression-test/data/external_table_p0/hive/test_transactional_hive.out +++ /dev/null @@ -1,163 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !q01 -- -1 A -2 B -3 CC - --- !q02 -- -A -B -CC - --- !q04 -- - --- !q05 -- -0 - --- !q01 -- -1 A 20230101 -2 BB 20230101 -3 C 20230101 -4 D 20230102 -5 E 20230102 -6 F 20230102 - --- !q02 -- -A -BB -C -D -E -F - --- !q03 -- -2 BB 20230101 - --- !q04 -- - --- !q05 -- -0 - --- !q01 -- -1 A -2 B -3 CC - --- !q02 -- -A -B -CC - --- !q04 -- - --- !q05 -- -0 - --- !q01 -- -1 A 20230101 -2 BB 20230101 -3 C 20230101 -4 D 20230102 -5 E 20230102 -6 F 20230102 - --- !q02 -- -A -BB -C -D -E -F - --- !q03 -- -2 BB 20230101 - --- !q04 -- - --- !q05 -- -0 - --- !2 -- -1 A 101 -2 BB 102 -3 CC 101 -4 D 102 - --- !3 -- -1 A 101 -3 CC 101 - --- !4 -- -2 BB 102 -4 D 102 - --- !5 -- -1 A 101 -2 BB 102 - --- !6 -- -4 D 102 - --- !7 -- -1 A -2 BB -4 DD - --- !10 -- -1 A -2 BB - --- !11 -- -4 DD - --- !12 -- -1 A -2 BB -4 DD - --- !15 -- -1 A -2 BB - --- !16 -- -4 DD - --- !17 -- -1 A - --- !18 -- -1 A - --- !19 -- -4 DD - --- !count_1 -- -3 - --- !count_2 -- -6 - --- !count_3 -- -4 - --- !count_4 -- -3 - --- !count_5 -- -3 - --- !q01_limit -- -1 A 20230101 -2 BB 20230101 -3 C 20230101 - --- !q02_limit -- -A -BB -C - --- !q03_limit -- -2 BB 20230101 - --- !q04_limit -- - diff --git a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out b/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out deleted file mode 100644 index f43a630f4a387c..00000000000000 --- a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out +++ /dev/null @@ -1,30 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -1 A -2 B -3 C -4 D - --- !2 -- -1 A -2 B -3 C -4 D -5 E - --- !3 -- -1 A -2 B -3 C -4 D -5 E - --- !count_1 -- -4 - --- !count_2 -- -5 - --- !count_3 -- -5 - diff --git a/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy b/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy index 0bd8fcd09f2973..836b72e21195f1 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_orc_predicate.groovy @@ -24,44 +24,31 @@ suite("test_hive_orc_predicate", "p0,external") { } for (String hivePrefix : ["hive2", "hive3"]) { - try { - String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") - String catalog_name = "${hivePrefix}_test_predicate" - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_predicate" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - sql """drop catalog if exists ${catalog_name}""" - sql """create catalog if not exists ${catalog_name} properties ( - "type"="hms", - 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' - );""" - sql """use `${catalog_name}`.`multi_catalog`""" + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """use `${catalog_name}`.`multi_catalog`""" - qt_predicate_fixed_char1 """ select * from fixed_char_table where c = 'a';""" - qt_predicate_fixed_char2 """ select * from fixed_char_table where c = 'a ';""" + qt_predicate_fixed_char1 """ select * from fixed_char_table where c = 'a';""" + qt_predicate_fixed_char2 """ select * from fixed_char_table where c = 'a ';""" - qt_predicate_changed_type1 """ select * from type_changed_table where id = '1';""" - qt_predicate_changed_type2 """ select * from type_changed_table where id = '2';""" - qt_predicate_changed_type3 """ select * from type_changed_table where id = '3';""" + qt_predicate_changed_type1 """ select * from type_changed_table where id = '1';""" + qt_predicate_changed_type2 """ select * from type_changed_table where id = '2';""" + qt_predicate_changed_type3 """ select * from type_changed_table where id = '3';""" - qt_predicate_null_aware_equal_in_rt """select * from table_a inner join table_b on table_a.age <=> table_b.age and table_b.id in (1,3) order by table_a.id;""" + qt_predicate_null_aware_equal_in_rt """select * from table_a inner join table_b on table_a.age <=> table_b.age and table_b.id in (1,3) order by table_a.id;""" - // use check_orc_init_sargs_success to test full acid push down - sql """use `${catalog_name}`.`default`""" - if (hivePrefix == "hive3") { - sql """ set check_orc_init_sargs_success = true; """ - } - qt_predicate_full_acid_push_down """ select * from orc_full_acid_par where value = 'BB' order by id;""" - sql """ set check_orc_init_sargs_success = false; """ + qt_lazy_materialization_for_list_type """ select l from complex_data_orc where id > 2 order by id; """ + qt_lazy_materialization_for_map_type """ select m from complex_data_orc where id > 2 order by id; """ + qt_lazy_materialization_for_list_and_map_type """ select * from complex_data_orc where id > 2 order by id; """ + qt_lazy_materialization_for_list_type2 """select t_struct_nested from `${catalog_name}`.`default`.orc_all_types_t where t_int=3;""" - sql """use `${catalog_name}`.`multi_catalog`""" - qt_lazy_materialization_for_list_type """ select l from complex_data_orc where id > 2 order by id; """ - qt_lazy_materialization_for_map_type """ select m from complex_data_orc where id > 2 order by id; """ - qt_lazy_materialization_for_list_and_map_type """ select * from complex_data_orc where id > 2 order by id; """ - qt_lazy_materialization_for_list_type2 """select t_struct_nested from `${catalog_name}`.`default`.orc_all_types_t where t_int=3;""" - - sql """drop catalog if exists ${catalog_name}""" - } finally { - sql """ set check_orc_init_sargs_success = false; """ - } + sql """drop catalog if exists ${catalog_name}""" } } diff --git a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy deleted file mode 100644 index c11d9165063ae9..00000000000000 --- a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy +++ /dev/null @@ -1,195 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_transactional_hive", "p0,external") { - String skip_checking_acid_version_file = "false" - - def q01 = { - sql """set skip_checking_acid_version_file=${skip_checking_acid_version_file}""" - qt_q01 """ - select * from orc_full_acid order by id; - """ - qt_q02 """ - select value from orc_full_acid order by id; - """ - qt_q04 """ - select * from orc_full_acid_empty; - """ - qt_q05 """ - select count(*) from orc_full_acid_empty; - """ - } - - def q01_par = { - sql """set skip_checking_acid_version_file=${skip_checking_acid_version_file}""" - qt_q01 """ - select * from orc_full_acid_par order by id; - """ - qt_q02 """ - select value from orc_full_acid_par order by id; - """ - qt_q03 """ - select * from orc_full_acid_par where value = 'BB' order by id; - """ - qt_q04 """ - select * from orc_full_acid_par_empty; - """ - qt_q05 """ - select count(*) from orc_full_acid_par_empty; - """ - } - - def q01_par_limit = { - qt_q01_limit """ - select * from orc_full_acid_par order by id limit 3; - """ - qt_q02_limit """ - select value from orc_full_acid_par order by id limit 3; - """ - qt_q03_limit """ - select * from orc_full_acid_par where value = 'BB' order by id limit 3; - """ - qt_q04_limit """ - select * from orc_full_acid_par_empty limit 3; - """ - } - - def test_acid = { - - sql """set enable_fallback_to_original_planner=false;""" - try { - sql """ select * from orc_to_acid_tb """ - }catch( Exception e) { - assertTrue(e.getMessage().contains("For no acid table convert to acid, please COMPACT")); - } - - qt_2 """ select * from orc_to_acid_compacted_tb order by id """ - qt_3 """ select * from orc_to_acid_compacted_tb where part_col=101 order by id """ - qt_4 """ select * from orc_to_acid_compacted_tb where part_col=102 order by id """ - qt_5 """ select * from orc_to_acid_compacted_tb where id < 3 order by id """ - qt_6 """ select * from orc_to_acid_compacted_tb where id > 3 order by id """ - - - qt_7 """ select * from orc_acid_minor order by id """ - qt_10 """ select * from orc_acid_minor where id < 3 order by id """ - qt_11 """ select * from orc_acid_minor where id > 3 order by id """ - - - qt_12 """ select * from orc_acid_major order by id """ - qt_15 """ select * from orc_acid_major where id < 3 order by id """ - qt_16 """ select * from orc_acid_major where id > 3 order by id """ - - qt_17 """ select * from orc_acid_major order by id limit 1 """ - qt_18 """ select * from orc_acid_major where id < 3 order by id limit 1 """ - qt_19 """ select * from orc_acid_major where id > 3 order by id limit 1""" - } - - def test_acid_write = { - sql """set enable_fallback_to_original_planner=false;""" - - - - try { - sql """ - CREATE TABLE acid_tb ( - `col1` BOOLEAN COMMENT 'col1', - `col2` INT COMMENT 'col2' - ) ENGINE=hive - PROPERTIES ( - 'file_format'='orc', - 'compression'='zlib', - 'bucketing_version'='2', - 'transactional'='true', - 'transactional_properties'='default' - ); - """ - }catch( Exception e) { - assertTrue(e.getMessage().contains("Not support create hive transactional table.")); - } - try { - sql """ insert into orc_acid_major(id,value) values(1,"a1"); """ - }catch (Exception e) { - assertTrue(e.getMessage().contains("Not supported insert into hive transactional table.")); - } - - try { - sql """ drop table orc_acid_major; """ - } catch (Exception e) { - assertTrue(e.getMessage().contains("Not support drop hive transactional table.")); - - } - } - - def test_acid_count = { - qt_count_1 """ select count(*) from orc_full_acid; """ // 3 - qt_count_2 """ select count(*) from orc_full_acid_par; """ // 6 - qt_count_3 """ select count(*) from orc_to_acid_compacted_tb; """ //4 - qt_count_4 """ select count(*) from orc_acid_minor; """ //3 - qt_count_5 """ select count(*) from orc_acid_major; """ //3 - } - - def test_explain_verbose = { - explain { - sql ("select count(*) from orc_full_acid") - verbose (true) - contains "deleteFileNum" - } - } - - String enabled = context.config.otherConfigs.get("enableHiveTest") - if (enabled == null || !enabled.equalsIgnoreCase("true")) { - logger.info("diable Hive test.") - return; - } - - for (String hivePrefix : ["hive3"]) { - try { - String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") - String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") - String catalog_name = "test_transactional_${hivePrefix}" - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - - sql """drop catalog if exists ${catalog_name}""" - sql """create catalog if not exists ${catalog_name} properties ( - "type"="hms", - 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' - ,'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}' - );""" - sql """use `${catalog_name}`.`default`""" - - skip_checking_acid_version_file = "false" - q01() - q01_par() - - skip_checking_acid_version_file = "true" - q01() - q01_par() - - test_acid() - test_acid_write() - - - test_acid_count() - test_explain_verbose() - - q01_par_limit() - - sql """drop catalog if exists ${catalog_name}""" - } finally { - } - } -} diff --git a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy b/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy deleted file mode 100644 index 2e385703dc2580..00000000000000 --- a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_hive_translation_insert_only", "p2,external") { - - String enabled = context.config.otherConfigs.get("enableExternalEmrTest") - //hudi hive use same catalog in p2. - if (enabled == null || !enabled.equalsIgnoreCase("true")) { - logger.info("disable test") - return; - } - - String props = context.config.otherConfigs.get("emrCatalogCommonProp") - String hms_catalog_name = "test_hive_translation_insert_only" - - sql """drop catalog if exists ${hms_catalog_name};""" - sql """ - CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} - PROPERTIES ( - "type" = "hms", - 'hive.version' = '3.1.3', - ${props} - ); - """ - - logger.info("catalog " + hms_catalog_name + " created") - sql """switch ${hms_catalog_name};""" - logger.info("switched to catalog " + hms_catalog_name) - sql """ use regression;""" - - qt_1 """ select * from text_insert_only order by id """ - qt_2 """ select * from parquet_insert_only_major order by id """ - qt_3 """ select * from orc_insert_only_minor order by id """ - - qt_count_1 """ select count(*) from text_insert_only """ //4 - qt_count_2 """ select count(*) from parquet_insert_only_major """ //5 - qt_count_3 """ select count(*) from orc_insert_only_minor """ //5 - - - sql """drop catalog ${hms_catalog_name};""" -} - - -/* -SET hive.support.concurrency=true; -SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; - - -create table text_insert_only (id INT, value STRING) -ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' -TBLPROPERTIES ('transactional' = 'true', -'transactional_properties'='insert_only'); -insert into text_insert_only values (1, 'A'); -insert into text_insert_only values (2, 'B'); -insert into text_insert_only values (3, 'C'); -Load data xxx (4,D) - - -create table parquet_insert_only_major (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -STORED AS parquet -TBLPROPERTIES ('transactional' = 'true', -'transactional_properties'='insert_only'); -insert into parquet_insert_only_major values (1, 'A'); -insert into parquet_insert_only_major values (2, 'B'); -insert into parquet_insert_only_major values (3, 'C'); -ALTER TABLE parquet_insert_only_major COMPACT 'major'; -insert into parquet_insert_only_major values (4, 'D'); -insert into parquet_insert_only_major values (5, 'E'); - - -create table orc_insert_only_minor (id INT, value STRING) -CLUSTERED BY (id) INTO 3 BUCKETS -stored as orc -TBLPROPERTIES ('transactional' = 'true', -'transactional_properties'='insert_only'); -insert into orc_insert_only_minor values (1, 'A'); -insert into orc_insert_only_minor values (2, 'B'); -insert into orc_insert_only_minor values (3, 'C'); -ALTER TABLE orc_insert_only_minor COMPACT 'minor'; -insert into orc_insert_only_minor values (4, 'D'); -insert into orc_insert_only_minor values (5, 'E'); - -*/