diff --git a/be/src/olap/collection_statistics.cpp b/be/src/olap/collection_statistics.cpp index a30a91722ad7d9..1292ede22dfcf8 100644 --- a/be/src/olap/collection_statistics.cpp +++ b/be/src/olap/collection_statistics.cpp @@ -91,21 +91,25 @@ Status CollectionStatistics::extract_collect_info( static_cast(expr->children()[1].get()); auto column_idx = tablet_schema->field_index(left_slot_ref->column_name()); auto column = tablet_schema->column(column_idx); - const auto* index_meta = tablet_schema->inverted_index(column); - - auto term_infos = InvertedIndexAnalyzer::get_analyse_result( - right_slot_ref->value(), index_meta->properties()); - - std::string field_name = std::to_string(column.unique_id()); - std::wstring ws_field_name = StringHelper::to_wstring(field_name); - auto iter = collect_infos->find(ws_field_name); - if (iter == collect_infos->end()) { - CollectInfo collect_info; - collect_info.term_infos.insert(term_infos.begin(), term_infos.end()); - collect_info.index_meta = index_meta; - (*collect_infos)[ws_field_name] = std::move(collect_info); - } else { - iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); + auto index_metas = tablet_schema->inverted_indexs(column); + for (const auto* index_meta : index_metas) { + if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) { + continue; + } + auto term_infos = InvertedIndexAnalyzer::get_analyse_result( + right_slot_ref->value(), index_meta->properties()); + + std::string field_name = std::to_string(column.unique_id()); + std::wstring ws_field_name = StringHelper::to_wstring(field_name); + auto iter = collect_infos->find(ws_field_name); + if (iter == collect_infos->end()) { + CollectInfo collect_info; + collect_info.term_infos.insert(term_infos.begin(), term_infos.end()); + collect_info.index_meta = index_meta; + (*collect_infos)[ws_field_name] = std::move(collect_info); + } else { + iter->second.term_infos.insert(term_infos.begin(), term_infos.end()); + } } } diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index d927d02cfdacb8..3eaf68a2492bb6 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -771,10 +771,10 @@ Status Compaction::do_inverted_index_compaction() { Status status = Status::OK(); for (auto&& column_uniq_id : ctx.columns_to_do_index_compaction) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); - const auto* index_meta = _cur_tablet_schema->inverted_index(col); + auto index_metas = _cur_tablet_schema->inverted_indexs(col); DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_can_not_find_index_meta", - { index_meta = nullptr; }) - if (index_meta == nullptr) { + { index_metas.clear(); }) + if (index_metas.empty()) { status = Status::Error( fmt::format("Can not find index_meta for col {}", col.name())); LOG(WARNING) << "failed to do index compaction, can not find index_meta for column" @@ -783,57 +783,61 @@ Status Compaction::do_inverted_index_compaction() { error_handler(-1, column_uniq_id); break; } - - std::vector dest_index_dirs(dest_segment_num); - try { - std::vector> src_idx_dirs( - src_segment_num); - for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) { - auto res = index_file_readers[src_segment_id]->open(index_meta); - DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_reader", { - res = ResultError(Status::Error( - "debug point: Compaction::open_index_file_reader error")); - }) - if (!res.has_value()) { - LOG(WARNING) << "failed to do index compaction, open inverted index file " - "reader failed" - << ". tablet=" << _tablet->tablet_id() - << ", column uniq id=" << column_uniq_id - << ", src_segment_id=" << src_segment_id; - throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg()); + for (const auto& index_meta : index_metas) { + std::vector dest_index_dirs(dest_segment_num); + try { + std::vector> src_idx_dirs( + src_segment_num); + for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) { + auto res = index_file_readers[src_segment_id]->open(index_meta); + DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_reader", { + res = ResultError(Status::Error( + "debug point: Compaction::open_index_file_reader error")); + }) + if (!res.has_value()) { + LOG(WARNING) << "failed to do index compaction, open inverted index file " + "reader failed" + << ". tablet=" << _tablet->tablet_id() + << ", column uniq id=" << column_uniq_id + << ", src_segment_id=" << src_segment_id; + throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, + res.error().msg()); + } + src_idx_dirs[src_segment_id] = std::move(res.value()); } - src_idx_dirs[src_segment_id] = std::move(res.value()); - } - for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; dest_segment_id++) { - auto res = inverted_index_file_writers[dest_segment_id]->open(index_meta); - DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_writer", { - res = ResultError(Status::Error( - "debug point: Compaction::open_inverted_index_file_writer error")); - }) - if (!res.has_value()) { - LOG(WARNING) << "failed to do index compaction, open inverted index file " - "writer failed" - << ". tablet=" << _tablet->tablet_id() - << ", column uniq id=" << column_uniq_id - << ", dest_segment_id=" << dest_segment_id; - throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, res.error().msg()); + for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; + dest_segment_id++) { + auto res = inverted_index_file_writers[dest_segment_id]->open(index_meta); + DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_writer", { + res = ResultError(Status::Error( + "debug point: Compaction::open_inverted_index_file_writer error")); + }) + if (!res.has_value()) { + LOG(WARNING) << "failed to do index compaction, open inverted index file " + "writer failed" + << ". tablet=" << _tablet->tablet_id() + << ", column uniq id=" << column_uniq_id + << ", dest_segment_id=" << dest_segment_id; + throw Exception(ErrorCode::INVERTED_INDEX_COMPACTION_ERROR, + res.error().msg()); + } + // Destination directories in dest_index_dirs do not need to be deconstructed, + // but their lifecycle must be managed by inverted_index_file_writers. + dest_index_dirs[dest_segment_id] = res.value().get(); } - // Destination directories in dest_index_dirs do not need to be deconstructed, - // but their lifecycle must be managed by inverted_index_file_writers. - dest_index_dirs[dest_segment_id] = res.value().get(); - } - auto st = compact_column(index_meta->index_id(), src_idx_dirs, dest_index_dirs, - index_tmp_path.native(), trans_vec, dest_segment_num_rows); - if (!st.ok()) { + auto st = compact_column(index_meta->index_id(), src_idx_dirs, dest_index_dirs, + index_tmp_path.native(), trans_vec, dest_segment_num_rows); + if (!st.ok()) { + error_handler(index_meta->index_id(), column_uniq_id); + status = Status::Error(st.msg()); + } + } catch (CLuceneError& e) { + error_handler(index_meta->index_id(), column_uniq_id); + status = Status::Error(e.what()); + } catch (const Exception& e) { error_handler(index_meta->index_id(), column_uniq_id); - status = Status::Error(st.msg()); + status = Status::Error(e.what()); } - } catch (CLuceneError& e) { - error_handler(index_meta->index_id(), column_uniq_id); - status = Status::Error(e.what()); - } catch (const Exception& e) { - error_handler(index_meta->index_id(), column_uniq_id); - status = Status::Error(e.what()); } } @@ -853,17 +857,19 @@ void Compaction::mark_skip_index_compaction( const std::function& error_handler) { for (auto&& column_uniq_id : context.columns_to_do_index_compaction) { auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); - const auto* index_meta = _cur_tablet_schema->inverted_index(col); + auto index_metas = _cur_tablet_schema->inverted_indexs(col); DBUG_EXECUTE_IF("Compaction::mark_skip_index_compaction_can_not_find_index_meta", - { index_meta = nullptr; }) - if (index_meta == nullptr) { + { index_metas.clear(); }) + if (index_metas.empty()) { LOG(WARNING) << "mark skip index compaction, can not find index_meta for column" << ". tablet=" << _tablet->tablet_id() << ", column uniq id=" << column_uniq_id; error_handler(-1, column_uniq_id); continue; } - error_handler(index_meta->index_id(), column_uniq_id); + for (const auto& index_meta : index_metas) { + error_handler(index_meta->index_id(), column_uniq_id); + } } } @@ -892,24 +898,30 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { bool is_continue = false; std::optional> first_properties; for (const auto& rowset : _input_rowsets) { - const auto* tablet_index = rowset->tablet_schema()->inverted_index(col_unique_id); + auto tablet_indexs = rowset->tablet_schema()->inverted_indexs(col_unique_id); // no inverted index or index id is different from current index id - if (tablet_index == nullptr || tablet_index->index_id() != index->index_id()) { + auto it = std::find_if(tablet_indexs.begin(), tablet_indexs.end(), + [&index](const auto& tablet_index) { + return tablet_index->index_id() == index->index_id(); + }); + if (it != tablet_indexs.end()) { + const auto* tablet_index = *it; + auto properties = tablet_index->properties(); + if (!first_properties.has_value()) { + first_properties = properties; + } else { + DBUG_EXECUTE_IF( + "Compaction::do_inverted_index_compaction_index_properties_different", + { properties.emplace("dummy_key", "dummy_value"); }) + if (properties != first_properties.value()) { + is_continue = true; + break; + } + } + } else { is_continue = true; break; } - auto properties = tablet_index->properties(); - if (!first_properties.has_value()) { - first_properties = properties; - } else { - DBUG_EXECUTE_IF( - "Compaction::do_inverted_index_compaction_index_properties_different", - { properties.emplace("dummy_key", "dummy_value"); }) - if (properties != first_properties.value()) { - is_continue = true; - break; - } - } } if (is_continue) { continue; @@ -934,91 +946,95 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) { return false; } - const auto* index_meta = rowset->tablet_schema()->inverted_index(col_unique_id); + auto index_metas = rowset->tablet_schema()->inverted_indexs(col_unique_id); DBUG_EXECUTE_IF("Compaction::construct_skip_inverted_index_index_meta_nullptr", - { index_meta = nullptr; }) - if (index_meta == nullptr) { + { index_metas.clear(); }) + if (index_metas.empty()) { LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" << col_unique_id << "] index meta is null, will skip index compaction"; return false; } - - for (auto i = 0; i < rowset->num_segments(); i++) { - // TODO: inverted_index_path - auto seg_path = rowset->segment_path(i); - DBUG_EXECUTE_IF("Compaction::construct_skip_inverted_index_seg_path_nullptr", { - seg_path = ResultError(Status::Error( - "construct_skip_inverted_index_seg_path_nullptr")); - }) - if (!seg_path) { - LOG(WARNING) << seg_path.error(); - return false; - } - - std::string index_file_path; - try { - auto index_file_reader = std::make_unique( - fs, - std::string {InvertedIndexDescriptor::get_index_file_path_prefix( - seg_path.value())}, - _cur_tablet_schema->get_inverted_index_storage_format(), - rowset->rowset_meta()->inverted_index_file_info(i)); - auto st = index_file_reader->init(config::inverted_index_read_buffer_size); - index_file_path = index_file_reader->get_index_file_path(index_meta); - DBUG_EXECUTE_IF( - "Compaction::construct_skip_inverted_index_index_file_reader_init_" - "status_not_ok", - { - st = Status::Error( - "debug point: " - "construct_skip_inverted_index_index_file_reader_init_" - "status_" - "not_ok"); - }) - if (!st.ok()) { - LOG(WARNING) << "init index " << index_file_path << " error:" << st; + for (const auto& index_meta : index_metas) { + for (auto i = 0; i < rowset->num_segments(); i++) { + // TODO: inverted_index_path + auto seg_path = rowset->segment_path(i); + DBUG_EXECUTE_IF("Compaction::construct_skip_inverted_index_seg_path_nullptr", { + seg_path = ResultError(Status::Error( + "construct_skip_inverted_index_seg_path_nullptr")); + }) + if (!seg_path) { + LOG(WARNING) << seg_path.error(); return false; } - // check index meta - auto result = index_file_reader->open(index_meta); - DBUG_EXECUTE_IF( - "Compaction::construct_skip_inverted_index_index_file_reader_open_" - "error", - { - result = ResultError( - Status::Error( - "CLuceneError occur when open idx file")); - }) - if (!result.has_value()) { - LOG(WARNING) - << "open index " << index_file_path << " error:" << result.error(); - return false; - } - auto reader = std::move(result.value()); - std::vector files; - reader->list(&files); - reader->close(); - DBUG_EXECUTE_IF( - "Compaction::construct_skip_inverted_index_index_reader_close_error", - { _CLTHROWA(CL_ERR_IO, "debug point: reader close error"); }) - - DBUG_EXECUTE_IF("Compaction::construct_skip_inverted_index_index_files_count", - { files.clear(); }) - - // why is 3? - // slice type index file at least has 3 files: null_bitmap, segments_N, segments.gen - if (files.size() < 3) { + std::string index_file_path; + try { + auto index_file_reader = std::make_unique( + fs, + std::string {InvertedIndexDescriptor::get_index_file_path_prefix( + seg_path.value())}, + _cur_tablet_schema->get_inverted_index_storage_format(), + rowset->rowset_meta()->inverted_index_file_info(i)); + auto st = index_file_reader->init(config::inverted_index_read_buffer_size); + index_file_path = index_file_reader->get_index_file_path(index_meta); + DBUG_EXECUTE_IF( + "Compaction::construct_skip_inverted_index_index_file_reader_init_" + "status_not_ok", + { + st = Status::Error( + "debug point: " + "construct_skip_inverted_index_index_file_reader_init_" + "status_" + "not_ok"); + }) + if (!st.ok()) { + LOG(WARNING) << "init index " << index_file_path << " error:" << st; + return false; + } + + // check index meta + auto result = index_file_reader->open(index_meta); + DBUG_EXECUTE_IF( + "Compaction::construct_skip_inverted_index_index_file_reader_open_" + "error", + { + result = ResultError( + Status::Error( + "CLuceneError occur when open idx file")); + }) + if (!result.has_value()) { + LOG(WARNING) << "open index " << index_file_path + << " error:" << result.error(); + return false; + } + auto reader = std::move(result.value()); + std::vector files; + reader->list(&files); + reader->close(); + DBUG_EXECUTE_IF( + "Compaction::construct_skip_inverted_index_index_reader_close_" + "error", + { _CLTHROWA(CL_ERR_IO, "debug point: reader close error"); }) + + DBUG_EXECUTE_IF( + "Compaction::construct_skip_inverted_index_index_files_count", + { files.clear(); }) + + // why is 3? + // slice type index file at least has 3 files: null_bitmap, segments_N, segments.gen + if (files.size() < 3) { + LOG(WARNING) + << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" + << col_unique_id << "]," << index_file_path + << " is corrupted, will skip index compaction"; + return false; + } + } catch (CLuceneError& err) { LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" - << col_unique_id << "]," << index_file_path - << " is corrupted, will skip index compaction"; + << col_unique_id << "] open index[" << index_file_path + << "], will skip index compaction, error:" << err.what(); return false; } - } catch (CLuceneError& err) { - LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" - << col_unique_id << "] open index[" << index_file_path - << "], will skip index compaction, error:" << err.what(); - return false; } } return true; diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h index f93c8844432807..6501d4aa13d409 100644 --- a/be/src/olap/comparison_predicate.h +++ b/be/src/olap/comparison_predicate.h @@ -71,9 +71,17 @@ class ComparisonPredicateBase : public ColumnPredicate { IndexIterator* iterator, uint32_t num_rows, roaring::Roaring* bitmap) const override { if (iterator == nullptr) { - return Status::OK(); + return Status::Error( + "Inverted index evaluate skipped, no inverted index reader can not support " + "comparison predicate"); + } + + if (iterator->get_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE) == nullptr && + iterator->get_reader(segment_v2::InvertedIndexReaderType::BKD) == nullptr) { + return Status::Error( + "Inverted index evaluate skipped, no inverted index reader can not support " + "comparison predicate"); } - std::string column_name = name_with_type.first; InvertedIndexQueryType query_type = InvertedIndexQueryType::UNKNOWN_QUERY; switch (PT) { @@ -104,7 +112,8 @@ class ComparisonPredicateBase : public ColumnPredicate { InvertedIndexQueryParamFactory::create_query_value(&_value, query_param)); InvertedIndexParam param; - param.column_name = column_name; + param.column_name = name_with_type.first; + param.column_type = name_with_type.second; param.query_value = query_param->get_value(); param.query_type = query_type; param.num_rows = num_rows; diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 290b2ccaf3fc76..b96776c19b3851 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -253,9 +253,9 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const PNodeInfo& node_info) auto cur_rowset = _rowset_builder->rowset(); auto tablet_schema = cur_rowset->rowset_meta()->tablet_schema(); if (!tablet_schema->skip_write_index_on_load()) { - for (auto& column : tablet_schema->columns()) { - const TabletIndex* index_meta = tablet_schema->inverted_index(*column); - if (index_meta) { + for (const auto& column : tablet_schema->columns()) { + auto index_metas = tablet_schema->inverted_indexs(*column); + for (const auto* index_meta : index_metas) { indices_ids.emplace_back(index_meta->index_id(), index_meta->get_index_suffix()); } } diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index d18d2a71fceffa..78160438fbfc7b 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -184,9 +184,18 @@ class InListPredicateBase : public ColumnPredicate { IndexIterator* iterator, uint32_t num_rows, roaring::Roaring* result) const override { if (iterator == nullptr) { - return Status::OK(); + return Status::Error( + "Inverted index evaluate skipped, no inverted index reader can not support " + "in_list"); + } + // only string type and bkd inverted index reader can be used for in + if (iterator->get_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE) == nullptr && + iterator->get_reader(segment_v2::InvertedIndexReaderType::BKD) == nullptr) { + //NOT support in list when parser is FULLTEXT for expr inverted index evaluate. + return Status::Error( + "Inverted index evaluate skipped, no inverted index reader can not support " + "in_list"); } - std::string column_name = name_with_type.first; roaring::Roaring indices; HybridSetBase::IteratorBase* iter = _values->begin(); while (iter->has_next()) { @@ -194,11 +203,12 @@ class InListPredicateBase : public ColumnPredicate { // auto&& value = PrimitiveTypeConvertor::to_storage_field_type( // *reinterpret_cast(ptr)); std::unique_ptr query_param = nullptr; - RETURN_IF_ERROR( - InvertedIndexQueryParamFactory::create_query_value(ptr, query_param)); + RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value((const T*)ptr, + query_param)); InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY; InvertedIndexParam param; - param.column_name = column_name; + param.column_name = name_with_type.first; + param.column_type = name_with_type.second; param.query_value = query_param->get_value(); param.query_type = query_type; param.num_rows = num_rows; diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 125105bca32f04..d0e3aec380da64 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -134,8 +134,8 @@ void BetaRowset::clear_inverted_index_cache() { auto index_path_prefix = InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path); for (const auto& column : tablet_schema()->columns()) { - const TabletIndex* index_meta = tablet_schema()->inverted_index(*column); - if (index_meta) { + auto index_metas = tablet_schema()->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { auto inverted_index_file_cache_key = InvertedIndexDescriptor::get_index_file_cache_key( index_path_prefix, index_meta->index_id(), @@ -240,9 +240,9 @@ Status BetaRowset::remove() { } if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (auto& column : _schema->columns()) { - const TabletIndex* index_meta = _schema->inverted_index(*column); - if (index_meta) { + for (const auto& column : _schema->columns()) { + auto index_metas = _schema->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), @@ -414,10 +414,10 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row auto src_path = local_segment_path(_tablet_path, rowset_id().to_string(), i); RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(src_path, dst_path)); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (auto& column : _schema->columns()) { + for (const auto& column : _schema->columns()) { // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->inverted_index(*column); - if (index_meta) { + auto index_metas = _schema->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { std::string inverted_index_src_file_path = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(src_path), @@ -473,10 +473,10 @@ Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new dest_paths.emplace_back(remote_seg_path); local_paths.emplace_back(local_seg_path); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (auto& column : _schema->columns()) { + for (const auto& column : _schema->columns()) { // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->inverted_index(*column); - if (index_meta) { + auto index_metas = _schema->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { std::string remote_inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix( @@ -682,9 +682,9 @@ Status BetaRowset::calc_file_crc(uint32_t* crc_value, int64_t* file_count) { auto seg_path = DORIS_TRY(segment_path(seg_id)); file_paths.emplace_back(seg_path); if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { - for (auto& column : _schema->columns()) { - const TabletIndex* index_meta = _schema->inverted_index(*column); - if (index_meta) { + for (const auto& column : _schema->columns()) { + auto index_metas = _schema->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_path_v1( InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), @@ -841,26 +841,18 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value, } else { rapidjson::Value indices(rapidjson::kArrayType); for (auto column : _rowset_meta->tablet_schema()->columns()) { - const auto* index_meta = _rowset_meta->tablet_schema()->inverted_index(*column); - if (index_meta == nullptr) { - continue; - } - rapidjson::Value index(rapidjson::kObjectType); - auto index_id = index_meta->index_id(); - auto index_suffix = index_meta->get_index_suffix(); - index.AddMember("index_id", rapidjson::Value(index_id).Move(), allocator); - index.AddMember("index_suffix", rapidjson::Value(index_suffix.c_str(), allocator), - allocator); - auto path = InvertedIndexDescriptor::get_index_file_path_v1(index_file_path_prefix, - index_id, index_suffix); - auto st = add_file_info_to_json(path, index); - if (!st.ok()) { - return st; - } - - auto status = process_files(*index_meta, indices, index); - if (!status.ok()) { - return status; + auto index_metas = _rowset_meta->tablet_schema()->inverted_indexs(*column); + for (const auto& index_meta : index_metas) { + rapidjson::Value index(rapidjson::kObjectType); + auto index_id = index_meta->index_id(); + auto index_suffix = index_meta->get_index_suffix(); + index.AddMember("index_id", rapidjson::Value(index_id).Move(), allocator); + index.AddMember("index_suffix", + rapidjson::Value(index_suffix.c_str(), allocator), allocator); + auto path = InvertedIndexDescriptor::get_index_file_path_v1( + index_file_path_prefix, index_id, index_suffix); + RETURN_IF_ERROR(add_file_info_to_json(path, index)); + RETURN_IF_ERROR(process_files(*index_meta, indices, index)); } } segment.AddMember("indices", indices, allocator); diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 35139fa4ba9666..5a7455df362bbd 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -566,8 +566,8 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u } // rename remaining inverted index files for (auto column : _context.tablet_schema->columns()) { - if (const auto& index_info = _context.tablet_schema->inverted_index(*column); - index_info != nullptr) { + auto index_infos = _context.tablet_schema->inverted_indexs(*column); + for (const auto& index_info : index_infos) { auto index_id = index_info->index_id(); if (_context.tablet_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index af8496bbbc9db2..2b37f166568197 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -178,7 +178,8 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e } // Delete inverted index files for (auto&& column : schema->columns()) { - if (const auto* index_info = schema->inverted_index(*column); index_info != nullptr) { + auto index_infos = schema->inverted_indexs(*column); + for (const auto& index_info : index_infos) { auto index_id = index_info->index_id(); if (schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index fb740e404303d6..0ff7ee800185ca 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -376,8 +376,11 @@ Status ColumnReader::new_index_iterator(const std::shared_ptr& RETURN_IF_ERROR(_load_index(index_file_reader, index_meta)); { std::shared_lock rlock(_load_index_lock); - if (_index_reader) { - RETURN_IF_ERROR(_index_reader->new_iterator(iterator)); + auto iter = _index_readers.find(index_meta->index_id()); + if (iter != _index_readers.end()) { + if (iter->second != nullptr) { + RETURN_IF_ERROR(iter->second->new_iterator(iterator)); + } } } return Status::OK(); @@ -661,8 +664,13 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f const TabletIndex* index_meta) { std::unique_lock wlock(_load_index_lock); - if (_index_reader != nullptr && index_meta && - _index_reader->get_index_id() == index_meta->index_id()) { + if (index_meta == nullptr) { + return Status::Error( + "Failed to load inverted index: index metadata is null"); + } + + auto it = _index_readers.find(index_meta->index_id()); + if (it != _index_readers.end()) { return Status::OK(); } @@ -676,17 +684,18 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f type = _type_info->type(); } + IndexReaderPtr index_reader; if (is_string_type(type)) { if (should_analyzer) { try { - _index_reader = FullTextIndexReader::create_shared(index_meta, index_file_reader); + index_reader = FullTextIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( "create FullTextIndexReader error: {}", e.what()); } } else { try { - _index_reader = + index_reader = StringTypeInvertedIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( @@ -695,18 +704,16 @@ Status ColumnReader::_load_index(const std::shared_ptr& index_f } } else if (is_numeric_type(type)) { try { - _index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader); + index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( "create BkdIndexReader error: {}", e.what()); } } else { - _index_reader.reset(); + return Status::Error( + "Field type {} is not supported for inverted index", type); } - // TODO: move has null to inverted_index_reader's query function - //bool has_null = true; - //RETURN_IF_ERROR(index_file_reader->has_null(index_meta, &has_null)); - //_inverted_index->set_has_null(has_null); + _index_readers[index_meta->index_id()] = index_reader; return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 081bbb302e1fa3..6711133789e5dd 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -293,7 +293,7 @@ class ColumnReader : public MetadataAdder { std::unique_ptr _bitmap_index; std::shared_ptr _bloom_filter_index; - IndexReaderPtr _index_reader; + std::unordered_map _index_readers; std::vector> _sub_readers; diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index 66d761dec556ec..4560e4ea8ade37 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -420,6 +420,7 @@ ScalarColumnWriter::ScalarColumnWriter(const ColumnWriterOptions& opts, DCHECK(opts.meta->has_compression()); DCHECK(opts.meta->has_is_nullable()); DCHECK(file_writer != nullptr); + _inverted_index_builders.resize(_opts.inverted_indexes.size()); } ScalarColumnWriter::~ScalarColumnWriter() { @@ -464,39 +465,43 @@ Status ScalarColumnWriter::init() { if (_opts.need_inverted_index) { do { - DBUG_EXECUTE_IF("column_writer.init", { - class InvertedIndexColumnWriterEmpty final : public IndexColumnWriter { - public: - Status init() override { return Status::OK(); } - Status add_values(const std::string name, const void* values, - size_t count) override { - return Status::OK(); - } - Status add_array_values(size_t field_size, const CollectionValue* values, - size_t count) override { - return Status::OK(); - } - Status add_array_values(size_t field_size, const void* value_ptr, - const uint8_t* null_map, const uint8_t* offsets_ptr, - size_t count) override { - return Status::OK(); - } - Status add_nulls(uint32_t count) override { return Status::OK(); } - Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override { - return Status::OK(); - } - Status finish() override { return Status::OK(); } - int64_t size() const override { return 0; } - void close_on_error() override {} - }; - - _index_builder = std::make_unique(); - - break; - }); - - RETURN_IF_ERROR(IndexColumnWriter::create( - get_field(), &_index_builder, _opts.index_file_writer, _opts.inverted_index)); + for (size_t i = 0; i < _opts.inverted_indexes.size(); i++) { + DBUG_EXECUTE_IF("column_writer.init", { + class InvertedIndexColumnWriterEmpty final : public IndexColumnWriter { + public: + Status init() override { return Status::OK(); } + Status add_values(const std::string name, const void* values, + size_t count) override { + return Status::OK(); + } + Status add_array_values(size_t field_size, const CollectionValue* values, + size_t count) override { + return Status::OK(); + } + Status add_array_values(size_t field_size, const void* value_ptr, + const uint8_t* null_map, const uint8_t* offsets_ptr, + size_t count) override { + return Status::OK(); + } + Status add_nulls(uint32_t count) override { return Status::OK(); } + Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override { + return Status::OK(); + } + Status finish() override { return Status::OK(); } + int64_t size() const override { return 0; } + void close_on_error() override {} + }; + + _inverted_index_builders[i] = + std::make_unique(); + + break; + }); + + RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_builders[i], + _opts.index_file_writer, + _opts.inverted_indexes[i])); + } } while (false); } if (_opts.need_bloom_filter) { @@ -522,7 +527,9 @@ Status ScalarColumnWriter::append_nulls(size_t num_rows) { _bitmap_index_builder->add_nulls(cast_set(num_rows)); } if (_opts.need_inverted_index) { - RETURN_IF_ERROR(_index_builder->add_nulls(cast_set(num_rows))); + for (const auto& builder : _inverted_index_builders) { + RETURN_IF_ERROR(builder->add_nulls(cast_set(num_rows))); + } } if (_opts.need_bloom_filter) { _bloom_filter_index_builder->add_nulls(cast_set(num_rows)); @@ -551,14 +558,16 @@ Status ScalarColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { Status ScalarColumnWriter::_internal_append_data_in_current_page(const uint8_t* data, size_t* num_written) { RETURN_IF_ERROR(_page_builder->add(data, num_written)); - if (_opts.need_zone_map) { - _zone_map_index_builder->add_values(data, *num_written); - } if (_opts.need_bitmap_index) { _bitmap_index_builder->add_values(data, *num_written); } + if (_opts.need_zone_map) { + _zone_map_index_builder->add_values(data, *num_written); + } if (_opts.need_inverted_index) { - RETURN_IF_ERROR(_index_builder->add_values(get_field()->name(), data, *num_written)); + for (const auto& builder : _inverted_index_builders) { + RETURN_IF_ERROR(builder->add_values(get_field()->name(), data, *num_written)); + } } if (_opts.need_bloom_filter) { RETURN_IF_ERROR(_bloom_filter_index_builder->add_values(data, *num_written)); @@ -650,7 +659,9 @@ Status ScalarColumnWriter::write_bitmap_index() { Status ScalarColumnWriter::write_inverted_index() { if (_opts.need_inverted_index) { - return _index_builder->finish(); + for (const auto& builder : _inverted_index_builders) { + RETURN_IF_ERROR(builder->finish()); + } } return Status::OK(); } @@ -908,8 +919,9 @@ Status ArrayColumnWriter::init() { if (_opts.need_inverted_index) { auto* writer = dynamic_cast(_item_writer.get()); if (writer != nullptr) { - RETURN_IF_ERROR(IndexColumnWriter::create( - get_field(), &_index_builder, _opts.index_file_writer, _opts.inverted_index)); + RETURN_IF_ERROR(IndexColumnWriter::create(get_field(), &_inverted_index_builder, + _opts.index_file_writer, + _opts.inverted_indexes[0])); } } return Status::OK(); @@ -917,7 +929,7 @@ Status ArrayColumnWriter::init() { Status ArrayColumnWriter::write_inverted_index() { if (_opts.need_inverted_index) { - return _index_builder->finish(); + return _inverted_index_builder->finish(); } return Status::OK(); } @@ -942,7 +954,7 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { // now only support nested type is scala if (writer != nullptr) { //NOTE: use array field name as index field, but item_writer size should be used when moving item_data_ptr - RETURN_IF_ERROR(_index_builder->add_array_values( + RETURN_IF_ERROR(_inverted_index_builder->add_array_values( _item_writer->get_field()->size(), reinterpret_cast(data), reinterpret_cast(nested_null_map), offsets_ptr, num_rows)); } @@ -963,7 +975,7 @@ Status ArrayColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t RETURN_IF_ERROR(append_data(ptr, num_rows)); if (is_nullable()) { if (_opts.need_inverted_index) { - RETURN_IF_ERROR(_index_builder->add_array_nulls(null_map, num_rows)); + RETURN_IF_ERROR(_inverted_index_builder->add_array_nulls(null_map, num_rows)); } RETURN_IF_ERROR(_null_writer->append_data(&null_map, num_rows)); } diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index e0a3f77c663ca7..1868b4bd448346 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -68,7 +68,6 @@ struct ColumnWriterOptions { uint16_t gram_bf_size; BloomFilterOptions bf_options; std::vector inverted_indexes; - const TabletIndex* inverted_index = nullptr; IndexFileWriter* index_file_writer = nullptr; SegmentFooterPB* footer = nullptr; @@ -292,7 +291,7 @@ class ScalarColumnWriter : public ColumnWriter { std::unique_ptr _ordinal_index_builder; std::unique_ptr _zone_map_index_builder; std::unique_ptr _bitmap_index_builder; - std::unique_ptr _index_builder; + std::vector> _inverted_index_builders; std::unique_ptr _bloom_filter_index_builder; // call before flush data page. @@ -421,7 +420,7 @@ class ArrayColumnWriter final : public ColumnWriter { std::unique_ptr _offset_writer; std::unique_ptr _null_writer; std::unique_ptr _item_writer; - std::unique_ptr _index_builder; + std::unique_ptr _inverted_index_builder; ColumnWriterOptions _opts; }; diff --git a/be/src/olap/rowset/segment_v2/index_iterator.h b/be/src/olap/rowset/segment_v2/index_iterator.h index 0da740627d84c5..96cd8f45914a79 100644 --- a/be/src/olap/rowset/segment_v2/index_iterator.h +++ b/be/src/olap/rowset/segment_v2/index_iterator.h @@ -34,18 +34,18 @@ class InvertedIndexQueryCacheHandle; struct InvertedIndexParam; using IndexParam = std::variant; +using IndexReaderType = std::variant; class IndexIterator { public: IndexIterator() = default; virtual ~IndexIterator() = default; - virtual IndexReaderPtr get_reader() = 0; - + virtual IndexReaderPtr get_reader(IndexReaderType reader_type) const = 0; virtual Status read_from_index(const IndexParam& param) = 0; virtual Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) = 0; - virtual bool has_null() = 0; + virtual Result has_null() = 0; void set_context(const IndexQueryContextPtr& context) { _context = context; } diff --git a/be/src/olap/rowset/segment_v2/index_reader_helper.h b/be/src/olap/rowset/segment_v2/index_reader_helper.h index 22140c17cdcdd5..11654e18722613 100644 --- a/be/src/olap/rowset/segment_v2/index_reader_helper.h +++ b/be/src/olap/rowset/segment_v2/index_reader_helper.h @@ -17,6 +17,7 @@ #pragma once +#include "olap/rowset/segment_v2/index_iterator.h" #include "olap/rowset/segment_v2/inverted_index_reader.h" namespace doris::segment_v2 { @@ -61,6 +62,32 @@ class IndexReaderHelper { return get_parser_phrase_support_string_from_properties(properties) == INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; } + + // only string type or bkd index reader can be used for equal + static bool has_string_or_bkd_index(const IndexIterator* iter) { + if (iter == nullptr) { + return false; + } + + return iter->get_reader(InvertedIndexReaderType::STRING_TYPE) != nullptr || + iter->get_reader(InvertedIndexReaderType::BKD) != nullptr; + } + + static bool has_bkd_index(const IndexIterator* iter) { + if (iter == nullptr) { + return false; + } + + return iter->get_reader(InvertedIndexReaderType::BKD) != nullptr; + } + + static bool has_string_index(const IndexIterator* iter) { + if (iter == nullptr) { + return false; + } + + return iter->get_reader(InvertedIndexReaderType::STRING_TYPE) != nullptr; + } }; #include "common/compile_check_end.h" diff --git a/be/src/olap/rowset/segment_v2/index_writer.cpp b/be/src/olap/rowset/segment_v2/index_writer.cpp index 9c1dccbe77a8a1..6bba37eb1e49fb 100644 --- a/be/src/olap/rowset/segment_v2/index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/index_writer.cpp @@ -24,11 +24,7 @@ namespace doris::segment_v2 { bool IndexColumnWriter::check_support_inverted_index(const TabletColumn& column) { // bellow types are not supported in inverted index for extracted columns - static std::set invalid_types = { - FieldType::OLAP_FIELD_TYPE_DOUBLE, - FieldType::OLAP_FIELD_TYPE_JSONB, - FieldType::OLAP_FIELD_TYPE_FLOAT, - }; + static std::set invalid_types = {FieldType::OLAP_FIELD_TYPE_JSONB}; if (invalid_types.contains(column.type())) { return false; } @@ -77,7 +73,7 @@ Status IndexColumnWriter::create(const Field* field, std::unique_ptr(reader); +InvertedIndexIterator::InvertedIndexIterator() {} + +void InvertedIndexIterator::add_reader(InvertedIndexReaderType type, + const InvertedIndexReaderPtr& reader) { + _readers[type] = reader; } Status InvertedIndexIterator::read_from_index(const IndexParam& param) { @@ -37,20 +40,22 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { DBUG_EXECUTE_IF("return_inverted_index_bypass", { return Status::Error("inverted index bypass"); }); - if (UNLIKELY(_reader == nullptr)) { - throw Exception(ErrorCode::INDEX_INVALID_PARAMETERS, "index reader is null"); - } + auto reader = DORIS_TRY(_select_best_reader(i_param->column_type, i_param->query_type)); + if (UNLIKELY(reader == nullptr)) { + throw CLuceneError(CL_ERR_NullPointer, "bkd index reader is null", false); + } auto* runtime_state = _context->runtime_state; - if (!i_param->skip_try && _reader->type() == InvertedIndexReaderType::BKD) { + if (!i_param->skip_try && reader->type() == InvertedIndexReaderType::BKD) { if (runtime_state != nullptr && runtime_state->query_options().inverted_index_skip_threshold > 0 && runtime_state->query_options().inverted_index_skip_threshold < 100) { auto query_bkd_limit_percent = runtime_state->query_options().inverted_index_skip_threshold; size_t hit_count = 0; - RETURN_IF_ERROR(try_read_from_inverted_index(i_param->column_name, i_param->query_value, - i_param->query_type, &hit_count)); + RETURN_IF_ERROR(try_read_from_inverted_index(reader, i_param->column_name, + i_param->query_value, i_param->query_type, + &hit_count)); if (hit_count > i_param->num_rows * query_bkd_limit_percent / 100) { return Status::Error( "hit count: {}, bkd inverted reached limit {}% , segment num " @@ -61,8 +66,8 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { } auto execute_query = [&]() { - return _reader->query(_context, i_param->column_name, i_param->query_value, - i_param->query_type, i_param->roaring); + return reader->query(_context, i_param->column_name, i_param->query_value, + i_param->query_type, i_param->roaring); }; if (runtime_state->query_options().enable_profile) { @@ -82,14 +87,17 @@ Status InvertedIndexIterator::read_from_index(const IndexParam& param) { } Status InvertedIndexIterator::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) { - return _reader->read_null_bitmap(_context, cache_handle, nullptr); + auto reader = DORIS_TRY(_select_best_reader()); + return reader->read_null_bitmap(_context, cache_handle, nullptr); } -bool InvertedIndexIterator::has_null() { - return _reader->has_null(); +Result InvertedIndexIterator::has_null() { + auto reader = DORIS_TRY(_select_best_reader()); + return reader->has_null(); } -Status InvertedIndexIterator::try_read_from_inverted_index(const std::string& column_name, +Status InvertedIndexIterator::try_read_from_inverted_index(const InvertedIndexReaderPtr& reader, + const std::string& column_name, const void* query_value, InvertedIndexQueryType query_type, size_t* count) { @@ -99,9 +107,67 @@ Status InvertedIndexIterator::try_read_from_inverted_index(const std::string& co query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY || query_type == InvertedIndexQueryType::LESS_THAN_QUERY || query_type == InvertedIndexQueryType::EQUAL_QUERY) { - RETURN_IF_ERROR(_reader->try_query(_context, column_name, query_value, query_type, count)); + RETURN_IF_ERROR(reader->try_query(_context, column_name, query_value, query_type, count)); } return Status::OK(); } +Result InvertedIndexIterator::_select_best_reader( + const vectorized::DataTypePtr& column_type, InvertedIndexQueryType query_type) { + if (_readers.empty()) { + return ResultError(Status::RuntimeError( + "No available inverted index readers. Check if index is properly initialized.")); + } + + // BKD and array types allow only one reader each + if (_readers.size() == 1) { + return _readers.begin()->second; + } + + // Check for string types + const auto field_type = column_type->get_storage_field_type(); + const bool is_string = is_string_type(field_type); + + InvertedIndexReaderType preferred_type = InvertedIndexReaderType::UNKNOWN; + // Handle string type columns + if (is_string) { + if (is_match_query(query_type)) { + preferred_type = InvertedIndexReaderType::FULLTEXT; + } else if (is_equal_query(query_type)) { + preferred_type = InvertedIndexReaderType::STRING_TYPE; + } + } + DBUG_EXECUTE_IF("inverted_index_reader._select_best_reader", { + auto type = DebugPoints::instance()->get_debug_param_or_default( + "inverted_index_reader._select_best_reader", "type", -1); + if ((int32_t)preferred_type != type) { + return ResultError(Status::RuntimeError( + "Inverted index reader type mismatch. Expected={}, Actual={}", + (int32_t)preferred_type, type)); + } + }) + + if (auto reader = get_reader(preferred_type)) { + return std::static_pointer_cast(reader); + } + + return ResultError(Status::RuntimeError("Index query type not supported")); +} + +Result InvertedIndexIterator::_select_best_reader() { + if (_readers.empty()) { + return ResultError(Status::RuntimeError( + "No available inverted index readers. Check if index is properly initialized.")); + } + return _readers.begin()->second; +} + +IndexReaderPtr InvertedIndexIterator::get_reader(IndexReaderType type) const { + auto iter = _readers.find(type); + if (iter == _readers.end()) { + return nullptr; + } + return iter->second; +} + } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_iterator.h b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h index 7e1d1a89797cb5..a3a1cc66dfe1a3 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_iterator.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h @@ -24,6 +24,7 @@ namespace doris::segment_v2 { struct InvertedIndexParam { std::string column_name; + vectorized::DataTypePtr column_type; const void* query_value; InvertedIndexQueryType query_type; uint32_t num_rows; @@ -33,25 +34,30 @@ struct InvertedIndexParam { class InvertedIndexIterator : public IndexIterator { public: - InvertedIndexIterator(const IndexReaderPtr& reader); + InvertedIndexIterator(); ~InvertedIndexIterator() override = default; - IndexReaderPtr get_reader() override { return std::static_pointer_cast(_reader); } + void add_reader(InvertedIndexReaderType type, const InvertedIndexReaderPtr& reader); Status read_from_index(const IndexParam& param) override; Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) override; - bool has_null() override; -private: - Status try_read_from_inverted_index(const std::string& column_name, const void* query_value, - InvertedIndexQueryType query_type, size_t* count); + [[nodiscard]] Result has_null() override; - InvertedIndexReaderPtr _reader; + IndexReaderPtr get_reader(IndexReaderType reader_type) const override; +private: ENABLE_FACTORY_CREATOR(InvertedIndexIterator); - friend class InvertedIndexReaderTest; + Status try_read_from_inverted_index(const InvertedIndexReaderPtr& reader, + const std::string& column_name, const void* query_value, + InvertedIndexQueryType query_type, size_t* count); + Result _select_best_reader(const vectorized::DataTypePtr& column_type, + InvertedIndexQueryType query_type); + Result _select_best_reader(); + + std::unordered_map _readers; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_query_type.h b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h index f1a47ebdd0f2eb..4595ed64cdf1e5 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_query_type.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h @@ -81,6 +81,10 @@ enum class InvertedIndexQueryType { MATCH_PHRASE_EDGE_QUERY = 10, }; +inline bool is_equal_query(InvertedIndexQueryType query_type) { + return query_type == InvertedIndexQueryType::EQUAL_QUERY; +} + inline bool is_range_query(InvertedIndexQueryType query_type) { return (query_type == InvertedIndexQueryType::GREATER_THAN_QUERY || query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY || diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 12853cbc916ac2..e899ca859bbd1c 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -39,6 +39,7 @@ #include "common/logging.h" #include "common/status.h" #include "inverted_index_query_type.h" +#include "olap/field.h" #include "olap/inverted_index_parser.h" #include "olap/key_coder.h" #include "olap/olap_common.h" @@ -59,47 +60,6 @@ namespace doris::segment_v2 { #include "common/compile_check_begin.h" -template -Status InvertedIndexQueryParamFactory::create_query_value( - const void* value, std::unique_ptr& result_param) { - using CPP_TYPE = typename PrimitiveTypeTraits::CppType; - std::unique_ptr> param = - InvertedIndexQueryParam::create_unique(); - auto&& storage_val = PrimitiveTypeConvertor::to_storage_field_type( - *reinterpret_cast(value)); - param->set_value(&storage_val); - result_param = std::move(param); - return Status::OK(); -}; - -#define CREATE_QUERY_VALUE_TEMPLATE(PT) \ - template Status InvertedIndexQueryParamFactory::create_query_value( \ - const void* value, std::unique_ptr& result_param); - -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BOOLEAN) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_TINYINT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_SMALLINT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_INT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BIGINT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_LARGEINT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_FLOAT) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DOUBLE) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_VARCHAR) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATE) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATEV2) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIME) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIMEV2) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_CHAR) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMALV2) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL32) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL64) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL128I) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL256) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_HLL) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_STRING) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV4) -CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV6) - std::string InvertedIndexReader::get_index_file_path() { return _index_file_reader->get_index_file_path(&_index_meta); } @@ -309,7 +269,12 @@ Status InvertedIndexReader::match_index_search( } Status FullTextIndexReader::new_iterator(std::unique_ptr* iterator) { - *iterator = InvertedIndexIterator::create_unique(shared_from_this()); + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(InvertedIndexReaderType::FULLTEXT, + dynamic_pointer_cast(shared_from_this())); return Status::OK(); } @@ -404,7 +369,12 @@ InvertedIndexReaderType FullTextIndexReader::type() { } Status StringTypeInvertedIndexReader::new_iterator(std::unique_ptr* iterator) { - *iterator = InvertedIndexIterator::create_unique(shared_from_this()); + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(InvertedIndexReaderType::STRING_TYPE, + dynamic_pointer_cast(shared_from_this())); return Status::OK(); } @@ -533,7 +503,12 @@ InvertedIndexReaderType StringTypeInvertedIndexReader::type() { } Status BkdIndexReader::new_iterator(std::unique_ptr* iterator) { - *iterator = InvertedIndexIterator::create_unique(shared_from_this()); + if (*iterator == nullptr) { + *iterator = InvertedIndexIterator::create_unique(); + } + dynamic_cast(iterator->get()) + ->add_reader(InvertedIndexReaderType::BKD, + dynamic_pointer_cast(shared_from_this())); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 4fa2ec7c2aa6e8..b3b7b70097a9c8 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -367,6 +367,9 @@ class BkdIndexReader : public InvertedIndexReader { const KeyCoder* _value_key_coder {}; }; +template +class InvertedIndexQueryParam; + /** * @brief InvertedIndexQueryParamFactory is a factory class to create QueryValue object. * we need a template function to make predict class like in_list_predict template class to use. @@ -379,17 +382,38 @@ class InvertedIndexQueryParamFactory { public: virtual ~InvertedIndexQueryParamFactory() = default; - template - static Status create_query_value(const void* value, - std::unique_ptr& result_param); + template + static Status create_query_value( + const ValueType* value, std::unique_ptr& result_param) { + static_assert(!std::is_same_v, + "ValueType cannot be void, as it is unsupported and dangerous."); + + using CPP_TYPE = typename PrimitiveTypeTraits::CppType; + std::unique_ptr> param = + InvertedIndexQueryParam::create_unique(); + + CPP_TYPE cpp_val; + if constexpr (std::is_same_v) { + auto field_val = + doris::vectorized::get>(*value); + cpp_val = static_cast(field_val); + } else { + cpp_val = static_cast(*value); + } + + auto storage_val = PrimitiveTypeConvertor::to_storage_field_type(cpp_val); + param->set_value(&storage_val); + result_param = std::move(param); + return Status::OK(); + } static Status create_query_value( - const PrimitiveType& primitiveType, const void* value, + const PrimitiveType& primitiveType, const doris::vectorized::Field* value, std::unique_ptr& result_param) { switch (primitiveType) { -#define M(TYPE) \ - case TYPE: { \ - return create_query_value(value, result_param); \ +#define M(TYPE) \ + case TYPE: { \ + return create_query_value(value, result_param); \ } M(PrimitiveType::TYPE_BOOLEAN) M(PrimitiveType::TYPE_TINYINT) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index dcc4c0f30235e9..2387717306ef30 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -708,7 +708,6 @@ Status InvertedIndexColumnWriter::finish() { template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; - template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; @@ -727,5 +726,7 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; +template class InvertedIndexColumnWriter; +template class InvertedIndexColumnWriter; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index d8c337e9b2dd7a..1ae3e7ad107ec5 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -826,13 +826,8 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred) { // UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors if (PredicateTypeTraits::is_range(pred->type()) && - _index_iterators[pred_column_id] != nullptr) { - auto reader = _index_iterators[pred_column_id]->get_reader(); - if (reader->index_type() == IndexType::INVERTED) { - if (IndexReaderHelper::is_string_index(reader)) { - return false; - } - } + !IndexReaderHelper::has_bkd_index(_index_iterators[pred_column_id].get())) { + return false; } // Function filter no apply inverted index @@ -908,16 +903,12 @@ bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) } bool SegmentIterator::_column_has_fulltext_index(int32_t cid) { - if (_index_iterators[cid] == nullptr) { - return false; - } - - auto reader = _index_iterators[cid]->get_reader(); - if (reader->index_type() != IndexType::INVERTED) { - return false; - } + bool has_fulltext_index = + _index_iterators[cid] != nullptr && + _index_iterators[cid]->get_reader(InvertedIndexReaderType::FULLTEXT) && + _index_iterators[cid]->get_reader(InvertedIndexReaderType::STRING_TYPE) == nullptr; - return IndexReaderHelper::is_fulltext_index(reader); + return has_fulltext_index; } inline bool SegmentIterator::_inverted_index_not_support_pred_type(const PredicateType& type) { @@ -1198,7 +1189,7 @@ Status SegmentIterator::_init_index_iterators() { } // If the column is not an extracted column, we can directly get the inverted index metadata from the tablet schema. else { - inverted_indexs = {_segment->_tablet_schema->inverted_index(column)}; + inverted_indexs = {_segment->_tablet_schema->inverted_indexs(column)}; } for (const auto& inverted_index : inverted_indexs) { RETURN_IF_ERROR(_segment->new_index_iterator(column, inverted_index, _opts, diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index fed429af04ca43..9356c34ff3b8ed 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -229,11 +229,7 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co if (!skip_inverted_index) { auto inverted_indexs = schema->inverted_indexs(column); if (!inverted_indexs.empty()) { - // TODO(lihangyu) multi indexes - // for (const auto& index : inverted_indexs) { - // opts.inverted_indexs.emplace_back(index); - // } - opts.inverted_index = inverted_indexs.front(); + opts.inverted_indexes = inverted_indexs; opts.need_inverted_index = true; DCHECK(_index_file_writer != nullptr); } diff --git a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp index 283957ad499766..088d8ec0f5a161 100644 --- a/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp +++ b/be/src/olap/rowset/segment_v2/variant/variant_column_writer_impl.cpp @@ -97,11 +97,9 @@ Status _create_column_writer(uint32_t cid, const TabletColumn& column, if (segment_v2::IndexColumnWriter::check_support_inverted_index(column)) { auto init_opt_inverted_index = [&]() { DCHECK(!subcolumn_indexes.empty()); - // TODO(lihangyu) multi indexes - // for (const auto& index : subcolumn_indexes) { - // opt->inverted_indexs.push_back(index.get()); - // } - opt->inverted_index = subcolumn_indexes.front().get(); + for (const auto& index : subcolumn_indexes) { + opt->inverted_indexes.push_back(index.get()); + } opt->need_inverted_index = true; DCHECK(inverted_index_file_writer != nullptr); opt->index_file_writer = inverted_index_file_writer; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 6f597a378051e1..f5e904fd016ec3 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -229,11 +229,7 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo if (!skip_inverted_index) { auto inverted_indexs = tablet_schema->inverted_indexs(column); if (!inverted_indexs.empty()) { - // TODO(lihangyu) multi indexes - // for (const auto& index : inverted_indexs) { - // opts.inverted_indexs.emplace_back(index); - // } - opts.inverted_index = inverted_indexs.front(); + opts.inverted_indexes = inverted_indexs; opts.need_inverted_index = true; DCHECK(_index_file_writer != nullptr); } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index e16e69ac4c8334..13d6b8776a0b69 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -1580,29 +1580,6 @@ bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id) const { return false; } -const TabletIndex* TabletSchema::inverted_index(int32_t col_unique_id, - const std::string& suffix_path) const { - const std::string escaped_suffix = escape_for_path_name(suffix_path); - auto it = _col_id_suffix_to_index.find( - std::make_tuple(IndexType::INVERTED, col_unique_id, escaped_suffix)); - if (it != _col_id_suffix_to_index.end() && !it->second.empty() && - it->second[0] < _indexes.size()) { - return _indexes[it->second[0]].get(); - } - return nullptr; -} - -const TabletIndex* TabletSchema::inverted_index(const TabletColumn& col) const { - // Some columns(Float, Double, JSONB ...) from the variant do not support inverted index - if (!segment_v2::IndexColumnWriter::check_support_inverted_index(col)) { - return nullptr; - } - // TODO use more efficient impl - // Use parent id if unique not assigned, this could happend when accessing subcolumns of variants - int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id() : col.unique_id(); - return inverted_index(col_unique_id, escape_for_path_name(col.suffix_path())); -} - std::vector TabletSchema::inverted_indexs( int32_t col_unique_id, const std::string& suffix_path) const { std::vector result; diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 23d1613154e39c..82ec82ea99901b 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -488,18 +488,7 @@ class TabletSchema : public MetadataAdder { return false; } bool has_inverted_index_with_index_id(int64_t index_id) const; - // todo: remove this func - // Check whether this column supports inverted index - // Some columns (Float, Double, JSONB ...) from the variant do not support index, but they are listed in TabletIndex. - const TabletIndex* inverted_index(const TabletColumn& col) const; - - // todo: remove this func - // Regardless of whether this column supports inverted index - // TabletIndex information will be returned as long as it exists. - const TabletIndex* inverted_index(int32_t col_unique_id, - const std::string& suffix_path = "") const; - - // TODO(lihangyu): multi indexes + void update_index(const TabletColumn& column, const IndexType& index_type, std::vector&& indexes); diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index e150389c39a6e4..0442da84aa38c5 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -112,34 +112,36 @@ Status IndexBuilder::update_inverted_index_info() { } } auto column = output_rs_tablet_schema->column(column_idx); - const auto* index_meta = output_rs_tablet_schema->inverted_index(column); - if (index_meta == nullptr) { + auto index_metas = output_rs_tablet_schema->inverted_indexs(column); + if (index_metas.empty()) { LOG(ERROR) << "failed to find column: " << column_name << " index_id: " << t_inverted_index.index_id; continue; } - if (output_rs_tablet_schema->get_inverted_index_storage_format() == - InvertedIndexStorageFormatPB::V1) { - const auto& fs = io::global_local_filesystem(); - - for (int seg_id = 0; seg_id < num_segments; seg_id++) { - auto seg_path = - local_segment_path(_tablet->tablet_path(), - input_rowset->rowset_id().to_string(), seg_id); - auto index_path = InvertedIndexDescriptor::get_index_file_path_v1( - InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), - index_meta->index_id(), index_meta->get_index_suffix()); - int64_t index_size = 0; - RETURN_IF_ERROR(fs->file_size(index_path, &index_size)); - VLOG_DEBUG << "inverted index file:" << index_path - << " size:" << index_size; - drop_index_size += index_size; + for (const auto& index_meta : index_metas) { + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + const auto& fs = io::global_local_filesystem(); + + for (int seg_id = 0; seg_id < num_segments; seg_id++) { + auto seg_path = local_segment_path( + _tablet->tablet_path(), input_rowset->rowset_id().to_string(), + seg_id); + auto index_path = InvertedIndexDescriptor::get_index_file_path_v1( + InvertedIndexDescriptor::get_index_file_path_prefix(seg_path), + index_meta->index_id(), index_meta->get_index_suffix()); + int64_t index_size = 0; + RETURN_IF_ERROR(fs->file_size(index_path, &index_size)); + VLOG_DEBUG << "inverted index file:" << index_path + << " size:" << index_size; + drop_index_size += index_size; + } } + _dropped_inverted_indexes.push_back(*index_meta); + // ATTN: DO NOT REMOVE INDEX AFTER OUTPUT_ROWSET_WRITER CREATED. + // remove dropped index_meta from output rowset tablet schema + output_rs_tablet_schema->remove_index(index_meta->index_id()); } - _dropped_inverted_indexes.push_back(*index_meta); - // ATTN: DO NOT REMOVE INDEX AFTER OUTPUT_ROWSET_WRITER CREATED. - // remove dropped index_meta from output rowset tablet schema - output_rs_tablet_schema->remove_index(index_meta->index_id()); } DBUG_EXECUTE_IF("index_builder.update_inverted_index_info.drop_index", { auto indexes_count = DebugPoints::instance()->get_debug_param_or_default( @@ -169,15 +171,21 @@ Status IndexBuilder::update_inverted_index_info() { continue; } const TabletColumn& col = output_rs_tablet_schema->column_by_uid(column_uid); - const TabletIndex* exist_index = output_rs_tablet_schema->inverted_index(col); - if (exist_index && exist_index->index_id() != index.index_id()) { - LOG(WARNING) << fmt::format( - "column: {} has a exist inverted index, but the index id not equal " - "request's index id, , exist index id: {}, request's index id: {}, " - "remove exist index in new output_rs_tablet_schema", - column_uid, exist_index->index_id(), index.index_id()); - without_index_uids.insert(exist_index->index_id()); - output_rs_tablet_schema->remove_index(exist_index->index_id()); + auto exist_indexs = output_rs_tablet_schema->inverted_indexs(col); + for (const auto& exist_index : exist_indexs) { + if (exist_index->index_id() != index.index_id()) { + if (exist_index->is_same_except_id(&index)) { + LOG(WARNING) << fmt::format( + "column: {} has a exist inverted index, but the index id not " + "equal " + "request's index id, , exist index id: {}, request's index id: " + "{}, " + "remove exist index in new output_rs_tablet_schema", + column_uid, exist_index->index_id(), index.index_id()); + without_index_uids.insert(exist_index->index_id()); + output_rs_tablet_schema->remove_index(exist_index->index_id()); + } + } } output_rs_tablet_schema->append_index(std::move(index)); } @@ -425,28 +433,36 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); std::unique_ptr field(FieldFactory::create(column)); - const auto* index_meta = output_rowset_schema->inverted_index(column); - std::unique_ptr inverted_index_builder; - try { - RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( - field.get(), &inverted_index_builder, index_file_writer.get(), - index_meta)); - DBUG_EXECUTE_IF( - "IndexBuilder::handle_single_rowset_index_column_writer_create_error", { - _CLTHROWA(CL_ERR_IO, - "debug point: " - "handle_single_rowset_index_column_writer_create_error"); - }) - } catch (const std::exception& e) { - return Status::Error( - "CLuceneError occured: {}", e.what()); - } + auto index_metas = output_rowset_schema->inverted_indexs(column); + for (const auto& index_meta : index_metas) { + if (index_meta->index_id() != index_id) { + continue; + } + std::unique_ptr inverted_index_builder; + try { + RETURN_IF_ERROR(segment_v2::IndexColumnWriter::create( + field.get(), &inverted_index_builder, index_file_writer.get(), + index_meta)); + DBUG_EXECUTE_IF( + "IndexBuilder::handle_single_rowset_index_column_writer_create_" + "error", + { + _CLTHROWA(CL_ERR_IO, + "debug point: " + "handle_single_rowset_index_column_writer_create_" + "error"); + }) + } catch (const std::exception& e) { + return Status::Error( + "CLuceneError occured: {}", e.what()); + } - if (inverted_index_builder) { - auto writer_sign = std::make_pair(seg_ptr->id(), index_id); - _inverted_index_builders.insert( - std::make_pair(writer_sign, std::move(inverted_index_builder))); - inverted_index_writer_signs.emplace_back(writer_sign); + if (inverted_index_builder) { + auto writer_sign = std::make_pair(seg_ptr->id(), index_id); + _inverted_index_builders.insert( + std::make_pair(writer_sign, std::move(inverted_index_builder))); + inverted_index_writer_signs.emplace_back(writer_sign); + } } } diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index c038877e4724ad..d222b2571a37db 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -617,20 +617,17 @@ bool has_schema_index_diff(const TabletSchema* new_schema, const TabletSchema* o auto new_schema_inverted_indexs = new_schema->inverted_indexs(column_new); auto old_schema_inverted_indexs = old_schema->inverted_indexs(column_old); - // TODO(lihangyu): multi indexes, use comment to replace this - return new_schema_inverted_indexs.size() != old_schema_inverted_indexs.size(); - - // if (new_schema_inverted_indexs.size() != old_schema_inverted_indexs.size()) { - // return true; - // } + if (new_schema_inverted_indexs.size() != old_schema_inverted_indexs.size()) { + return true; + } - // for (size_t i = 0; i < new_schema_inverted_indexs.size(); ++i) { - // if (!new_schema_inverted_indexs[i]->is_same_except_id(old_schema_inverted_indexs[i])) { - // return true; - // } - // } + for (size_t i = 0; i < new_schema_inverted_indexs.size(); ++i) { + if (!new_schema_inverted_indexs[i]->is_same_except_id(old_schema_inverted_indexs[i])) { + return true; + } + } - // return false; + return false; } TabletColumn create_sparse_column(const TabletColumn& variant) { diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index 1cb67394b05e4f..cf343e31ae1bd1 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -138,7 +138,7 @@ class FunctionArrayIndex : public IFunction { if (iter == nullptr) { return Status::OK(); } - if (IndexReaderHelper::is_fulltext_index(iter->get_reader())) { + if (!segment_v2::IndexReaderHelper::has_string_or_bkd_index(iter)) { // parser is not none we can not make sure the result is correct in expr combination // for example, filter: !array_index(array, 'tall:120cm, weight: 35kg') // here we have rows [tall:120cm, weight: 35kg, hobbies: reading book] which be tokenized @@ -167,6 +167,7 @@ class FunctionArrayIndex : public IFunction { query_param)); InvertedIndexParam param; param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; param.query_value = query_param->get_value(); param.query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY; param.num_rows = num_rows; diff --git a/be/src/vec/functions/array/function_arrays_overlap.h b/be/src/vec/functions/array/function_arrays_overlap.h index a8d9e0fd5977de..07f49f32cd35dc 100644 --- a/be/src/vec/functions/array/function_arrays_overlap.h +++ b/be/src/vec/functions/array/function_arrays_overlap.h @@ -174,9 +174,9 @@ class FunctionArraysOverlap : public IFunction { return Status::OK(); } auto data_type_with_name = data_type_with_names[0]; - if (segment_v2::IndexReaderHelper::is_fulltext_index(iter->get_reader())) { + if (!segment_v2::IndexReaderHelper::has_string_or_bkd_index(iter)) { return Status::Error( - "Inverted index evaluate skipped, FULLTEXT reader can not support " + "Inverted index evaluate skipped, no inverted index reader can not support " "array_overlap"); } // in arrays_overlap param is array Field and const Field @@ -213,6 +213,7 @@ class FunctionArraysOverlap : public IFunction { InvertedIndexParam param; param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; param.query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY; param.num_rows = num_rows; for (auto nested_query_val : query_val) { diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h index a7f7aad48a6ef0..90bd34b00f2995 100644 --- a/be/src/vec/functions/function_ip.h +++ b/be/src/vec/functions/function_ip.h @@ -657,7 +657,7 @@ class FunctionIsIPAddressInRange : public IFunction { return Status::OK(); } - if (!segment_v2::IndexReaderHelper::is_bkd_index(iter->get_reader())) { + if (!segment_v2::IndexReaderHelper::has_bkd_index(iter)) { // Not support only bkd index return Status::Error( "Inverted index evaluate skipped, ip range reader can only support by bkd " @@ -715,6 +715,7 @@ class FunctionIsIPAddressInRange : public IFunction { param_type, &min_ip, query_param)); segment_v2::InvertedIndexParam res_param; res_param.column_name = data_type_with_name.first; + res_param.column_type = data_type_with_name.second; res_param.query_type = segment_v2::InvertedIndexQueryType::GREATER_EQUAL_QUERY; res_param.query_value = query_param->get_value(); res_param.num_rows = num_rows; @@ -726,6 +727,7 @@ class FunctionIsIPAddressInRange : public IFunction { param_type, &max_ip, query_param)); segment_v2::InvertedIndexParam max_param; max_param.column_name = data_type_with_name.first; + max_param.column_type = data_type_with_name.second; max_param.query_type = segment_v2::InvertedIndexQueryType::LESS_EQUAL_QUERY; max_param.query_value = query_param->get_value(); max_param.num_rows = num_rows; diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h index 13c87e9202b434..cf6ec9a4d8928c 100644 --- a/be/src/vec/functions/functions_comparison.h +++ b/be/src/vec/functions/functions_comparison.h @@ -578,8 +578,7 @@ class FunctionComparison : public IFunction { if (iter == nullptr) { return Status::OK(); } - if (segment_v2::IndexReaderHelper::is_fulltext_index(iter->get_reader())) { - //NOT support comparison predicate when parser is FULLTEXT for expr inverted index evaluate. + if (!segment_v2::IndexReaderHelper::has_string_or_bkd_index(iter)) { return Status::OK(); } segment_v2::InvertedIndexQueryType query_type; @@ -599,11 +598,10 @@ class FunctionComparison : public IFunction { } if (segment_v2::is_range_query(query_type) && - segment_v2::IndexReaderHelper::is_string_index(iter->get_reader())) { + iter->get_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE)) { // untokenized strings exceed ignore_above, they are written as null, causing range query errors return Status::OK(); } - std::string column_name = data_type_with_name.first; Field param_value; arguments[0].column->get(0, param_value); auto param_type = arguments[0].type->get_primitive_type(); @@ -614,7 +612,8 @@ class FunctionComparison : public IFunction { param_type, ¶m_value, query_param)); segment_v2::InvertedIndexParam param; - param.column_name = column_name; + param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; param.query_value = query_param->get_value(); param.query_type = query_type; param.num_rows = num_rows; diff --git a/be/src/vec/functions/in.h b/be/src/vec/functions/in.h index 76d195d9fb2dd9..6324cdfb97f2d8 100644 --- a/be/src/vec/functions/in.h +++ b/be/src/vec/functions/in.h @@ -149,7 +149,7 @@ class FunctionIn : public IFunction { if (iter == nullptr) { return Status::OK(); } - if (segment_v2::IndexReaderHelper::is_fulltext_index(iter->get_reader())) { + if (!segment_v2::IndexReaderHelper::has_string_or_bkd_index(iter)) { //NOT support in list when parser is FULLTEXT for expr inverted index evaluate. return Status::OK(); } @@ -158,7 +158,6 @@ class FunctionIn : public IFunction { RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); null_bitmap = null_bitmap_cache_handle.get_bitmap(); } - std::string column_name = data_type_with_name.first; for (const auto& arg : arguments) { Field param_value; arg.column->get(0, param_value); @@ -176,7 +175,8 @@ class FunctionIn : public IFunction { param_type, ¶m_value, query_param)); InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY; segment_v2::InvertedIndexParam param; - param.column_name = column_name; + param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; param.query_value = query_param->get_value(); param.query_type = query_type; param.num_rows = num_rows; diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp index d0178b9c90ce95..97902241436b30 100644 --- a/be/src/vec/functions/match.cpp +++ b/be/src/vec/functions/match.cpp @@ -44,8 +44,8 @@ Status FunctionMatchBase::evaluate_inverted_index( if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION || function_name == MATCH_PHRASE_EDGE_FUNCTION) { - if (segment_v2::IndexReaderHelper::is_fulltext_index(iter->get_reader()) && - !segment_v2::IndexReaderHelper::is_support_phrase(iter->get_reader())) { + auto reader = iter->get_reader(InvertedIndexReaderType::FULLTEXT); + if (reader && !segment_v2::IndexReaderHelper::is_support_phrase(reader)) { return Status::Error( "phrase queries require setting support_phrase = true"); } @@ -67,6 +67,7 @@ Status FunctionMatchBase::evaluate_inverted_index( InvertedIndexParam param; param.column_name = data_type_with_name.first; + param.column_type = data_type_with_name.second; param.query_value = query_param->get_value(); param.query_type = get_query_type_from_fn_name(); param.num_rows = num_rows; diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp index 8ea5a9c7767f08..3ca134ac1120a4 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/index_compaction_test.cpp @@ -898,7 +898,7 @@ TEST_F(IndexCompactionTest, test_tablet_index_id_not_equal) { data_files.push_back(data_file2); std::vector rowsets(data_files.size()); - auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 3); }; + auto custom_check_build_rowsets = [](const int32_t& size) { EXPECT_EQ(size, 4); }; IndexCompactionUtils::build_rowsets( _data_dir, _tablet_schema, _tablet, _engine_ref, rowsets, data_files, _inc_id, custom_check_build_rowsets); @@ -923,11 +923,9 @@ TEST_F(IndexCompactionTest, test_tablet_index_id_not_equal) { _tablet_schema->get_inverted_index_storage_format()); // check index file - // index 10002 cannot be found in idx file + // index 10002 can be found in idx file auto dir_idx_compaction = inverted_index_file_reader_index->_open(10002, ""); - EXPECT_TRUE(!dir_idx_compaction.has_value()) << dir_idx_compaction.error(); - EXPECT_THAT(dir_idx_compaction.error().to_string(), - testing::HasSubstr("No index with id 10002 found")); + EXPECT_TRUE(dir_idx_compaction.has_value()) << dir_idx_compaction.error(); } TEST_F(IndexCompactionTest, test_tablet_schema_tablet_index_is_null) { diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp index 2ac867474f265b..8fa19c823e9b2b 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp @@ -595,8 +595,8 @@ class IndexCompactionUtils { for (const auto& [col_uid, query_data] : query_map) { const auto& column = tablet_schema->column_by_uid(col_uid); - const auto* index = tablet_schema->inverted_index(column); - EXPECT_TRUE(index != nullptr); + auto indexs = tablet_schema->inverted_indexs(column); + EXPECT_FALSE(indexs.empty()); if (col_uid == 0 || col_uid == 3) { // BKD index @@ -604,14 +604,15 @@ class IndexCompactionUtils { for (const auto& data : query_data.first) { query_data_int.push_back(std::stoi(data)); } - EXPECT_TRUE(query_bkd(index, index_file_reader, query_data_int, query_data.second)); + EXPECT_TRUE( + query_bkd(indexs[0], index_file_reader, query_data_int, query_data.second)); } else if (col_uid == 1) { // String index - EXPECT_TRUE(query_string(index, index_file_reader, std::to_string(col_uid), + EXPECT_TRUE(query_string(indexs[0], index_file_reader, std::to_string(col_uid), query_data.first, query_data.second)); } else if (col_uid == 2) { // Fulltext index - EXPECT_TRUE(query_fulltext(index, index_file_reader, std::to_string(col_uid), + EXPECT_TRUE(query_fulltext(indexs[0], index_file_reader, std::to_string(col_uid), query_data.first, query_data.second)); } } diff --git a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp index e57091bd6ea7fd..ae85a05fb72b20 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp @@ -2352,8 +2352,8 @@ class InvertedIndexReaderTest : public testing::Test { EXPECT_NE(iterator, nullptr); // Test iterator properties - auto inverted_index_reader = - std::static_pointer_cast(iterator->get_reader()); + auto inverted_index_reader = std::static_pointer_cast( + iterator->get_reader(InvertedIndexReaderType::STRING_TYPE)); EXPECT_EQ(inverted_index_reader->type(), InvertedIndexReaderType::STRING_TYPE); EXPECT_FALSE(inverted_index_reader->get_index_properties().empty()); EXPECT_TRUE(inverted_index_reader->has_null()); @@ -2378,7 +2378,8 @@ class InvertedIndexReaderTest : public testing::Test { auto* inverted_index_iterator = static_cast(iterator.get()); inverted_index_iterator->set_context(context); status = inverted_index_iterator->try_read_from_inverted_index( - "c2", &str_ref, InvertedIndexQueryType::EQUAL_QUERY, &count); + std::static_pointer_cast(inverted_index_reader), "c2", + &str_ref, InvertedIndexQueryType::EQUAL_QUERY, &count); EXPECT_TRUE(status.ok()); } @@ -2928,6 +2929,8 @@ class InvertedIndexReaderTest : public testing::Test { // Test try_read_from_inverted_index with non-BKD compatible query size_t count = 0; status = inverted_index_iterator->try_read_from_inverted_index( + std::static_pointer_cast( + iterator->get_reader(InvertedIndexReaderType::STRING_TYPE)), "c1", &query_value, InvertedIndexQueryType::MATCH_ANY_QUERY, &count); EXPECT_TRUE(status.ok()); // Should succeed but not do anything for non-BKD queries } diff --git a/be/test/olap/tablet_index_test.cpp b/be/test/olap/tablet_index_test.cpp index 7842f9af18d51d..da07dd22060443 100644 --- a/be/test/olap/tablet_index_test.cpp +++ b/be/test/olap/tablet_index_test.cpp @@ -61,11 +61,11 @@ TEST_F(TabletIndexTest, test_inverted_index) { EXPECT_TRUE(tablet_schema->has_inverted_index()); EXPECT_EQ(tablet_schema->inverted_indexes().size(), 2); - EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(0)) != nullptr); - EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(1)) != nullptr); - EXPECT_TRUE(tablet_schema->inverted_index(tablet_schema->column_by_uid(2)) == nullptr); - EXPECT_TRUE(tablet_schema->inverted_index(3) == nullptr); - EXPECT_TRUE(tablet_schema->inverted_index(4, "v1.a") == nullptr); + EXPECT_FALSE(tablet_schema->inverted_indexs(tablet_schema->column_by_uid(0)).empty()); + EXPECT_FALSE(tablet_schema->inverted_indexs(tablet_schema->column_by_uid(1)).empty()); + EXPECT_TRUE(tablet_schema->inverted_indexs(tablet_schema->column_by_uid(2)).empty()); + EXPECT_TRUE(tablet_schema->inverted_indexs(3).empty()); + EXPECT_TRUE(tablet_schema->inverted_indexs(4, "v1.a").empty()); } TEST_F(TabletIndexTest, test_schema_index_diff) { diff --git a/be/test/olap/tablet_schema_index_test.cpp b/be/test/olap/tablet_schema_index_test.cpp index 26a2b726220950..763dda26a37628 100644 --- a/be/test/olap/tablet_schema_index_test.cpp +++ b/be/test/olap/tablet_schema_index_test.cpp @@ -75,10 +75,10 @@ TEST_F(TabletSchemaIndexTest, TestAddInvertedIndex) { _tablet_schema->append_index(std::move(index)); // Verify index mapping - auto* found_index = _tablet_schema->inverted_index(100, "suffix1"); - ASSERT_NE(found_index, nullptr); - EXPECT_EQ(found_index->index_id(), 1); - EXPECT_EQ(found_index->get_index_suffix(), "suffix1"); + auto found_indexs = _tablet_schema->inverted_indexs(100, "suffix1"); + ASSERT_FALSE(found_indexs.empty()); + EXPECT_EQ(found_indexs[0]->index_id(), 1); + EXPECT_EQ(found_indexs[0]->get_index_suffix(), "suffix1"); } TEST_F(TabletSchemaIndexTest, TestRemoveIndex) { @@ -90,28 +90,28 @@ TEST_F(TabletSchemaIndexTest, TestRemoveIndex) { _tablet_schema->remove_index(1); // Verify index 1 removed - EXPECT_EQ(_tablet_schema->inverted_index(100, "suffix1"), nullptr); + EXPECT_TRUE(_tablet_schema->inverted_indexs(100, "suffix1").empty()); // Verify index 2 still exists - auto* found_index = _tablet_schema->inverted_index(200, "suffix2"); - ASSERT_NE(found_index, nullptr); - EXPECT_EQ(found_index->index_id(), 2); + auto found_indexs = _tablet_schema->inverted_indexs(200, "suffix2"); + ASSERT_FALSE(found_indexs.empty()); + EXPECT_EQ(found_indexs[0]->index_id(), 2); } TEST_F(TabletSchemaIndexTest, TestUpdateIndex) { // Add initial index _tablet_schema->append_index(create_test_index(1, IndexType::INVERTED, {100}, "old_suffix")); - ASSERT_NE(_tablet_schema->inverted_index(100, "old_suffix"), nullptr); + ASSERT_FALSE(_tablet_schema->inverted_indexs(100, "old_suffix").empty()); // Update index with new suffix _tablet_schema->remove_index(1); _tablet_schema->append_index(create_test_index(1, IndexType::INVERTED, {100}, "new_suffix")); // Verify update - EXPECT_EQ(_tablet_schema->inverted_index(100, "old_suffix"), nullptr); - auto* found_index = _tablet_schema->inverted_index(100, "new_suffix"); - ASSERT_NE(found_index, nullptr); - EXPECT_EQ(found_index->get_index_suffix(), "new%5Fsuffix"); + EXPECT_TRUE(_tablet_schema->inverted_indexs(100, "old_suffix").empty()); + auto found_indexs = _tablet_schema->inverted_indexs(100, "new_suffix"); + ASSERT_FALSE(found_indexs.empty()); + EXPECT_EQ(found_indexs[0]->get_index_suffix(), "new%5Fsuffix"); } TEST_F(TabletSchemaIndexTest, TestMultipleColumnsIndex) { @@ -120,10 +120,11 @@ TEST_F(TabletSchemaIndexTest, TestMultipleColumnsIndex) { _tablet_schema->append_index(std::move(index)); // Verify both columns mapped - auto* index1 = _tablet_schema->inverted_index(100, "multi_col"); - auto* index2 = _tablet_schema->inverted_index(200, "multi_col"); - ASSERT_NE(index1, nullptr); - ASSERT_EQ(index1, index2); // Should point to same index + auto index1 = _tablet_schema->inverted_indexs(100, "multi_col"); + auto index2 = _tablet_schema->inverted_indexs(200, "multi_col"); + ASSERT_FALSE(index1.empty()); + ASSERT_FALSE(index2.empty()); + ASSERT_EQ(index1[0]->index_id(), index2[0]->index_id()); // Should point to same index } TEST_F(TabletSchemaIndexTest, TestDuplicateIndexKey) { @@ -132,16 +133,16 @@ TEST_F(TabletSchemaIndexTest, TestDuplicateIndexKey) { _tablet_schema->append_index(create_test_index(2, IndexType::INVERTED, {100}, "suffix")); // The last added should override - auto* found_index = _tablet_schema->inverted_index(100, "suffix"); - ASSERT_NE(found_index, nullptr); - EXPECT_EQ(found_index->index_id(), 1); + auto found_indexs = _tablet_schema->inverted_indexs(100, "suffix"); + ASSERT_FALSE(found_indexs.empty()); + EXPECT_EQ(found_indexs[0]->index_id(), 1); } TEST_F(TabletSchemaIndexTest, TestClearIndexes) { _tablet_schema->append_index(create_test_index(1, IndexType::INVERTED, {100})); _tablet_schema->clear_index(); - EXPECT_EQ(_tablet_schema->inverted_index(100, ""), nullptr); + EXPECT_TRUE(_tablet_schema->inverted_indexs(100, "").empty()); EXPECT_TRUE(_tablet_schema->inverted_indexes().empty()); } @@ -159,10 +160,10 @@ TEST_F(TabletSchemaIndexTest, TestUpdateIndexMethod) { _tablet_schema->update_index(col, IndexType::INVERTED, {std::move(new_index)}); - const TabletIndex* updated_index = _tablet_schema->inverted_index(100, "v2"); - ASSERT_NE(updated_index, nullptr); - EXPECT_EQ(updated_index->index_id(), 1); - EXPECT_EQ(updated_index->properties().at("new_prop"), "value"); + auto updated_indexs = _tablet_schema->inverted_indexs(100, "v2"); + ASSERT_FALSE(updated_indexs.empty()); + EXPECT_EQ(updated_indexs[0]->index_id(), 1); + EXPECT_EQ(updated_indexs[0]->properties().at("new_prop"), "value"); auto key = std::make_tuple(IndexType::INVERTED, 100, "v2"); EXPECT_NE(_tablet_schema->_col_id_suffix_to_index.find(key), @@ -177,8 +178,8 @@ TEST_F(TabletSchemaIndexTest, TestUpdateIndexAddNewWhenNotExist) { TabletIndex new_index = create_test_index(2, IndexType::INVERTED, {200}, "v3"); _tablet_schema->update_index(col, IndexType::INVERTED, {std::move(new_index)}); - const TabletIndex* index = _tablet_schema->inverted_index(200, "v3"); - ASSERT_EQ(index, nullptr); + auto indexs = _tablet_schema->inverted_indexs(200, "v3"); + ASSERT_TRUE(indexs.empty()); } TEST_F(TabletSchemaIndexTest, TestUpdateIndexWithMultipleColumns) { @@ -194,7 +195,7 @@ TEST_F(TabletSchemaIndexTest, TestUpdateIndexWithMultipleColumns) { TabletIndex new_multi_index = create_test_index(3, IndexType::NGRAM_BF, {300, 400}); _tablet_schema->append_index(std::move(new_multi_index)); - ASSERT_NE(_tablet_schema->inverted_index(300, "multi"), nullptr); + EXPECT_FALSE(_tablet_schema->inverted_indexs(300, "multi").empty()); EXPECT_NE(_tablet_schema->get_ngram_bf_index(400), nullptr); } diff --git a/be/test/vec/common/schema_util_test.cpp b/be/test/vec/common/schema_util_test.cpp index 5d5c535f715e24..3988ed1bb9a624 100644 --- a/be/test/vec/common/schema_util_test.cpp +++ b/be/test/vec/common/schema_util_test.cpp @@ -237,7 +237,7 @@ TEST_F(SchemaUtilTest, inherit_column_attributes) { construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_STRING, 1, "v1.b", &subcolumns); construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_INT, 1, "v1.c", &subcolumns); - construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_ARRAY, 3, "v3.d", &subcolumns); + construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_DOUBLE, 3, "v3.d", &subcolumns); construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_FLOAT, 3, "v3.a", &subcolumns); schema_util::inherit_column_attributes(tablet_schema); @@ -247,8 +247,7 @@ TEST_F(SchemaUtilTest, inherit_column_attributes) { EXPECT_EQ(tablet_schema->inverted_indexs(col).size(), 1); break; case 3: - // TODO(lihangyu): uncomment, since double is not supported in inverted index now - EXPECT_EQ(tablet_schema->inverted_indexs(col).size(), 0); + EXPECT_EQ(tablet_schema->inverted_indexs(col).size(), 1); break; default: EXPECT_TRUE(false); diff --git a/be/test/vec/function/function_is_null_test.cpp b/be/test/vec/function/function_is_null_test.cpp index fd41517ac4d295..2f1bead49e2a15 100644 --- a/be/test/vec/function/function_is_null_test.cpp +++ b/be/test/vec/function/function_is_null_test.cpp @@ -222,15 +222,17 @@ TEST_F(FunctionIsNullTest, gc_binlogs_test) { auto index_file_reader = std::make_shared( io::global_local_filesystem(), index_prefix, InvertedIndexStorageFormatPB::V2); EXPECT_TRUE(index_file_reader->init().ok()); - auto index_meta = _tablet_schema->inverted_index(0); - EXPECT_TRUE(index_meta); + auto index_metas = _tablet_schema->inverted_indexs(0); + EXPECT_FALSE(index_metas.empty()); + auto index_meta = index_metas[0]; auto bkd_reader = BkdIndexReader::create_shared(index_meta, index_file_reader); EXPECT_TRUE(bkd_reader); check_result(bkd_reader.get(), true, 1); check_result(bkd_reader.get(), false, 2); - auto index_meta2 = _tablet_schema->inverted_index(1); - EXPECT_TRUE(index_meta2); + auto index_metas2 = _tablet_schema->inverted_indexs(1); + EXPECT_FALSE(index_metas2.empty()); + auto index_meta2 = index_metas2[0]; auto string_reader = StringTypeInvertedIndexReader::create_shared(index_meta2, index_file_reader); EXPECT_TRUE(string_reader); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index c746f34563f364..09a0b4f8fb3490 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -126,7 +126,6 @@ public abstract class Type { public static final VariantType VARIANT = new VariantType(); public static final AnyType ANY_STRUCT_TYPE = new AnyStructType(); public static final AnyType ANY_ELEMENT_TYPE = new AnyElementType(); - private static final Map typeMap = new HashMap<>(); private static final Logger LOG = LogManager.getLogger(Type.class); private static final ArrayList integerTypes; @@ -140,42 +139,6 @@ public abstract class Type { private static final ArrayList variantSubTypes; private static final ArrayList trivialTypes; - static { - typeMap.put("TINYINT", Type.TINYINT); - typeMap.put("SMALLINT", Type.SMALLINT); - typeMap.put("INT", Type.INT); - typeMap.put("BIGINT", Type.BIGINT); - typeMap.put("LARGEINT", Type.LARGEINT); - typeMap.put("UNSIGNED_TINYINT", Type.UNSUPPORTED); - typeMap.put("UNSIGNED_SMALLINT", Type.UNSUPPORTED); - typeMap.put("UNSIGNED_INT", Type.UNSUPPORTED); - typeMap.put("UNSIGNED_BIGINT", Type.UNSUPPORTED); - typeMap.put("FLOAT", Type.FLOAT); - typeMap.put("DISCRETE_DOUBLE", Type.DOUBLE); - typeMap.put("DOUBLE", Type.DOUBLE); - typeMap.put("CHAR", Type.CHAR); - typeMap.put("DATE", Type.DATE); - typeMap.put("DATEV2", Type.DATEV2); - typeMap.put("DATETIMEV2", Type.DATETIMEV2); - typeMap.put("DATETIME", Type.DATETIME); - typeMap.put("DECIMAL32", Type.DECIMAL32); - typeMap.put("DECIMAL64", Type.DECIMAL64); - typeMap.put("DECIMAL128I", Type.DECIMAL128); - typeMap.put("DECIMAL", Type.DECIMALV2); - typeMap.put("VARCHAR", Type.VARCHAR); - typeMap.put("STRING", Type.STRING); - typeMap.put("JSONB", Type.JSONB); - typeMap.put("VARIANT", Type.VARIANT); - typeMap.put("BOOLEAN", Type.BOOLEAN); - typeMap.put("HLL", Type.HLL); - typeMap.put("STRUCT", Type.STRUCT); - typeMap.put("LIST", Type.UNSUPPORTED); - typeMap.put("MAP", Type.MAP); - typeMap.put("OBJECT", Type.UNSUPPORTED); - typeMap.put("ARRAY", Type.ARRAY); - typeMap.put("QUANTILE_STATE", Type.QUANTILE_STATE); - } - static { integerTypes = Lists.newArrayList(); integerTypes.add(TINYINT); @@ -2286,8 +2249,4 @@ public static boolean isSameDecimalTypeWithDifferentPrecision(int precision1, in } return false; } - - public static Type getTypeFromTypeName(String typeName) { - return typeMap.getOrDefault(typeName, Type.UNSUPPORTED); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 6270e04a56b0e4..7855413ea7190c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.DropIndexClause; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.analysis.ModifyColumnClause; import org.apache.doris.analysis.ModifyTablePropertiesClause; import org.apache.doris.analysis.ReorderColumnsClause; @@ -98,6 +99,7 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.ClearAlterTask; import org.apache.doris.task.UpdateTabletMetaInfoTask; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTaskType; @@ -1056,6 +1058,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP throw new DdlException("Not supporting alter table add generated columns."); } + if (newColumn.getType().isVariantType() && olapTable.hasVariantColumns()) { + checkAddVariantColumnAllowed(olapTable, newColumn); + } + /* * add new column to indexes. * UNIQUE: @@ -2724,6 +2730,9 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab indexDef.checkColumn(column, olapTable.getKeysType(), olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat()); + if (!InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()).isEmpty()) { + throw new DdlException("Can not create index with field pattern"); + } } else { throw new DdlException("index column does not exist in table. invalid column: " + col); } @@ -2752,9 +2761,28 @@ private boolean checkDuplicateIndexes(List indexes, IndexDef indexDef, Se Set existedIdxColSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); existedIdxColSet.addAll(index.getColumns()); if (index.getIndexType() == indexDef.getIndexType() && newColset.equals(existedIdxColSet)) { - throw new DdlException( + if (newColset.size() == 1 + && olapTable.getInvertedIndexFileStorageFormat() + .compareTo(TInvertedIndexFileStorageFormat.V2) >= 0) { + String columnName = indexDef.getColumns().get(0); + Column column = olapTable.getColumn(columnName); + if (column != null && (column.getType().isStringType() || column.getType().isVariantType())) { + boolean isExistingIndexAnalyzer = index.isAnalyzedInvertedIndex(); + boolean isNewIndexAnalyzer = indexDef.isAnalyzedInvertedIndex(); + if (isExistingIndexAnalyzer == isNewIndexAnalyzer) { + throw new DdlException( + indexDef.getIndexType() + " index for column (" + columnName + ") with " + + (isNewIndexAnalyzer ? "analyzed" : "non-analyzed") + " type already exists."); + } + } else { + throw new DdlException( + indexDef.getIndexType() + " index for column (" + columnName + ") already exists."); + } + } else { + throw new DdlException( indexDef.getIndexType() + " index for columns (" + String.join(",", indexDef.getColumns()) - + ") already exist."); + + ") already exist."); + } } existedIndexIdSet.add(index.getIndexId()); } @@ -2784,6 +2812,10 @@ private boolean processDropIndex(DropIndexClause alterClause, OlapTable olapTabl throw new DdlException("index " + indexName + " does not exist"); } + if (!InvertedIndexUtil.getInvertedIndexFieldPattern(found.getProperties()).isEmpty()) { + throw new DdlException("Can not drop index with field pattern"); + } + Iterator itr = indexes.iterator(); while (itr.hasNext()) { Index idx = itr.next(); @@ -3401,4 +3433,20 @@ private void checkOrder(List targetIndexSchema, List orderedColN nameSet.add(colName); } } + + private void checkAddVariantColumnAllowed(OlapTable olapTable, Column newColumn) throws DdlException { + int currentCount = newColumn.getVariantMaxSubcolumnsCount(); + for (Column column : olapTable.getBaseSchema()) { + if (column.getType().isVariantType()) { + if (currentCount == 0 && column.getVariantMaxSubcolumnsCount() != 0) { + throw new DdlException("The variant_max_subcolumns_count must either be 0 in all columns" + + " or greater than 0 in all columns"); + } + if (currentCount > 0 && column.getVariantMaxSubcolumnsCount() == 0) { + throw new DdlException("The variant_max_subcolumns_count must either be 0 in all columns" + + " or greater than 0 in all columns"); + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 5f7a45a14da554..701f3971bed383 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -310,4 +310,11 @@ public static void parseAndValidateProperty(Map properties, Stri throw new AnalysisException("Invalid value for '" + key + "': " + valueStr, e); } } + + public boolean isAnalyzedInvertedIndex() { + return indexType == IndexDef.IndexType.INVERTED + && properties != null + && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY) + || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java index 01b9f11c3ba591..f1e73c75ca8c55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java @@ -21,11 +21,14 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +69,8 @@ public class InvertedIndexUtil { public static String INVERTED_INDEX_CUSTOM_ANALYZER_KEY = "analyzer"; + public static String INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY = "field_pattern"; + public static String getInvertedIndexParser(Map properties) { String parser = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_KEY); // default is "none" if not set @@ -81,6 +86,22 @@ public static String getInvertedIndexParserMode(Map properties) INVERTED_INDEX_PARSER_COARSE_GRANULARITY; } + public static String getInvertedIndexFieldPattern(Map properties) { + String fieldPattern = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY); + // default is "none" if not set + return fieldPattern != null ? fieldPattern : ""; + } + + public static boolean getInvertedIndexSupportPhrase(Map properties) { + String supportPhrase = properties == null ? null : properties.get(INVERTED_INDEX_SUPPORT_PHRASE_KEY); + return supportPhrase != null ? Boolean.parseBoolean(supportPhrase) : true; + } + + public static String getCustomAnalyzer(Map properties) { + String customAnalyzer = properties == null ? null : properties.get(INVERTED_INDEX_CUSTOM_ANALYZER_KEY); + return customAnalyzer != null ? customAnalyzer : ""; + } + public static Map getInvertedIndexCharFilter(Map properties) { if (properties == null) { return new HashMap<>(); @@ -193,7 +214,8 @@ public static void checkInvertedIndexProperties(Map properties, INVERTED_INDEX_PARSER_LOWERCASE_KEY, INVERTED_INDEX_PARSER_STOPWORDS_KEY, INVERTED_INDEX_DICT_COMPRESSION_KEY, - INVERTED_INDEX_CUSTOM_ANALYZER_KEY + INVERTED_INDEX_CUSTOM_ANALYZER_KEY, + INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY )); for (String key : properties.keySet()) { @@ -311,4 +333,29 @@ public static void checkInvertedIndexProperties(Map properties, } } } + + public static boolean canHaveMultipleInvertedIndexes(DataType colType, List indexDefs) { + if (indexDefs.size() == 0 || indexDefs.size() == 1) { + return true; + } + if (!colType.isStringLikeType() && !colType.isVariantType()) { + return false; + } + if (indexDefs.size() > 2) { + return false; + } + boolean findParsedInvertedIndex = false; + boolean findNonParsedInvertedIndex = false; + for (IndexDefinition indexDef : indexDefs) { + if (indexDef.isAnalyzedInvertedIndex()) { + findParsedInvertedIndex = true; + } else { + findNonParsedInvertedIndex = true; + } + } + if (findParsedInvertedIndex && findNonParsedInvertedIndex) { + return true; + } + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 47acb268702dab..2ed52fbfff06e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -946,6 +946,18 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { if (generatedColumnInfo != null || other.getGeneratedColumnInfo() != null) { throw new DdlException("Not supporting alter table modify generated columns."); } + + if (type.isVariantType() && other.type.isVariantType()) { + if (this.getVariantMaxSubcolumnsCount() != other.getVariantMaxSubcolumnsCount()) { + throw new DdlException("Can not change variant max subcolumns count"); + } + if (this.getVariantEnableTypedPathsToSparse() != other.getVariantEnableTypedPathsToSparse()) { + throw new DdlException("Can not change variant enable typed paths to sparse"); + } + if (!this.getChildren().isEmpty() || !other.getChildren().isEmpty()) { + throw new DdlException("Can not change variant schema templates"); + } + } } public boolean nameEquals(String otherColName, boolean ignorePrefix) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 9d22cd5095dd8a..5bc891a40e1e81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -176,6 +176,14 @@ public String getInvertedIndexParserStopwords() { return InvertedIndexUtil.getInvertedIndexParserStopwords(properties); } + public String getInvertedIndexFieldPattern() { + return InvertedIndexUtil.getInvertedIndexFieldPattern(properties); + } + + public boolean getInvertedIndexSupportPhrase() { + return InvertedIndexUtil.getInvertedIndexSupportPhrase(properties); + } + // Whether the index can be changed in light mode public boolean isLightIndexChangeSupported() { return indexType == IndexDef.IndexType.INVERTED; @@ -363,4 +371,11 @@ public static void checkConflict(Collection indices, Set bloomFil bfColumns.add(column); } } + + public boolean isAnalyzedInvertedIndex() { + return indexType == IndexDef.IndexType.INVERTED + && properties != null + && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY) + || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index d0875f4968b5ce..bc0975473b1b59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -80,6 +80,7 @@ import org.apache.doris.thrift.TNodeInfo; import org.apache.doris.thrift.TOlapTable; import org.apache.doris.thrift.TPaloNodesInfo; +import org.apache.doris.thrift.TPatternType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TSortType; import org.apache.doris.thrift.TStorageFormat; @@ -3550,4 +3551,62 @@ protected void addIndexIdToMetaForUnitTest(long id, MaterializedIndexMeta meta) protected void addIndexNameToIdForUnitTest(String name, long id) { indexNameToId.put(name, id); } + + public Index getInvertedIndex(Column column, List subPath) { + List invertedIndexes = new ArrayList<>(); + for (Index index : indexes.getIndexes()) { + if (index.getIndexType() == IndexDef.IndexType.INVERTED) { + List columns = index.getColumns(); + if (columns != null && !columns.isEmpty() && column.getName().equals(columns.get(0))) { + invertedIndexes.add(index); + } + } + } + + if (subPath == null || subPath.isEmpty()) { + return invertedIndexes.size() == 1 ? invertedIndexes.get(0) + : invertedIndexes.stream().filter(Index::isAnalyzedInvertedIndex).findFirst().orElse(null); + } + + // subPath is not empty, means it is a variant column, find the field pattern from children + String subPathString = String.join(".", subPath); + String fieldPattern = ""; + for (Column child : column.getChildren()) { + String childName = child.getName(); + if (child.getFieldPatternType() == TPatternType.MATCH_NAME_GLOB) { + try { + java.nio.file.PathMatcher matcher = java.nio.file.FileSystems.getDefault() + .getPathMatcher("glob:" + childName); + if (matcher.matches(java.nio.file.Paths.get(subPathString))) { + fieldPattern = childName; + } + } catch (Exception e) { + continue; + } + } else if (child.getFieldPatternType() == TPatternType.MATCH_NAME) { + if (childName.equals(subPathString)) { + fieldPattern = childName; + } + } + } + + List invertedIndexesWithFieldPattern = new ArrayList<>(); + for (Index index : indexes.getIndexes()) { + if (index.getIndexType() == IndexDef.IndexType.INVERTED) { + List columns = index.getColumns(); + if (columns != null && !columns.isEmpty() && column.getName().equals(columns.get(0)) + && fieldPattern.equals(index.getInvertedIndexFieldPattern())) { + invertedIndexesWithFieldPattern.add(index); + } + } + } + if (invertedIndexesWithFieldPattern.isEmpty()) { + return invertedIndexes.size() == 1 ? invertedIndexes.get(0) + : invertedIndexes.stream().filter(Index::isAnalyzedInvertedIndex).findFirst().orElse(null); + } else { + return invertedIndexesWithFieldPattern.size() == 1 ? invertedIndexesWithFieldPattern.get(0) + : invertedIndexesWithFieldPattern.stream() + .filter(Index::isAnalyzedInvertedIndex).findFirst().orElse(null); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java index 00147207c143db..986a7b193ccdd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.Type; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,6 +70,59 @@ public FetchRemoteTabletSchemaUtil(List tablets) { this.tableColumns = Lists.newArrayList(); } + private static final Map typeMap = new HashMap<>(); + + static { + typeMap.put("TINYINT", Type.TINYINT); + typeMap.put("SMALLINT", Type.SMALLINT); + typeMap.put("INT", Type.INT); + typeMap.put("BIGINT", Type.BIGINT); + typeMap.put("LARGEINT", Type.LARGEINT); + typeMap.put("UNSIGNED_TINYINT", Type.UNSUPPORTED); + typeMap.put("UNSIGNED_SMALLINT", Type.UNSUPPORTED); + typeMap.put("UNSIGNED_INT", Type.UNSUPPORTED); + typeMap.put("UNSIGNED_BIGINT", Type.UNSUPPORTED); + typeMap.put("FLOAT", Type.FLOAT); + typeMap.put("DISCRETE_DOUBLE", Type.DOUBLE); + typeMap.put("DOUBLE", Type.DOUBLE); + typeMap.put("CHAR", Type.CHAR); + typeMap.put("DATE", Type.DATE); + typeMap.put("DATEV2", Type.DATEV2); + typeMap.put("DATETIMEV2", Type.DATETIMEV2); + typeMap.put("DATETIME", Type.DATETIME); + typeMap.put("DECIMAL32", Type.DECIMAL32); + typeMap.put("DECIMAL64", Type.DECIMAL64); + typeMap.put("DECIMAL128I", Type.DECIMAL128); + typeMap.put("DECIMAL", Type.DECIMALV2); + typeMap.put("VARCHAR", Type.VARCHAR); + typeMap.put("STRING", Type.STRING); + typeMap.put("JSONB", Type.JSONB); + typeMap.put("VARIANT", Type.VARIANT); + typeMap.put("BOOLEAN", Type.BOOLEAN); + typeMap.put("HLL", Type.HLL); + typeMap.put("STRUCT", Type.STRUCT); + typeMap.put("LIST", Type.UNSUPPORTED); + typeMap.put("MAP", Type.MAP); + typeMap.put("OBJECT", Type.UNSUPPORTED); + typeMap.put("ARRAY", Type.ARRAY); + typeMap.put("IPV4", Type.IPV4); + typeMap.put("IPV6", Type.IPV6); + typeMap.put("QUANTILE_STATE", Type.QUANTILE_STATE); + } + + public static Type getTypeFromTypeName(String typeName, int precision, int scale) { + Type res = typeMap.getOrDefault(typeName, Type.UNSUPPORTED); + if (res.isScalarType() && (res.isDecimalV3() || res.isDecimalV2())) { + // set precision and scale + res = ScalarType.createType(res.getPrimitiveType(), 0, precision, scale); + } + return res; + } + + public static Type getTypeFromTypeName(String typeName) { + return typeMap.getOrDefault(typeName, Type.UNSUPPORTED); + } + public List fetch() { // 1. Find which Backend (BE) servers the tablets are on Preconditions.checkNotNull(remoteTablets); @@ -179,7 +234,7 @@ public int compare(Column c1, Column c2) { private Column initColumnFromPB(ColumnPB column) throws AnalysisException { try { AggregateType aggType = AggregateType.getAggTypeFromAggName(column.getAggregation()); - Type type = Type.getTypeFromTypeName(column.getType()); + Type type = getTypeFromTypeName(column.getType(), column.getPrecision(), column.getFrac()); String columnName = column.getName(); boolean isKey = column.getIsKey(); boolean isNullable = column.getIsNullable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 0580cbc53cd027..4442fc25a52044 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -31,7 +31,6 @@ import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; import org.apache.doris.analysis.FunctionParams; -import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.LambdaFunctionCallExpr; import org.apache.doris.analysis.LambdaFunctionExpr; @@ -211,7 +210,6 @@ public Expr visitElementAt(ElementAt elementAt, PlanTranslatorContext context) { @Override public Expr visitMatch(Match match, PlanTranslatorContext context) { - Index invertedIndex = null; // Get the first slot from match's left expr SlotReference slot = match.getInputSlots().stream() .findFirst() @@ -229,18 +227,7 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) { throw new AnalysisException("SlotReference in Match failed to get OlapTable, SQL is " + match.toSql()); } - List indexes = olapTbl.getIndexes(); - if (indexes != null) { - for (Index index : indexes) { - if (index.getIndexType() == IndexDef.IndexType.INVERTED) { - List columns = index.getColumns(); - if (columns != null && !columns.isEmpty() && column.getName().equals(columns.get(0))) { - invertedIndex = index; - break; - } - } - } - } + Index invertedIndex = olapTbl.getInvertedIndex(column, slot.getSubPath()); MatchPredicate.Operator op = match.op(); MatchPredicate matchPredicate = new MatchPredicate(op, match.left().accept(this, context), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index af2fbcec4a93d9..14c5b7612dff55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -20,7 +20,8 @@ import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.DistributionDesc; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.analysis.KeysDesc; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.SlotRef; @@ -35,7 +36,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; -import org.apache.doris.common.Pair; import org.apache.doris.common.util.AutoBucketUtils; import org.apache.doris.common.util.GeneratedColumnUtil; import org.apache.doris.common.util.InternalDatabaseUtil; @@ -70,6 +70,8 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.VariantField; +import org.apache.doris.nereids.types.VariantType; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -141,6 +143,10 @@ public class CreateTableInfo { private PartitionDesc partitionDesc; private DistributionDesc distributionDesc; + // get when validate + private Map>> columnToIndexes = new HashMap<>(); + private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; + /** * constructor for create table */ @@ -340,17 +346,35 @@ public void validate(ConnectContext ctx) { Preconditions.checkState(!Strings.isNullOrEmpty(ctlName), "catalog name is null or empty"); Preconditions.checkState(!Strings.isNullOrEmpty(dbName), "database name is null or empty"); - //check datev1 and decimalv2 + //check datatype: datev1, decimalv2, variant + boolean allZero = false; + boolean allPositive = false; for (ColumnDefinition columnDef : columns) { String columnNameUpperCase = columnDef.getName().toUpperCase(); if (columnNameUpperCase.startsWith("__DORIS_")) { throw new AnalysisException( "Disable to create table column with name start with __DORIS_: " + columnNameUpperCase); } - if (columnDef.getType().isVariantType() && columnNameUpperCase.indexOf('.') != -1) { - throw new AnalysisException( + if (columnDef.getType().isVariantType()) { + if (columnNameUpperCase.indexOf('.') != -1) { + throw new AnalysisException( "Disable to create table of `VARIANT` type column named with a `.` character: " + columnNameUpperCase); + } + VariantType variantType = (VariantType) columnDef.getType(); + if (variantType.getVariantMaxSubcolumnsCount() == 0) { + allZero = true; + if (allPositive) { + throw new AnalysisException("The variant_max_subcolumns_count must either be 0" + + " in all columns, or greater than 0 in all columns"); + } + } else { + allPositive = true; + if (allZero) { + throw new AnalysisException("The variant_max_subcolumns_count must either be 0" + + " in all columns, or greater than 0 in all columns"); + } + } } if (columnDef.getType().isDateType() && Config.disable_datev1) { throw new AnalysisException( @@ -678,8 +702,6 @@ public void validate(ConnectContext ctx) { // validate index if (!indexes.isEmpty()) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - Set>> distinctCol = new HashSet<>(); - TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; try { invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( new HashMap<>(properties)); @@ -700,6 +722,9 @@ public void validate(ConnectContext ctx) { indexDef.checkColumn(column, keysType, isEnableMergeOnWrite, invertedIndexFileStorageFormat); found = true; + columnToIndexes.computeIfAbsent(column, k -> new HashMap<>()) + .computeIfAbsent(indexDef.getIndexType(), k -> new ArrayList<>()) + .add(indexDef); break; } } @@ -709,17 +734,12 @@ public void validate(ConnectContext ctx) { } } distinct.add(indexDef.getIndexName()); - distinctCol.add(Pair.of(indexDef.getIndexType(), indexDef.getColumnNames().stream() - .map(String::toUpperCase).collect(Collectors.toList()))); } if (distinct.size() != indexes.size()) { throw new AnalysisException("index name must be unique."); } - if (distinctCol.size() != indexes.size()) { - throw new AnalysisException( - "same index columns have multiple same type index is not allowed."); - } } + columnToIndexesCheck(); generatedColumnCheck(ctx); analyzeEngine(); } @@ -1215,4 +1235,89 @@ public List getRollupAlterClauseList() { public KeysDesc getKeysDesc() { return new KeysDesc(keysType, keys, clusterKeysColumnNames); } + + // 1. if the column is variant type, check it's field pattern is valid + // 2. if the column is not variant type, check it's index def is valid + private void columnToIndexesCheck() { + for (Map.Entry>> entry : columnToIndexes.entrySet()) { + ColumnDefinition column = entry.getKey(); + Map> indexTypeToIndexDefs = entry.getValue(); + for (Map.Entry> indexDefEntry : indexTypeToIndexDefs.entrySet()) { + IndexType indexType = indexDefEntry.getKey(); + List indexDefs = indexDefEntry.getValue(); + if (indexType != IndexType.INVERTED) { + if (indexDefs.size() > 1) { + throw new AnalysisException("column: " + column.getName() + + " cannot have multiple indexes, index type: " + indexType); + } else { + continue; + } + } + + // check inverted index + if (column.getType().isVariantType()) { + Map> fieldPatternToIndexDef = new HashMap<>(); + Map fieldPatternToDataType = new HashMap<>(); + for (IndexDefinition indexDef : indexDefs) { + String fieldPattern = InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()); + if (fieldPattern.isEmpty()) { + fieldPatternToIndexDef.computeIfAbsent(fieldPattern, k -> new ArrayList<>()).add(indexDef); + fieldPatternToDataType.put(fieldPattern, column.getType()); + continue; + } + boolean findFieldPattern = false; + VariantType variantType = (VariantType) column.getType(); + List predefinedFields = variantType.getPredefinedFields(); + for (VariantField field : predefinedFields) { + if (field.getPattern().equals(fieldPattern)) { + findFieldPattern = true; + if (!IndexDefinition.isSupportIdxType(field.getDataType())) { + throw new AnalysisException("field pattern: " + + fieldPattern + " is not supported for inverted index" + + " of column: " + column.getName()); + } + fieldPatternToIndexDef.computeIfAbsent(fieldPattern, k -> new ArrayList<>()) + .add(indexDef); + fieldPatternToDataType.put(fieldPattern, field.getDataType()); + break; + } + } + if (!findFieldPattern) { + throw new AnalysisException("can not find field pattern: " + fieldPattern + + " in column: " + column.getName()); + } + } + for (Map.Entry> fieldIndexEntry : fieldPatternToIndexDef.entrySet()) { + String fieldPattern = fieldIndexEntry.getKey(); + List fieldPatternIndexDefs = fieldIndexEntry.getValue(); + DataType dataType = fieldPatternToDataType.get(fieldPattern); + if (!InvertedIndexUtil.canHaveMultipleInvertedIndexes(dataType, fieldPatternIndexDefs)) { + throw new AnalysisException("column: " + + column.getName() + + " cannot have multiple inverted indexes with field pattern: " + + fieldPattern); + } + } + } else { + for (IndexDefinition indexDef : indexDefs) { + if (!InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()).isEmpty()) { + throw new AnalysisException("column: " + column.getName() + + " cannot have field pattern in index."); + } + } + if (!InvertedIndexUtil.canHaveMultipleInvertedIndexes(column.getType(), indexDefs)) { + throw new AnalysisException("column: " + column.getName() + + " cannot have multiple inverted indexes."); + } + if (invertedIndexFileStorageFormat != null + && invertedIndexFileStorageFormat.compareTo(TInvertedIndexFileStorageFormat.V2) < 0 + && indexDefs.size() > 1) { + throw new AnalysisException("column: " + column.getName() + + " cannot have multiple inverted indexes with file storage format: " + + invertedIndexFileStorageFormat); + } + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 0ec4c4cd58ea54..4899fc56c00963 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -112,7 +112,7 @@ public IndexDefinition(String name, PartitionNamesInfo partitionNames, IndexType /** * Check if the column type is supported for inverted index */ - public boolean isSupportIdxType(DataType columnType) { + public static boolean isSupportIdxType(DataType columnType) { if (columnType.isArrayType()) { DataType itemType = ((ArrayType) columnType).getItemType(); if (itemType.isArrayType()) { @@ -123,7 +123,7 @@ public boolean isSupportIdxType(DataType columnType) { return columnType.isDateLikeType() || columnType.isDecimalLikeType() || columnType.isIntegralType() || columnType.isStringLikeType() || columnType.isBooleanType() || columnType.isVariantType() - || columnType.isIPType(); + || columnType.isIPType() || columnType.isFloatLikeType(); } /** @@ -323,4 +323,15 @@ public String toSql(String tableName) { } return sb.toString(); } + + public Map getProperties() { + return properties; + } + + public boolean isAnalyzedInvertedIndex() { + return indexType == IndexType.INVERTED + && properties != null + && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY) + || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index e96b86e7fd5e4e..b97dbb50c769d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -791,7 +791,8 @@ public void testAddDuplicateInvertedIndexException() throws Exception { alterTable(addInvertedIndexStmtStr, connectContext); } catch (Exception e) { // Verify the error message contains relevant info - Assertions.assertTrue(e.getMessage().contains("INVERTED index for columns (error_msg) already exist")); + Assertions.assertTrue(e.getMessage().contains("INVERTED index for column (error_msg) " + + "with non-analyzed type already exists")); } addInvertedIndexStmtStr = "alter table test.sc_dup add index idx_error_msg(error_msg), " + "add index idx_error_msg(error_msg)"; diff --git a/regression-test/data/inverted_index_p0/test_single_column_multi_index.out b/regression-test/data/inverted_index_p0/test_single_column_multi_index.out new file mode 100644 index 00000000000000..97b211e6e3c3c0 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_single_column_multi_index.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +8630 + +-- !sql -- +990 + +-- !sql -- +8630 + +-- !sql -- +990 + diff --git a/regression-test/data/inverted_index_p0/test_single_column_multi_index1.out b/regression-test/data/inverted_index_p0/test_single_column_multi_index1.out new file mode 100644 index 00000000000000..4ed03cdac84e1b --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_single_column_multi_index1.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +297 + +-- !sql -- +2589 + +-- !sql -- +594 + +-- !sql -- +5178 + +-- !sql -- +594 + +-- !sql -- +5178 + +-- !sql -- +594 + +-- !sql -- +5178 + diff --git a/regression-test/data/variant_p0/predefine/test_predefine_ddl.out b/regression-test/data/variant_p0/predefine/test_predefine_ddl.out index c04e5a73f5acf0..617122ab75fbc9 100644 --- a/regression-test/data/variant_p0/predefine/test_predefine_ddl.out +++ b/regression-test/data/variant_p0/predefine/test_predefine_ddl.out @@ -1,5 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- id bigint Yes true \N -var variant No false \N NONE +var variant Yes false \N NONE + +-- !sql -- +id bigint Yes true \N +var variant Yes false \N NONE +var2 variant Yes false \N NONE +var3 variant Yes false \N NONE diff --git a/regression-test/suites/inverted_index_p0/test_float_double.groovy b/regression-test/suites/inverted_index_p0/test_float_double.groovy new file mode 100644 index 00000000000000..b8f46d0a17f45c --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_float_double.groovy @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_float_double", "p0, nonConcurrent"){ + def tableName = "test_float_double" + + sql """ set describe_extend_variant_column = true """ + sql """ set enable_match_without_inverted_index = false """ + sql """ set enable_common_expr_pushdown = true """ + sql """ set inverted_index_skip_threshold = 0 """ + + def queryAndCheck = { String sqlQuery, int expectedFilteredRows = -1, boolean checkFilterUsed = true -> + def checkpoints_name = "segment_iterator.inverted_index.filtered_rows" + try { + GetDebugPoint().enableDebugPointForAllBEs("segment_iterator.apply_inverted_index") + GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [filtered_rows: expectedFilteredRows]) + sql "set experimental_enable_parallel_scan = false" + sql "sync" + sql "${sqlQuery}" + } finally { + GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name) + GetDebugPoint().disableDebugPointForAllBEs("segment_iterator.apply_inverted_index") + } + } + + sql "DROP TABLE IF EXISTS ${tableName}" + + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NULL, + `float_col` float NULL, + `double_col` double NULL, + INDEX idx_float_col (float_col) USING INVERTED, + INDEX idx_double_col (double_col) USING INVERTED + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true") + """ + + sql """ insert into ${tableName} values (1, 1.5, 1.5239849328948), (2, 2.23, 2.239849328948), (3, 3.02, 3.029849328948) """ + + + queryAndCheck("select count() from ${tableName} where double_col = 1.5239849328948", 2) + queryAndCheck("select count() from ${tableName} where double_col = 2.239849328948", 2) + queryAndCheck("select count() from ${tableName} where double_col = 3.029849328948", 2) + + + queryAndCheck("select count() from ${tableName} where float_col = cast(1.5 as float)", 2) + queryAndCheck("select count() from ${tableName} where float_col = cast(2.23 as float)", 2) + queryAndCheck("select count() from ${tableName} where float_col = cast(3.02 as float)", 2) + +} diff --git a/regression-test/suites/inverted_index_p0/test_single_column_multi_index.groovy b/regression-test/suites/inverted_index_p0/test_single_column_multi_index.groovy new file mode 100644 index 00000000000000..bacd6b3f6ce0c5 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_single_column_multi_index.groovy @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_single_column_multi_index", "nonConcurrent") { + def isCloudMode = isCloudMode() + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) + + boolean disableAutoCompaction = false + + def tableName = "test_single_column_multi_index" + + // Function to create the test table + def createTestTable = { -> + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `@timestamp` int(11) NULL COMMENT "", + `clientip` varchar(20) NULL COMMENT "", + `request` text NULL COMMENT "", + `status` int(11) NULL COMMENT "", + `size` int(11) NULL COMMENT "", + INDEX request_keyword_idx (`request`) USING INVERTED COMMENT '', + INDEX request_text_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT "OLAP" + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + } + + def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false, + expected_succ_rows = -1, load_to_single_tablet = 'true' -> + + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'label', label + "_" + UUID.randomUUID().toString() + set 'read_json_by_line', read_flag + set 'format', format_flag + file file_name // import json file + time 10000 // limit inflight 10s + if (expected_succ_rows >= 0) { + set 'max_filter_ratio', '1' + } + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (ignore_failure && expected_succ_rows < 0) { return } + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + if (expected_succ_rows >= 0) { + assertEquals(json.NumberLoadedRows, expected_succ_rows) + } else { + assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + } + + def set_be_config = { key, value -> + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + + def check_config = { String key, String value -> + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == key) { + assertEquals(value, ((List) ele)[2]) + } + } + } + } + + def get_rowset_count = { tablets -> + int rowsetCount = 0 + for (def tablet in tablets) { + def (code, out, err) = curl("GET", tablet.CompactionStatus) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + rowsetCount +=((List) tabletJson.rowsets).size() + } + return rowsetCount + } + + def trigger_full_compaction_on_tablets = { tablets -> + for (def tablet : tablets) { + String tablet_id = tablet.TabletId + String backend_id = tablet.BackendId + int times = 1 + + String compactionStatus; + do{ + def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + ++times + sleep(2000) + compactionStatus = parseJson(out.trim()).status.toLowerCase(); + } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010") + + + if (compactionStatus == "fail") { + assertEquals(disableAutoCompaction, false) + logger.info("Compaction was done automatically!") + } + if (disableAutoCompaction && compactionStatus!="e-6010") { + assertEquals("success", compactionStatus) + } + } + } + + def wait_full_compaction_done = { tablets -> + for (def tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet.TabletId + String backend_id = tablet.BackendId + def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + } + + // Function to load test data + def loadTestData = { times = 10 -> + for (int i = 0; i < times; i++) { + load_httplogs_data.call(tableName, 'test_single_column_multi_index', 'true', 'json', 'documents-1000.json') + } + sql "sync" + } + + // Function to run match queries with debug points + def runMatchQueries = { -> + sql """ set enable_common_expr_pushdown = true; """ + sql """ set enable_common_expr_pushdown_for_inverted_index = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") + + try { + GetDebugPoint().enableDebugPointForAllBEs("inverted_index_reader._select_best_reader", [type: 0]) + try { + qt_sql """ select count() from ${tableName} where (request match 'images'); """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("inverted_index_reader._select_best_reader") + } + + GetDebugPoint().enableDebugPointForAllBEs("inverted_index_reader._select_best_reader", [type: 1]) + try { + qt_sql """ select count() from ${tableName} where (request = 'GET /images/hm_bg.jpg HTTP/1.0'); """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("inverted_index_reader._select_best_reader") + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") + } + } + + // Function to check and update BE config + def checkAndUpdateBeConfig = { -> + def invertedIndexCompactionEnable = false + def has_update_be_config = false + + String backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "inverted_index_compaction_enable") { + invertedIndexCompactionEnable = Boolean.parseBoolean(((List) ele)[2]) + logger.info("inverted_index_compaction_enable: ${((List) ele)[2]}") + } + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + logger.info("disable_auto_compaction: ${((List) ele)[2]}") + } + } + + return invertedIndexCompactionEnable + } + + // Main test execution + try { + sql """ set global enable_match_without_inverted_index = false """ + + createTestTable() + loadTestData() + runMatchQueries() + + def invertedIndexCompactionEnable = checkAndUpdateBeConfig() + + try { + set_be_config.call("inverted_index_compaction_enable", "true") + check_config.call("inverted_index_compaction_enable", "true") + + def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + int replicaNum = 1 + def dedup_tablets = deduplicate_tablets(tablets) + if (dedup_tablets.size() > 0) { + replicaNum = Math.round(tablets.size() / dedup_tablets.size()) + if (replicaNum != 1 && replicaNum != 3) { + assert(false) + } + } + + // Verify rowset count before compaction + int rowsetCount = get_rowset_count.call(tablets) + assert (rowsetCount == 11 * replicaNum) + + // Run compaction + trigger_full_compaction_on_tablets.call(tablets) + wait_full_compaction_done.call(tablets) + + // Verify rowset count after compaction + rowsetCount = get_rowset_count.call(tablets) + if (isCloudMode) { + assert (rowsetCount == (1 + 1) * replicaNum) + } else { + assert (rowsetCount == 1 * replicaNum) + } + + runMatchQueries() + } finally { + set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString()) + } + } finally { + sql """ set global enable_match_without_inverted_index = true """ + } +} \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_single_column_multi_index1.groovy b/regression-test/suites/inverted_index_p0/test_single_column_multi_index1.groovy new file mode 100644 index 00000000000000..b07f168e1f6c74 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_single_column_multi_index1.groovy @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_single_column_multi_index1", "p0") { + def tableName = "test_single_column_multi_index1" + + // Function to create the test table + def createTestTable = { -> + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `@timestamp` int(11) NULL COMMENT "", + `clientip` varchar(20) NULL COMMENT "", + `request` text NULL COMMENT "", + `status` int(11) NULL COMMENT "", + `size` int(11) NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT "OLAP" + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ) + """ + } + + def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false, + expected_succ_rows = -1, load_to_single_tablet = 'true' -> + + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'label', label + "_" + UUID.randomUUID().toString() + set 'read_json_by_line', read_flag + set 'format', format_flag + file file_name // import json file + time 10000 // limit inflight 10s + if (expected_succ_rows >= 0) { + set 'max_filter_ratio', '1' + } + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (ignore_failure && expected_succ_rows < 0) { return } + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + if (expected_succ_rows >= 0) { + assertEquals(json.NumberLoadedRows, expected_succ_rows) + } else { + assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + } + + // Function to load test data + def loadTestData = { times = 3 -> + for (int i = 0; i < times; i++) { + load_httplogs_data.call(tableName, 'test_single_column_multi_index', 'true', 'json', 'documents-1000.json') + } + sql "sync" + } + + // Function to run match queries with debug points + def runMatchQueries = { -> + sql """ set enable_common_expr_pushdown = true; """ + sql """ set enable_common_expr_pushdown_for_inverted_index = true; """ + try { + qt_sql """ select /*+ SET_VAR(enable_match_without_inverted_index = true) */ count() from ${tableName} where (request = 'GET /images/hm_bg.jpg HTTP/1.0'); """ + qt_sql """ select /*+ SET_VAR(enable_match_without_inverted_index = true) */ count() from ${tableName} where (request match 'images'); """ + } finally { + } + } + + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + def expected_finished_num = alter_res.size(); + def finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + + try { + createTestTable() + loadTestData() + runMatchQueries() + + sql """ alter table ${tableName} add index request_text_idx(`request`) USING INVERTED PROPERTIES("support_phrase" = "true", "parser" = "unicode", "lower_case" = "true"); """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ alter table ${tableName} add index request_keyword_idx(`request`) USING INVERTED;; """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + loadTestData() + runMatchQueries() + + + if (!isCloudMode()) { + sql """ BUILD INDEX request_text_idx ON ${tableName}; """ + wait_for_build_index_on_partition_finish(tableName, timeout) + } + + if (!isCloudMode()) { + sql """ BUILD INDEX request_keyword_idx ON ${tableName}; """ + wait_for_build_index_on_partition_finish(tableName, timeout) + } + + runMatchQueries() + + sql """ DROP INDEX request_text_idx ON ${tableName}; """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX request_keyword_idx ON ${tableName}; """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + runMatchQueries() + } finally { + } +} \ No newline at end of file diff --git a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy index d386312b79ced4..a2367a2e89ea59 100644 --- a/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy +++ b/regression-test/suites/variant_p0/predefine/test_predefine_ddl.groovy @@ -16,23 +16,353 @@ // under the License. suite("test_predefine_ddl", "p0") { + + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(tableName + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + def tableName = "test_ddl_table" + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : string, + MATCH_NAME '*cc' : string, + MATCH_NAME 'b?b' : string + > NOT NULL, + INDEX idx_a_b (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_bb (var) USING INVERTED PROPERTIES("field_pattern"="*cc", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_b_b (var) USING INVERTED PROPERTIES("field_pattern"="b?b", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_bb_glob (var) USING INVERTED PROPERTIES("field_pattern"="bb*", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_bx_glob (var) USING INVERTED PROPERTIES("field_pattern"="bx?", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("can not find field pattern: bb* in column: var") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : string, + MATCH_NAME '*cc' : string, + MATCH_NAME 'b?b' : string + > NOT NULL, + INDEX idx_a_b (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant NOT NULL, + INDEX idx_a_b (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("can not find field pattern: ab in column: var") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + + test { + sql """ create index idx_ab on ${tableName} (var) using inverted properties("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") """ + exception("Can not create index with field pattern") + } + + test { + sql """ create index idx_ab on ${tableName} (var) using inverted properties("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") """ + exception("Can not create index with field pattern") + } + + sql """ alter table ${tableName} add column var2 variant<'ab' : string, properties("variant_max_subcolumns_count" = "5")> NULL """ + + test { + sql """ alter table ${tableName} add column var3 variant<'ab' : string, properties("variant_max_subcolumns_count" = "0")> NULL """ + exception("The variant_max_subcolumns_count must either be 0 in all columns or greater than 0 in all columns") + } + + test { + sql """ alter table ${tableName} modify column var variant<'ab' : string, properties("variant_max_subcolumns_count" = "10")> NULL """ + exception("Can not change variant schema templates") + } + + sql "DROP TABLE IF EXISTS ${tableName}" sql """CREATE TABLE ${tableName} ( `id` bigint NULL, `var` variant< MATCH_NAME 'ab' : string, MATCH_NAME '*cc' : string, - MATCH_NAME_GLOB 'b?b' : string, - PROPERTIES("variant_max_subcolumns_count" = "10", "variant_enable_typed_paths_to_sparse" = "true") - > NOT NULL + MATCH_NAME 'b?b' : string + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + + test { + sql """ alter table ${tableName} modify column var variant NULL """ + exception("Can not change variant schema templates") + } + + test { + sql """ alter table ${tableName} drop index idx_ab """ + exception("Can not drop index with field pattern") + } + + sql """ alter table ${tableName} drop column var """ + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : json + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: json") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : int, + MATCH_NAME 'ab' : string, + properties("variant_max_subcolumns_count" = "10", "variant_enable_typed_paths_to_sparse" = "true") + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("""Duplicate field name ab in variant variant""") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : decimalv2(22, 2) + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: decimalv2(22,2)") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : datev1 + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: date") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : datetimev1 + > NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("VARIANT unsupported sub-type: datetime") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : double + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : int + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("column: var cannot have multiple inverted indexes with field pattern: ab") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : string + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : string + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("column: var cannot have multiple inverted indexes with field pattern: ab") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : string + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("column: var cannot have multiple inverted indexes with field pattern: ab") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant< + MATCH_NAME 'ab' : array + > NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("field_pattern"="ab", "parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED PROPERTIES("field_pattern"="ab") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true")""" + exception("column: var cannot have multiple inverted indexes with field pattern: ab") + } + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` string NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("parser"="unicode", "support_phrase" = "true") COMMENT '', + INDEX idx_ab_2 (var) USING INVERTED + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "v1")""" + exception("column: var cannot have multiple inverted indexes with file storage format: V1") + } + + test { + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant <'c' :char(10)> NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "v1")""" + exception("VARIANT unsupported sub-type: char(10)") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant <'c' :text, properties("variant_max_subcolumns_count" = "10")> NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" + + + test { + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var1` variant NULL, + `var2` variant NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" + exception("The variant_max_subcolumns_count must either be 0 in all columns, or greater than 0 in all columns") + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql "set default_variant_max_subcolumns_count = 10" + sql "set default_variant_enable_typed_paths_to_sparse = false" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant NULL + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" + + qt_sql "desc ${tableName}" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """CREATE TABLE ${tableName} ( + `id` bigint NULL, + `var` variant NULL, + INDEX idx_ab (var) USING INVERTED PROPERTIES("parser"="unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) + BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1")""" + + sql "create index idx_ab2 on ${tableName} (var) using inverted" + wait_for_latest_op_on_table_finish("${tableName}", timeout) + + sql """alter table ${tableName} add column var2 variant NULL""" + wait_for_latest_op_on_table_finish("${tableName}", timeout) + + test { + sql """alter table ${tableName} add column var3 variant NULL""" + exception("The variant_max_subcolumns_count must either be 0 in all columns or greater than 0 in all columns") + } + + sql "alter table ${tableName} add column var3 variant NULL" + wait_for_latest_op_on_table_finish("${tableName}", timeout) - sql """ insert into ${tableName} values (1, '{"ab": "1", "cc": "2", "b?b": "3"}') """ + qt_sql "desc ${tableName}" - qt_sql """ desc ${tableName} """ + sql "create index idx_ab3 on ${tableName} (var2) using inverted" + wait_for_latest_op_on_table_finish("${tableName}", timeout) + sql "create index idx_ab4 on ${tableName} (var2) using inverted properties(\"parser\"=\"unicode\")" + wait_for_latest_op_on_table_finish("${tableName}", timeout) } \ No newline at end of file