Skip to content

[enhance](inverted index) support for multi indexes #54406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldenmoon merged 8 commits into from
Aug 11, 2025
Merged

[enhance](inverted index) support for multi indexes #54406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e4477df
[enhance](inverted index) support for multi indexes
csun5285 Aug 6, 2025
bea9ed8
part
csun5285 Aug 6, 2025
19e0358
fix test
csun5285 Aug 6, 2025
0d6359e
fix test
csun5285 Aug 7, 2025
1f4be8f
fix case
csun5285 Aug 7, 2025
445f0de
fix
csun5285 Aug 8, 2025
bc0a8aa
fix
csun5285 Aug 8, 2025
1686e95
fix
csun5285 Aug 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 19 additions & 15 deletions be/src/olap/collection_statistics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,21 +91,25 @@ Status CollectionStatistics::extract_collect_info(
static_cast<vectorized::VLiteral*>(expr->children()[1].get());
auto column_idx = tablet_schema->field_index(left_slot_ref->column_name());
auto column = tablet_schema->column(column_idx);
const auto* index_meta = tablet_schema->inverted_index(column);

auto term_infos = InvertedIndexAnalyzer::get_analyse_result(
right_slot_ref->value(), index_meta->properties());

std::string field_name = std::to_string(column.unique_id());
std::wstring ws_field_name = StringHelper::to_wstring(field_name);
auto iter = collect_infos->find(ws_field_name);
if (iter == collect_infos->end()) {
CollectInfo collect_info;
collect_info.term_infos.insert(term_infos.begin(), term_infos.end());
collect_info.index_meta = index_meta;
(*collect_infos)[ws_field_name] = std::move(collect_info);
} else {
iter->second.term_infos.insert(term_infos.begin(), term_infos.end());
auto index_metas = tablet_schema->inverted_indexs(column);
for (const auto* index_meta : index_metas) {
if (!InvertedIndexAnalyzer::should_analyzer(index_meta->properties())) {
continue;
}
auto term_infos = InvertedIndexAnalyzer::get_analyse_result(
right_slot_ref->value(), index_meta->properties());

std::string field_name = std::to_string(column.unique_id());
std::wstring ws_field_name = StringHelper::to_wstring(field_name);
auto iter = collect_infos->find(ws_field_name);
if (iter == collect_infos->end()) {
CollectInfo collect_info;
collect_info.term_infos.insert(term_infos.begin(), term_infos.end());
collect_info.index_meta = index_meta;
(*collect_infos)[ws_field_name] = std::move(collect_info);
} else {
iter->second.term_infos.insert(term_infos.begin(), term_infos.end());
}
}
}

Expand Down
300 changes: 158 additions & 142 deletions be/src/olap/compaction.cpp
View file
Open in desktop

Large diffs are not rendered by default.

15 changes: 12 additions & 3 deletions be/src/olap/comparison_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,17 @@ class ComparisonPredicateBase : public ColumnPredicate {
IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override {
if (iterator == nullptr) {
return Status::OK();
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, no inverted index reader can not support "
"comparison predicate");
}

if (iterator->get_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE) == nullptr &&
iterator->get_reader(segment_v2::InvertedIndexReaderType::BKD) == nullptr) {
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, no inverted index reader can not support "
"comparison predicate");
}
std::string column_name = name_with_type.first;

InvertedIndexQueryType query_type = InvertedIndexQueryType::UNKNOWN_QUERY;
switch (PT) {
Expand Down Expand Up @@ -104,7 +112,8 @@ class ComparisonPredicateBase : public ColumnPredicate {
InvertedIndexQueryParamFactory::create_query_value<Type>(&_value, query_param));

InvertedIndexParam param;
param.column_name = column_name;
param.column_name = name_with_type.first;
param.column_type = name_with_type.second;
param.query_value = query_param->get_value();
param.query_type = query_type;
param.num_rows = num_rows;
Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/delta_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,9 +253,9 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(const PNodeInfo& node_info)
auto cur_rowset = _rowset_builder->rowset();
auto tablet_schema = cur_rowset->rowset_meta()->tablet_schema();
if (!tablet_schema->skip_write_index_on_load()) {
for (auto& column : tablet_schema->columns()) {
const TabletIndex* index_meta = tablet_schema->inverted_index(*column);
if (index_meta) {
for (const auto& column : tablet_schema->columns()) {
auto index_metas = tablet_schema->inverted_indexs(*column);
for (const auto* index_meta : index_metas) {
indices_ids.emplace_back(index_meta->index_id(), index_meta->get_index_suffix());
}
}
Expand Down
20 changes: 15 additions & 5 deletions be/src/olap/in_list_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,21 +184,31 @@ class InListPredicateBase : public ColumnPredicate {
IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* result) const override {
if (iterator == nullptr) {
return Status::OK();
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, no inverted index reader can not support "
"in_list");
}
// only string type and bkd inverted index reader can be used for in
if (iterator->get_reader(segment_v2::InvertedIndexReaderType::STRING_TYPE) == nullptr &&
iterator->get_reader(segment_v2::InvertedIndexReaderType::BKD) == nullptr) {
//NOT support in list when parser is FULLTEXT for expr inverted index evaluate.
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, no inverted index reader can not support "
"in_list");
}
std::string column_name = name_with_type.first;
roaring::Roaring indices;
HybridSetBase::IteratorBase* iter = _values->begin();
while (iter->has_next()) {
const void* ptr = iter->get_value();
// auto&& value = PrimitiveTypeConvertor<Type>::to_storage_field_type(
// *reinterpret_cast<const T*>(ptr));
std::unique_ptr<InvertedIndexQueryParamFactory> query_param = nullptr;
RETURN_IF_ERROR(
InvertedIndexQueryParamFactory::create_query_value<Type>(ptr, query_param));
RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value<Type>((const T*)ptr,
query_param));
InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY;
InvertedIndexParam param;
param.column_name = column_name;
param.column_name = name_with_type.first;
param.column_type = name_with_type.second;
param.query_value = query_param->get_value();
param.query_type = query_type;
param.num_rows = num_rows;
Expand Down
60 changes: 26 additions & 34 deletions be/src/olap/rowset/beta_rowset.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,8 @@ void BetaRowset::clear_inverted_index_cache() {

auto index_path_prefix = InvertedIndexDescriptor::get_index_file_path_prefix(*seg_path);
for (const auto& column : tablet_schema()->columns()) {
const TabletIndex* index_meta = tablet_schema()->inverted_index(*column);
if (index_meta) {
auto index_metas = tablet_schema()->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
auto inverted_index_file_cache_key =
InvertedIndexDescriptor::get_index_file_cache_key(
index_path_prefix, index_meta->index_id(),
Expand Down Expand Up @@ -240,9 +240,9 @@ Status BetaRowset::remove() {
}

if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
for (const auto& column : _schema->columns()) {
auto index_metas = _schema->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
Expand Down Expand Up @@ -414,10 +414,10 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row
auto src_path = local_segment_path(_tablet_path, rowset_id().to_string(), i);
RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(src_path, dst_path));
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
for (const auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
auto index_metas = _schema->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
std::string inverted_index_src_file_path =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(src_path),
Expand Down Expand Up @@ -473,10 +473,10 @@ Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new
dest_paths.emplace_back(remote_seg_path);
local_paths.emplace_back(local_seg_path);
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
for (const auto& column : _schema->columns()) {
// if (column.has_inverted_index()) {
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
auto index_metas = _schema->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
std::string remote_inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(
Expand Down Expand Up @@ -682,9 +682,9 @@ Status BetaRowset::calc_file_crc(uint32_t* crc_value, int64_t* file_count) {
auto seg_path = DORIS_TRY(segment_path(seg_id));
file_paths.emplace_back(seg_path);
if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) {
for (auto& column : _schema->columns()) {
const TabletIndex* index_meta = _schema->inverted_index(*column);
if (index_meta) {
for (const auto& column : _schema->columns()) {
auto index_metas = _schema->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
std::string inverted_index_file =
InvertedIndexDescriptor::get_index_file_path_v1(
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path),
Expand Down Expand Up @@ -841,26 +841,18 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value,
} else {
rapidjson::Value indices(rapidjson::kArrayType);
for (auto column : _rowset_meta->tablet_schema()->columns()) {
const auto* index_meta = _rowset_meta->tablet_schema()->inverted_index(*column);
if (index_meta == nullptr) {
continue;
}
rapidjson::Value index(rapidjson::kObjectType);
auto index_id = index_meta->index_id();
auto index_suffix = index_meta->get_index_suffix();
index.AddMember("index_id", rapidjson::Value(index_id).Move(), allocator);
index.AddMember("index_suffix", rapidjson::Value(index_suffix.c_str(), allocator),
allocator);
auto path = InvertedIndexDescriptor::get_index_file_path_v1(index_file_path_prefix,
index_id, index_suffix);
auto st = add_file_info_to_json(path, index);
if (!st.ok()) {
return st;
}

auto status = process_files(*index_meta, indices, index);
if (!status.ok()) {
return status;
auto index_metas = _rowset_meta->tablet_schema()->inverted_indexs(*column);
for (const auto& index_meta : index_metas) {
rapidjson::Value index(rapidjson::kObjectType);
auto index_id = index_meta->index_id();
auto index_suffix = index_meta->get_index_suffix();
index.AddMember("index_id", rapidjson::Value(index_id).Move(), allocator);
index.AddMember("index_suffix",
rapidjson::Value(index_suffix.c_str(), allocator), allocator);
auto path = InvertedIndexDescriptor::get_index_file_path_v1(
index_file_path_prefix, index_id, index_suffix);
RETURN_IF_ERROR(add_file_info_to_json(path, index));
RETURN_IF_ERROR(process_files(*index_meta, indices, index));
}
}
segment.AddMember("indices", indices, allocator);
Expand Down
4 changes: 2 additions & 2 deletions be/src/olap/rowset/beta_rowset_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -566,8 +566,8 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u
}
// rename remaining inverted index files
for (auto column : _context.tablet_schema->columns()) {
if (const auto& index_info = _context.tablet_schema->inverted_index(*column);
index_info != nullptr) {
auto index_infos = _context.tablet_schema->inverted_indexs(*column);
for (const auto& index_info : index_infos) {
auto index_id = index_info->index_id();
if (_context.tablet_schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/rowset/segcompaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
}
// Delete inverted index files
for (auto&& column : schema->columns()) {
if (const auto* index_info = schema->inverted_index(*column); index_info != nullptr) {
auto index_infos = schema->inverted_indexs(*column);
for (const auto& index_info : index_infos) {
auto index_id = index_info->index_id();
if (schema->get_inverted_index_storage_format() ==
InvertedIndexStorageFormatPB::V1) {
Expand Down
31 changes: 19 additions & 12 deletions be/src/olap/rowset/segment_v2/column_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,11 @@ Status ColumnReader::new_index_iterator(const std::shared_ptr<IndexFileReader>&
RETURN_IF_ERROR(_load_index(index_file_reader, index_meta));
{
std::shared_lock<std::shared_mutex> rlock(_load_index_lock);
if (_index_reader) {
RETURN_IF_ERROR(_index_reader->new_iterator(iterator));
auto iter = _index_readers.find(index_meta->index_id());
if (iter != _index_readers.end()) {
if (iter->second != nullptr) {
RETURN_IF_ERROR(iter->second->new_iterator(iterator));
}
}
}
return Status::OK();
Expand Down Expand Up @@ -661,8 +664,13 @@ Status ColumnReader::_load_index(const std::shared_ptr<IndexFileReader>& index_f
const TabletIndex* index_meta) {
std::unique_lock<std::shared_mutex> wlock(_load_index_lock);

if (_index_reader != nullptr && index_meta &&
_index_reader->get_index_id() == index_meta->index_id()) {
if (index_meta == nullptr) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"Failed to load inverted index: index metadata is null");
}

auto it = _index_readers.find(index_meta->index_id());
if (it != _index_readers.end()) {
return Status::OK();
}

Expand All @@ -676,17 +684,18 @@ Status ColumnReader::_load_index(const std::shared_ptr<IndexFileReader>& index_f
type = _type_info->type();
}

IndexReaderPtr index_reader;
if (is_string_type(type)) {
if (should_analyzer) {
try {
_index_reader = FullTextIndexReader::create_shared(index_meta, index_file_reader);
index_reader = FullTextIndexReader::create_shared(index_meta, index_file_reader);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"create FullTextIndexReader error: {}", e.what());
}
} else {
try {
_index_reader =
index_reader =
StringTypeInvertedIndexReader::create_shared(index_meta, index_file_reader);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
Expand All @@ -695,18 +704,16 @@ Status ColumnReader::_load_index(const std::shared_ptr<IndexFileReader>& index_f
}
} else if (is_numeric_type(type)) {
try {
_index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"create BkdIndexReader error: {}", e.what());
}
} else {
_index_reader.reset();
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"Field type {} is not supported for inverted index", type);
}
// TODO: move has null to inverted_index_reader's query function
//bool has_null = true;
//RETURN_IF_ERROR(index_file_reader->has_null(index_meta, &has_null));
//_inverted_index->set_has_null(has_null);
_index_readers[index_meta->index_id()] = index_reader;
return Status::OK();
}

Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/column_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ class ColumnReader : public MetadataAdder<ColumnReader> {
std::unique_ptr<BitmapIndexReader> _bitmap_index;
std::shared_ptr<BloomFilterIndexReader> _bloom_filter_index;

IndexReaderPtr _index_reader;
std::unordered_map<int64_t, IndexReaderPtr> _index_readers;

std::vector<std::unique_ptr<ColumnReader>> _sub_readers;

Expand Down
Loading
Loading