diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 02f24454ef5b..8c4d4833072f 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -163,6 +163,7 @@ static void splitAndModifyMutationCommands( NameSet mutated_columns; NameSet dropped_columns; NameSet ignored_columns; + NameSet extra_columns_for_indices_and_projections; for (const auto & command : commands) { @@ -212,6 +213,40 @@ static void splitAndModifyMutationCommands( ignored_columns.emplace(col.name); } } + if (command.type == MutationCommand::Type::MATERIALIZE_INDEX) + { + const auto & all_indices = metadata_snapshot->getSecondaryIndices(); + for (const auto & index : all_indices) + { + if (index.name == command.index_name) + { + auto required_columns = index.expression->getRequiredColumns(); + for (const auto & column : required_columns) + { + if (!part_columns.has(column)) + extra_columns_for_indices_and_projections.insert(column); + } + break; + } + } + } + + if (command.type == MutationCommand::Type::MATERIALIZE_PROJECTION) + { + const auto & all_projections = metadata_snapshot->getProjections(); + for (const auto & projection : all_projections) + { + if (projection.name == command.projection_name) + { + for (const auto & column : projection.required_columns) + { + if (!part_columns.has(column)) + extra_columns_for_indices_and_projections.insert(column); + } + break; + } + } + } } else if (command.type == MutationCommand::Type::DROP_INDEX || command.type == MutationCommand::Type::DROP_PROJECTION @@ -313,6 +348,27 @@ static void splitAndModifyMutationCommands( }); } } + for (const auto & column_name : extra_columns_for_indices_and_projections) + { + if (mutated_columns.contains(column_name)) + continue; + + if (column_name == "_part_offset") + continue; + + auto data_type = metadata_snapshot->getColumns().getColumn( + GetColumnsOptions::AllPhysical, + column_name).type; + + for_interpreter.push_back( + MutationCommand + { + .type = MutationCommand::Type::READ_COLUMN, + .column_name = column_name, + .data_type = std::move(data_type), + } + ); + } } else { @@ -1567,7 +1623,8 @@ class MutateAllPartColumnsTask : public IExecutableTask renamed_stats[STATS_FILE_PREFIX + command.column_name + STATS_FILE_SUFFIX] = STATS_FILE_PREFIX + command.rename_to + STATS_FILE_SUFFIX; } - bool is_full_part_storage = isFullPartStorage(ctx->new_data_part->getDataPartStorage()); + bool is_full_part_storage = isFullPartStorage(ctx->source_part->getDataPartStorage()); + bool is_full_wide_part = is_full_part_storage && isWidePart(ctx->new_data_part); const auto & indices = ctx->metadata_snapshot->getSecondaryIndices(); MergeTreeIndices skip_indices; @@ -1576,9 +1633,12 @@ class MutateAllPartColumnsTask : public IExecutableTask if (removed_indices.contains(idx.name)) continue; + /// For packed part we need to recalculate all indices because they are stored inside packed parts format + /// For compact parts we need to recalculate indices because rewrite of compact part may produce a little bit different data part + /// with different number of marks. bool need_recalculate = ctx->materialized_indices.contains(idx.name) - || (!is_full_part_storage && ctx->source_part->hasSecondaryIndex(idx.name)); + || (!is_full_wide_part && ctx->source_part->hasSecondaryIndex(idx.name)); if (need_recalculate) { diff --git a/tests/queries/0_stateless/03755_ttl_materialize_bug.reference b/tests/queries/0_stateless/03755_ttl_materialize_bug.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/03755_ttl_materialize_bug.sh b/tests/queries/0_stateless/03755_ttl_materialize_bug.sh new file mode 100755 index 000000000000..9afd8db37f16 --- /dev/null +++ b/tests/queries/0_stateless/03755_ttl_materialize_bug.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS index_corruption" + +${CLICKHOUSE_CLIENT} -n -q "CREATE TABLE IF NOT EXISTS index_corruption +( + 1F740 LowCardinality(String) CODEC(ZSTD(1)), + B7E44AC String CODEC(ZSTD(1)), + 3F0ED7DB LowCardinality(String) CODEC(ZSTD(1)), + CCD44A405D Map(LowCardinality(String), String) CODEC(ZSTD(1)), + 38C3868AC DateTime64(9) CODEC(Delta(8), ZSTD(1)), + ABC0C7B_EA Nullable(String) MATERIALIZED CCD44A405D['ABC0C7B_EA'], + 7AF String MATERIALIZED CCD44A405D['7AF'], + 21DB_EA String MATERIALIZED CCD44A405D['21DB_EA'], + 4482745CA_EF39713558 Nullable(String) MATERIALIZED CCD44A405D['4482745CA.EF39713558'], + ABC0C7B_938F19 LowCardinality(String) MATERIALIZED CCD44A405D['B7E44AC_618D.6726ED6AA465A'], + CCC98F_2221 String MATERIALIZED CCD44A405D['CCC98F_2221'], + INDEX idx_836_ADE2_72AB mapValues(CCD44A405D) TYPE bloom_filter(0.05) GRANULARITY 1, + INDEX idx_0989 tokens(lower(B7E44AC)) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_7AF 7AF TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree() +ORDER BY (toStartOfMinute(38C3868AC), toUnixTimestamp(38C3868AC)) +SETTINGS min_rows_for_wide_part = 10000, materialize_ttl_recalculate_only = 1, index_granularity = 8192;" + +# This data file contains random data which lead to index marks mismatch after mutation. It happens +# because during mutation we rewrite compact part completely. And surprisingly it may lead to a little bit different +# representation of data on disk, because we can add small last block to the last mark instead of adding new one. +# So rewritten data part may have 1 less mark, than original one. However skip indices were not rewritten during mutation, +# so they still contain marks for original data part. And when we try to read mutated part with skip indices, +# we can "Too many marks" error. This test is added for patch which forces indices recalculation when we rewrite compact part. +${CLICKHOUSE_CLIENT} -q 'insert into index_corruption FORMAT TSVWithNamesAndTypes' < $CURDIR/data_zstd/test_03755.tsv.zst + +${CLICKHOUSE_CLIENT} -q 'ALTER TABLE index_corruption (MODIFY TTL toDateTime(38C3868AC) + toIntervalYear(140));' + +${CLICKHOUSE_CLIENT} -q "SELECT * FROM index_corruption WHERE (7AF = 'xyz');" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS index_corruption" diff --git a/tests/queries/0_stateless/data_zstd/test_03755.tsv.zst b/tests/queries/0_stateless/data_zstd/test_03755.tsv.zst new file mode 100644 index 000000000000..bc163272ed44 Binary files /dev/null and b/tests/queries/0_stateless/data_zstd/test_03755.tsv.zst differ