Cherry-pick: Force mutation to recalculate indices when rewriting compact parts

src/Storages/MergeTree/MutateTask.cpp

@@ -163,6 +163,7 @@ static void splitAndModifyMutationCommands(
         NameSet mutated_columns;
         NameSet dropped_columns;
         NameSet ignored_columns;
+        NameSet extra_columns_for_indices_and_projections;
 
         for (const auto & command : commands)
         {
@@ -212,6 +213,40 @@ static void splitAndModifyMutationCommands(
                             ignored_columns.emplace(col.name);
                     }
                 }
+                if (command.type == MutationCommand::Type::MATERIALIZE_INDEX)
+                {
+                    const auto & all_indices = metadata_snapshot->getSecondaryIndices();
+                    for (const auto & index : all_indices)
+                    {
+                        if (index.name == command.index_name)
+                        {
+                            auto required_columns = index.expression->getRequiredColumns();
+                            for (const auto & column : required_columns)
+                            {
+                                if (!part_columns.has(column))
+                                    extra_columns_for_indices_and_projections.insert(column);
+                            }
+                            break;
+                        }
+                    }
+                }
+
+                if (command.type == MutationCommand::Type::MATERIALIZE_PROJECTION)
+                {
+                    const auto & all_projections = metadata_snapshot->getProjections();
+                    for (const auto & projection : all_projections)
+                    {
+                        if (projection.name == command.projection_name)
+                        {
+                            for (const auto & column : projection.required_columns)
+                            {
+                                if (!part_columns.has(column))
+                                    extra_columns_for_indices_and_projections.insert(column);
+                            }
+                            break;
+                        }
+                    }
+                }
             }
             else if (command.type == MutationCommand::Type::DROP_INDEX
                      || command.type == MutationCommand::Type::DROP_PROJECTION
@@ -313,6 +348,27 @@ static void splitAndModifyMutationCommands(
                 });
             }
         }
+        for (const auto & column_name : extra_columns_for_indices_and_projections)
+        {
+            if (mutated_columns.contains(column_name))
+                continue;
+
+            if (column_name == "_part_offset")
+                continue;
+
+            auto data_type = metadata_snapshot->getColumns().getColumn(
+                GetColumnsOptions::AllPhysical,
+                column_name).type;
+
+            for_interpreter.push_back(
+                MutationCommand
+                {
+                    .type = MutationCommand::Type::READ_COLUMN,
+                    .column_name = column_name,
+                    .data_type = std::move(data_type),
+                }
+            );
+        }
     }
     else
     {
@@ -1567,7 +1623,8 @@ class MutateAllPartColumnsTask : public IExecutableTask
                 renamed_stats[STATS_FILE_PREFIX + command.column_name + STATS_FILE_SUFFIX] = STATS_FILE_PREFIX + command.rename_to + STATS_FILE_SUFFIX;
         }
 
-        bool is_full_part_storage = isFullPartStorage(ctx->new_data_part->getDataPartStorage());
+        bool is_full_part_storage = isFullPartStorage(ctx->source_part->getDataPartStorage());
+        bool is_full_wide_part = is_full_part_storage && isWidePart(ctx->new_data_part);
         const auto & indices = ctx->metadata_snapshot->getSecondaryIndices();
 
         MergeTreeIndices skip_indices;
@@ -1576,9 +1633,12 @@ class MutateAllPartColumnsTask : public IExecutableTask
             if (removed_indices.contains(idx.name))
                 continue;
 
+            /// For packed part we need to recalculate all indices because they are stored inside packed parts format
+            /// For compact parts we need to recalculate indices because rewrite of compact part may produce a little bit different data part
+            /// with different number of marks.
             bool need_recalculate =
                 ctx->materialized_indices.contains(idx.name)
-                || (!is_full_part_storage && ctx->source_part->hasSecondaryIndex(idx.name));
+                || (!is_full_wide_part && ctx->source_part->hasSecondaryIndex(idx.name));
 
             if (need_recalculate)
             {

tests/queries/0_stateless/03755_ttl_materialize_bug.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS index_corruption"
+
+${CLICKHOUSE_CLIENT} -n -q "CREATE TABLE IF NOT EXISTS index_corruption
+(
+    1F740 LowCardinality(String) CODEC(ZSTD(1)),
+    B7E44AC String CODEC(ZSTD(1)),
+    3F0ED7DB LowCardinality(String) CODEC(ZSTD(1)),
+    CCD44A405D Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    38C3868AC DateTime64(9) CODEC(Delta(8), ZSTD(1)),
+    ABC0C7B_EA Nullable(String) MATERIALIZED CCD44A405D['ABC0C7B_EA'],
+    7AF String MATERIALIZED CCD44A405D['7AF'],
+    21DB_EA String MATERIALIZED CCD44A405D['21DB_EA'],
+    4482745CA_EF39713558 Nullable(String) MATERIALIZED CCD44A405D['4482745CA.EF39713558'],
+    ABC0C7B_938F19 LowCardinality(String) MATERIALIZED CCD44A405D['B7E44AC_618D.6726ED6AA465A'],
+    CCC98F_2221 String MATERIALIZED CCD44A405D['CCC98F_2221'],
+    INDEX idx_836_ADE2_72AB mapValues(CCD44A405D) TYPE bloom_filter(0.05) GRANULARITY 1,
+    INDEX idx_0989 tokens(lower(B7E44AC)) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_7AF 7AF TYPE bloom_filter(0.01) GRANULARITY 1
+)
+ENGINE = MergeTree()
+ORDER BY (toStartOfMinute(38C3868AC), toUnixTimestamp(38C3868AC))
+SETTINGS min_rows_for_wide_part = 10000, materialize_ttl_recalculate_only = 1, index_granularity = 8192;"
+
+# This data file contains random data which lead to index marks mismatch after mutation. It happens
+# because during mutation we rewrite compact part completely. And surprisingly it may lead to a little bit different
+# representation of data on disk, because we can add small last block to the last mark instead of adding new one.
+# So rewritten data part may have 1 less mark, than original one. However skip indices were not rewritten during mutation,
+# so they still contain marks for original data part. And when we try to read mutated part with skip indices,
+# we can "Too many marks" error. This test is added for patch which forces indices recalculation when we rewrite compact part.
+${CLICKHOUSE_CLIENT} -q 'insert into index_corruption FORMAT TSVWithNamesAndTypes' < $CURDIR/data_zstd/test_03755.tsv.zst
+
+${CLICKHOUSE_CLIENT} -q 'ALTER TABLE index_corruption (MODIFY TTL toDateTime(38C3868AC) + toIntervalYear(140));'
+
+${CLICKHOUSE_CLIENT} -q "SELECT * FROM index_corruption WHERE (7AF = 'xyz');"
+
+${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS index_corruption"