perf: reuse IVF coarse-quantizer routing across index segments

Cargo.toml

@@ -205,6 +205,7 @@ rstest = "0.26.1"
 serde = { version = "^1" }
 serde_json = { version = "1" }
 semver = "1.0"
+sha2 = "0.10"
 serial_test = "3"
 snafu = "0.9"
 syn = { version = "2.0.37", features = ["full"] }

protos/index.proto

@@ -225,6 +225,14 @@ message VectorIndexDetails {
   // Keys use reverse-DNS namespacing (e.g., "lance.ivf.max_iters", "lancedb.accelerator").
   // Unrecognized keys must be silently ignored by all runtimes.
   map<string, string> runtime_hints = 9;
+
+  /*
+   * Content-derived identity of this segment's final IVF coarse-routing state.
+   * Readers may reuse routing only for identical, well-formed values. It is
+   * absent for legacy or unsupported routers. The current encoding is a
+   * 32-byte SHA-256 digest.
+   */
+  optional bytes coarse_quantizer_fingerprint = 10;
 }
 
 // Hierarchical Navigable Small World (HNSW) parameters, used as an optional configuration for IVF indexes.
@@ -248,4 +256,4 @@ message BloomFilterIndexDetails {}
 
 message RTreeIndexDetails {}
 
-message FMIndexDetails {}
+message FMIndexDetails {}

rust/lance-namespace-impls/Cargo.toml

@@ -71,7 +71,7 @@ roaring.workspace = true
 uuid.workspace = true
 
 # Shared credential vending dependencies
-sha2 = { version = "0.10", optional = true }
+sha2 = { workspace = true, optional = true }
 base64 = { version = "0.22", optional = true }
 
 # AWS credential vending dependencies (optional, enabled by "credential-vendor-aws" feature)

rust/lance/Cargo.toml

@@ -86,6 +86,7 @@ snafu = { workspace = true }
 log = { workspace = true }
 serde_json = { workspace = true }
 serde = { workspace = true }
+sha2 = { workspace = true }
 permutation = { version = "0.4.0" }
 aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 tracing.workspace = true

rust/lance/src/index.rs

@@ -1302,10 +1302,11 @@ pub(crate) async fn remap_index(
             .await?;
 
             CreatedIndex {
-                index_details: prost_types::Any::from_msg(
-                    &lance_index::pb::VectorIndexDetails::default(),
-                )
-                .unwrap(),
+                index_details: matched
+                    .index_details
+                    .as_deref()
+                    .cloned()
+                    .unwrap_or_else(vector_index_details_default),
                 index_version,
                 files: table_files_to_index(files),
             }
@@ -5932,6 +5933,30 @@ mod tests {
         assert_eq!(new_uuid, RemapResult::Keep(index_uuid));
     }
 
+    #[tokio::test]
+    async fn test_remap_vector_index_preserves_details() {
+        let data = gen_batch()
+            .col("vector", array::rand_vec::<Float32Type>(Dimension::from(8)))
+            .into_reader_rows(RowCount::from(64), BatchCount::from(1));
+        let mut dataset = Dataset::write(data, "memory://", None).await.unwrap();
+
+        let params = VectorIndexParams::ivf_flat(1, DistanceType::L2);
+        dataset
+            .create_index(&["vector"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        let index = dataset.load_indices().await.unwrap()[0].clone();
+        let result = remap_index(&dataset, &index.uuid, &RowAddrRemap::empty())
+            .await
+            .unwrap();
+        let RemapResult::Remapped(remapped) = result else {
+            panic!("vector index should be remapped");
+        };
+
+        assert_eq!(remapped.index_details, *index.index_details.unwrap());
+    }
+
     /// The `fields.len() > 1` rejection in `remap_index`, which had no dedicated
     /// coverage. A covered index never reaches it: the withdrawal above returns
     /// `RemapResult::Drop` first, which is what

rust/lance/src/index/append.rs

@@ -40,6 +40,7 @@ use crate::dataset::rowids::load_row_id_sequences;
 use crate::index::scalar::{
     IndexDetails, fetch_index_details, load_fts_training_data, load_training_data,
 };
+use crate::index::vector::coarse_quantizer::with_fingerprint;
 use crate::index::vector_index_details_default;
 
 #[derive(Debug, Clone)]
@@ -925,7 +926,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 let mut append_options = options.clone();
                 append_options.num_indices_to_merge = Some(0);
                 append_options.retrain = false;
-                let (new_uuid, indices_merged, files) = optimize_vector_indices(
+                let (new_uuid, summary) = optimize_vector_indices(
                     dataset.as_ref().clone(),
                     Some(new_data_stream),
                     &field_path,
@@ -934,24 +935,29 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 )
                 .boxed()
                 .await?;
-                if indices_merged != 0 {
+                if summary.indices_merged != 0 {
                     return Err(Error::index(format!(
-                        "Optimize vector index append unexpectedly merged {indices_merged} existing segments"
+                        "Optimize vector index append unexpectedly merged {} existing segments",
+                        summary.indices_merged
                     )));
                 }
+                let new_index_details = with_fingerprint(
+                    reference_metadata
+                        .index_details
+                        .as_deref()
+                        .cloned()
+                        .unwrap_or_else(vector_index_details_default),
+                    summary.coarse_quantizer_fingerprint,
+                )?;
                 return Ok(Some(IndexMergeResults {
                     new_uuid,
                     removed_indices: Vec::new(),
                     new_fragment_bitmap: base_unindexed_bitmap,
                     new_dataset_version: dataset.manifest.version,
                     new_index_version: index_type_for_segmented_optimize(reference_index.as_ref())?
                         .version(),
-                    new_index_details: reference_metadata
-                        .index_details
-                        .as_deref()
-                        .cloned()
-                        .unwrap_or_else(vector_index_details_default),
-                    files,
+                    new_index_details,
+                    files: summary.files,
                 }));
             }
 
@@ -1002,7 +1008,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     vec![(selected_metadata, selected_index)],
                 )?;
                 let selected_ivf_view = selected_logical_index.as_ivf()?;
-                let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices(
+                let (new_uuid, summary) = Box::pin(optimize_vector_indices(
                     dataset.as_ref().clone(),
                     Option::<
                         lance_io::stream::RecordBatchStreamAdapter<
@@ -1014,7 +1020,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     options,
                 ))
                 .await?;
-                if indices_merged == 0 {
+                if summary.indices_merged == 0 {
                     return Ok(None);
                 }
 
@@ -1028,13 +1034,16 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     vec![removed_segment],
                     new_fragment_bitmap,
                     CreatedIndex {
-                        index_details: removed_segment
-                            .index_details
-                            .as_deref()
-                            .cloned()
-                            .unwrap_or_else(vector_index_details_default),
+                        index_details: with_fingerprint(
+                            removed_segment
+                                .index_details
+                                .as_deref()
+                                .cloned()
+                                .unwrap_or_else(vector_index_details_default),
+                            summary.coarse_quantizer_fingerprint,
+                        )?,
                         index_version: removed_segment.index_version as u32,
-                        files: table_files_to_index(files),
+                        files: table_files_to_index(summary.files),
                     },
                     removed_segment.dataset_version,
                 ))
@@ -1072,7 +1081,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     )
                 };
 
-                let (new_uuid, indices_merged, files) = optimize_vector_indices(
+                let (new_uuid, summary) = optimize_vector_indices(
                     dataset.as_ref().clone(),
                     new_data_stream,
                     &field_path,
@@ -1082,7 +1091,8 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 .boxed()
                 .await?;
 
-                let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec();
+                let removed_indices =
+                    old_indices[old_indices.len() - summary.indices_merged..].to_vec();
                 let new_dataset_version = removed_indices
                     .iter()
                     .map(|index| index.dataset_version)
@@ -1122,6 +1132,8 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                             .cloned()
                     })
                     .unwrap_or_else(vector_index_details_default);
+                let index_details =
+                    with_fingerprint(index_details, summary.coarse_quantizer_fingerprint)?;
                 let index_version = if let Some(metadata) = removed_indices.first() {
                     metadata.index_version as u32
                 } else {
@@ -1135,7 +1147,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     CreatedIndex {
                         index_details,
                         index_version,
-                        files: table_files_to_index(files),
+                        files: table_files_to_index(summary.files),
                     },
                     new_dataset_version,
                 ))
@@ -1397,6 +1409,7 @@ mod tests {
     use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams};
     use crate::index::CreateIndexBuilder;
     use crate::index::vector::VectorIndexParams;
+    use crate::index::vector::coarse_quantizer::common_fingerprint;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
 
     #[test]
@@ -1783,6 +1796,10 @@ mod tests {
             .unwrap();
         let retrained = dataset.load_indices_by_name(INDEX_NAME).await.unwrap();
         assert_eq!(retrained.len(), 1);
+        assert!(
+            common_fingerprint(&retrained).is_some(),
+            "retrain must persist the new final IVF routing identity"
+        );
         assert_eq!(
             retrained[0].fragment_bitmap.as_ref().unwrap(),
             dataset.fragment_bitmap.as_ref(),
@@ -2501,6 +2518,10 @@ mod tests {
         assert_eq!(stats["num_indexed_fragments"], 2);
         assert_eq!(stats["num_unindexed_fragments"], 0);
         let appended_segments = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert!(
+            common_fingerprint(&appended_segments).is_some(),
+            "append must persist the shared final IVF routing identity"
+        );
         assert!(
             appended_segments
                 .iter()
@@ -2598,6 +2619,10 @@ mod tests {
             dataset.fragment_bitmap.as_ref(),
             "the compatible merge must preserve exact fragment coverage"
         );
+        assert!(
+            common_fingerprint(&merged).is_some(),
+            "merge must preserve the final IVF routing identity"
+        );
     }
 
     #[tokio::test]