From a12f3dd891b2c30a4a45dbbb24aa3d59d25f08b8 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 18 Nov 2026 19:15:45 +0800 Subject: [PATCH 1/7] [spark] Build and query centroid-sharded vector indexes --- .../multimodal-table/global-index/vector.mdx | 105 +++ .../AbstractVectorGlobalIndexReader.java | 110 ++++ .../GlobalIndexSingleColumnWriter.java | 12 +- .../globalindex/VectorGlobalIndexer.java | 36 + .../globalindex/VectorGlobalModelTrainer.java | 33 + .../globalindex/VectorIndexShardMeta.java | 141 ++++ .../paimon/globalindex/VectorIndexUtils.java | 58 ++ .../globalindex/VectorTrainingModel.java | 40 ++ .../globalindex/VectorTrainingModelIO.java | 78 +++ .../VectorTrainingModelIOTest.java | 103 +++ .../globalindex/CentroidVectorIndexUtils.java | 85 +++ .../DataEvolutionGlobalIndexCoverage.java | 5 +- .../DataEvolutionGlobalIndexScanner.java | 24 +- .../globalindex/GlobalIndexBuilderUtils.java | 2 +- .../apache/paimon/index/IndexFileMeta.java | 49 +- .../paimon/index/IndexFileMetaSerializer.java | 6 +- .../index/IndexFileMetaV5Deserializer.java | 40 ++ .../paimon/manifest/IndexManifestEntry.java | 4 +- .../IndexManifestEntrySerializer.java | 8 +- .../manifest/IndexManifestFileHandler.java | 35 + .../GlobalIndexVisibilityChecker.java | 3 +- .../commit/DataEvolutionChangesProvider.java | 6 +- .../DataEvolutionConflictDetection.java | 65 +- .../table/sink/CommitMessageSerializer.java | 11 +- .../AbstractDataEvolutionVectorRead.java | 14 +- .../table/source/DataEvolutionVectorScan.java | 34 +- .../table/source/IndexVectorSearchSplit.java | 15 +- .../CentroidVectorIndexUtilsTest.java | 94 +++ .../index/IndexFileMetaSerializerTest.java | 51 ++ .../IndexManifestFileHandlerTest.java | 54 ++ .../commit/ConflictDetectionTest.java | 88 +++ .../CentroidGlobalIndexTopoBuilder.java | 615 ++++++++++++++++++ .../globalindex/CentroidTrainingSample.java | 116 ++++ .../DefaultGlobalIndexTopoBuilder.java | 7 + .../GlobalIndexTopologyBuilderUtils.java | 33 + .../procedure/CreateGlobalIndexProcedure.java | 3 +- .../CentroidTrainingSampleTest.java | 81 +++ .../DefaultGlobalIndexTopoBuilderTest.java | 31 + paimon-spark/paimon-spark-ut/pom.xml | 7 + .../spark/sql/CentroidVectorIndexTest.scala | 223 +++++++ paimon-vector/pom.xml | 2 +- .../NativeCentroidRoutedIndexReader.java | 353 ++++++++++ .../index/NativeVectorGlobalIndexReader.java | 119 +--- .../index/NativeVectorGlobalIndexWriter.java | 120 +++- .../index/NativeVectorGlobalIndexer.java | 115 +++- .../NativeVectorGlobalIndexerFactory.java | 1 + .../index/NativeVectorGlobalModelTrainer.java | 71 ++ .../index/NativeVectorTrainingModel.java | 70 ++ .../index/CentroidVectorGlobalIndexTest.java | 421 ++++++++++++ .../index/NativeVectorGlobalIndexTest.java | 6 + 50 files changed, 3642 insertions(+), 161 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/AbstractVectorGlobalIndexReader.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalModelTrainer.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModel.java create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java create mode 100644 paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java create mode 100644 paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java create mode 100644 paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java create mode 100644 paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java create mode 100644 paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala create mode 100644 paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java create mode 100644 paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java create mode 100644 paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java create mode 100644 paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java diff --git a/docs/docs/multimodal-table/global-index/vector.mdx b/docs/docs/multimodal-table/global-index/vector.mdx index 0f52f5fee2e8..d924cf63b114 100644 --- a/docs/docs/multimodal-table/global-index/vector.mdx +++ b/docs/docs/multimodal-table/global-index/vector.mdx @@ -336,6 +336,7 @@ Search-time options are passed with each vector search request: | Option | Default | Description | |---|---|---| | `ivf.nprobe` | Automatic | Explicit number of IVF clusters to probe. When omitted, paimon-vindex derives the width from the index, `top_k`, and filter selectivity. | +| `centroid.nprobe` | Automatic | Number of centroid shards to search. Within each selected shard, `ivf.nprobe` controls the IVF lists to search and `diskann.l_search` controls the DiskANN graph search width. | | `ivf.max_initial_filter_expansion_factor` | Disabled | Positive integer limiting filter-driven expansion of the initial automatic IVF probe width. A factor of `1` disables initial filter expansion. Progressive retries may still probe more clusters when fewer than `top_k` filtered results are found. | | `ivf.refine_factor` | Disabled | Retrieves `top_k * refine_factor` IVF candidates and reranks them with the original vectors stored in the Paimon table. It is most useful for compressed indexes such as `ivf-pq`, `ivf-sq`, and `ivf-rq` when recall is more important than latency. | | `ivf_pq.batch_table_reuse` | `auto` | IVF-PQ batch search distance-table reuse mode: `auto`, `on`, or `off`. Other index types and scalar searches ignore it. | @@ -448,6 +449,110 @@ CALL sys.create_global_index( Pass the same option keys to Python's `create_global_index(..., options={...})`. Use full 768-dimensional query vectors when searching either of these indexes. +## Centroid-Based Sharding in Spark + +For a Data Evolution append table, Spark supports centroid-based sharding for +`ivf-flat`, `ivf-pq`, `ivf-sq`, `ivf-rq`, and `diskann`. It trains a partition-level +routing model and writes one file per centroid. The table must enable row tracking and +Data Evolution, have no primary key, and use `bucket=-1`. + +```sql +ALTER TABLE db.model_embeddings SET TBLPROPERTIES ( + 'ivf-pq.dimension' = '768', + 'ivf-pq.distance.metric' = 'cosine' +); + +CALL sys.create_global_index( + table => 'db.model_embeddings', + index_column => 'embedding', + index_type => 'ivf-pq', + partitions => 'dt="2026-11-17"', + options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=256' +); +``` + +Set `.shard=centroid-based` and `.nlist` for any of these +types. The default `.shard=range-based` uses the existing build path. A centroid +build pins one snapshot. Spark map tasks count non-null vectors in their input +splits. Using those counts and the configured training ratio, the driver plans +the exact row ordinals that the outer IVF-FLAT trainer would retain. Each +map task reads its own splits and shuffles selected vectors by reservoir slot. +A single task trains the routing model on those vectors in the original training +order. The driver broadcasts the model for the subsequent centroid shuffle. +Sampling requires at least one +non-null vector. Build one partition per call; a non-partitioned table counts +as one partition. Partial builds and mixing range and centroid shards in the +same partition are unsupported. Repeating the call atomically replaces the +partition's existing model and shards of the requested type. + +Queries load the routing model, select centroids with `centroid.nprobe`, and open +only their shard files. Inside each selected shard, `ivf.nprobe` controls how many +local IVF lists to search. For DiskANN, `diskann.l_search` controls the graph search. +Automatic probe sizing and filtered search remain available. +Centroid shards store absolute row IDs. The model records the exact source row +ranges so gaps belonging to other partitions remain eligible for raw search in +`vector-index.search-mode=detail` or `chunk`. The default `fast` mode searches +only indexed rows. +Persist the metric and dimension on the table when using raw fallback so indexed +and unindexed ranges use the same distance definition. +Batch queries share each selected shard across the queries routed to it. + +For all four IVF types, Spark shuffles vectors by their assigned routing centroid. +Each populated shard trains an independent IVF index on the vectors it receives. +For IVF-PQ, this gives each shard its own inner centroids and PQ codebook. The outer +`.nlist` sets the number of shard slots. The optional +`.centroid.local-nlist` sets the number of IVF lists per shard and +defaults to automatic sizing from that shard's vector count. + +For example, build with +`ivf-pq.shard=centroid-based,ivf-pq.nlist=64,ivf-pq.centroid.local-nlist=32`. +A query with `map('centroid.nprobe','4','ivf.nprobe','8')` selects four shard +files and searches eight local lists in each populated shard. Both levels +support automatic probe sizing. Empty local shards have a manifest row count +of zero and queries skip them. + +DiskANN uses the same outer router as IVF. Each populated centroid uses the +ordinary DiskANN trainer, writer and reader. The shard trains its PQ codebook +and builds its graph from the rows it receives. DiskANN’s seed and memory budget +apply to this local build. The outer sample and coarse centers use the same +IVF-FLAT training options as the IVF shard types. An empty shard uses an +eight-byte `PCEMPTY1` marker with manifest row count zero. Queries skip that file. Empty slots remain in the complete +generation so centroid identifiers stay stable. + +For example, use `index_type => 'diskann'` and +`options => 'diskann.shard=centroid-based,diskann.nlist=64'`, then query with +`map('centroid.nprobe','8','diskann.l_search','64')`. Persist +`diskann.dimension` and `diskann.metric` on the table for raw fallback. + +If source rows change while the build is running, including through compaction, +the commit fails and the build must be retried. Appends outside its source ranges +can proceed. Maintenance that reassigns row IDs invalidates the complete index +generation and requires a rebuild. + +This build mode and its reader currently require the Java/Spark implementation +and a matching native library with centroid model APIs. Upgrade every component +that reads these indexes before enabling it. Older readers and Python index +builders do not support this layout; drop the centroid index before downgrading +or rebuilding it through those clients. The model file and each populated shard carry +training metadata, so measure model-loading cost and total index size alongside +recall, latency, and the number of opened files. + +For quantized IVF indexes, each range or centroid shard fits its own codebooks +or bounds. Compare performance at a common recall target +using separate calibration and validation queries. To measure the effect of +quantization alone, probe every outer centroid and every local list. If it remains below the target, +increasing `centroid.nprobe` alone cannot remove quantization error. For IVF-RQ, +calibrate `rq.bits`; for DiskANN, calibrate both outer probes and local graph +search width. A shared routing model does not guarantee equal recall. + +Calibrate `ivf-pq.nlist` and `ivf-pq.pq.m` together with the query probe width. +Increasing `nlist` creates more centroid files, each populated file carrying its own +local model. Increasing `pq.m` uses more encoded bytes per vector; +for the current 8-bit PQ codes, `pq.m=64` uses twice the code bytes of +`pq.m=32`. OPQ and larger training limits are also workload-dependent choices, +not recall guarantees. Record the resolved build options, total index size, +query file reads, and achieved recall when comparing configurations. + ## Per-Field Options The options above can also be set at the table level (in `TBLPROPERTIES`), where they are shared diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/AbstractVectorGlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/AbstractVectorGlobalIndexReader.java new file mode 100644 index 000000000000..6718ccdde053 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/AbstractVectorGlobalIndexReader.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.predicate.FieldRef; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +/** Base for vector-only readers: scalar predicates remain unavailable for index evaluation. */ +public abstract class AbstractVectorGlobalIndexReader implements GlobalIndexReader { + @Override + public CompletableFuture> visitIsNotNull(FieldRef fieldRef) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitIsNull(FieldRef fieldRef) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitStartsWith( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitEndsWith( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitContains( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitLike( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitLessThan( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitGreaterOrEqual( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitNotEqual( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitLessOrEqual( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitEqual( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitGreaterThan( + FieldRef fieldRef, Object literal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitIn( + FieldRef fieldRef, List literals) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture> visitNotIn( + FieldRef fieldRef, List literals) { + return CompletableFuture.completedFuture(Optional.empty()); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexSingleColumnWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexSingleColumnWriter.java index 301abbccd105..ff27f4d7a638 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexSingleColumnWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexSingleColumnWriter.java @@ -22,16 +22,20 @@ import java.util.List; -/** Index writer for single-column global index with relative row id (from 0 to rowCnt - 1). */ +/** + * Single-column global index writer. Range writers consume relative row IDs; centroid vector + * writers consume absolute Paimon row IDs, as recorded in their index metadata. + */ public interface GlobalIndexSingleColumnWriter extends GlobalIndexWriter { /** - * Write the indexed key and its related relative row id to the index file. + * Write the indexed key and its row ID in the writer's declared encoding. * * @param key nullable index key - * @param relativeRowId local row id calculated by {@code rowId - rangeStart} + * @param rowId relative ID calculated by {@code absoluteRowId - rangeStart} for range writers, + * or the absolute row ID for a centroid vector writer */ - void write(@Nullable Object key, long relativeRowId); + void write(@Nullable Object key, long rowId); /** * Finishes a writer whose normalized entries cover {@code sourceRowCount} source rows. diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java index 63166ef7c1f1..ccba5595f268 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java @@ -18,9 +18,45 @@ package org.apache.paimon.globalindex; +import java.io.Serializable; +import java.util.Iterator; + /** A {@link GlobalIndexer} that supports vector similarity search. */ public interface VectorGlobalIndexer extends GlobalIndexer { /** Returns the metric name used to convert vector distances to comparable scores. */ String metric(); + + /** Zero-based ordinals in the non-null input stream, in the order required for training. */ + default long[] planGlobalTrainingSample(long vectorCount) { + throw new UnsupportedOperationException("Centroid training is not supported."); + } + + /** Train using ordered, preselected batches and the full non-null corpus size. */ + default VectorTrainingModel trainGlobalModel( + long vectorCount, Iterator batches) { + throw new UnsupportedOperationException("Centroid training is not supported."); + } + + /** A row-major batch selected by the distributed sampler. */ + final class TrainingBatch implements Serializable { + private static final long serialVersionUID = 1L; + public final float[] vectors; + public final int count; + + public TrainingBatch(float[] vectors, int count) { + this.vectors = vectors; + this.count = count; + } + } + + default VectorGlobalModelTrainer createGlobalModelTrainer() { + throw new UnsupportedOperationException( + "Centroid sharding is not supported by this vector index."); + } + + default VectorTrainingModel restoreTrainingModel(byte[] payload) { + throw new UnsupportedOperationException( + "Centroid sharding is not supported by this vector index."); + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalModelTrainer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalModelTrainer.java new file mode 100644 index 000000000000..3707f0981883 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalModelTrainer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** Bounded-memory ingestion of samples for one partition-level model. */ +public interface VectorGlobalModelTrainer extends AutoCloseable { + void write(@Nullable Object vector, long absoluteRowId); + + VectorTrainingModel finishTraining() throws IOException; + + @Override + void close() throws IOException; +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java new file mode 100644 index 000000000000..483b11160d26 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.utils.Range; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Manifest metadata shared by vector implementations and engine-independent planning. */ +public final class VectorIndexShardMeta { + public static final String CENTROID_BASED = "centroid-based"; + public static final String ABSOLUTE_ROW_ID = "ABSOLUTE_ROW_ID"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final TypeReference> TYPE = + new TypeReference>() {}; + private final Map values; + + private VectorIndexShardMeta(Map values) { + this.values = values; + } + + public static VectorIndexShardMeta deserialize(@Nullable byte[] bytes) { + try { + return new VectorIndexShardMeta( + bytes == null ? Collections.emptyMap() : MAPPER.readValue(bytes, TYPE)); + } catch (IOException e) { + throw new IllegalArgumentException("Invalid vector shard metadata", e); + } + } + + public static VectorIndexShardMeta routing(String modelId, int nlist, List coverage) { + checkArgument( + modelId != null && !modelId.isEmpty() && nlist > 0 && !coverage.isEmpty(), + "Invalid routing model metadata."); + Map values = base(modelId); + values.put("nlist", Integer.toString(nlist)); + values.put( + "sourceRowRanges", + Range.sortAndMergeOverlap(coverage, true).stream() + .map(r -> r.from + ":" + r.to) + .collect(Collectors.joining(","))); + return new VectorIndexShardMeta(values); + } + + public static VectorIndexShardMeta centroid(String modelId, int centroid) { + checkArgument(centroid >= 0, "Centroid must be nonnegative."); + Map values = base(modelId); + values.put("centroid", Integer.toString(centroid)); + values.put("rowIdEncoding", ABSOLUTE_ROW_ID); + return new VectorIndexShardMeta(values); + } + + private static Map base(String modelId) { + checkArgument(modelId != null && !modelId.isEmpty(), "Model id is required."); + Map values = new LinkedHashMap<>(); + values.put("shardMode", CENTROID_BASED); + values.put("modelId", modelId); + return values; + } + + public boolean centroidBased() { + return CENTROID_BASED.equals(values.get("shardMode")); + } + + public boolean routingModel() { + return centroidBased() && values.containsKey("nlist"); + } + + public String modelId() { + String id = values.get("modelId"); + checkArgument(id != null && !id.isEmpty(), "Missing centroid model id."); + return id; + } + + public int centroid() { + checkArgument( + ABSOLUTE_ROW_ID.equals(values.get("rowIdEncoding")), + "Unsupported centroid row ID encoding."); + int centroid = Integer.parseInt(values.get("centroid")); + checkArgument(centroid >= 0, "Invalid centroid id."); + return centroid; + } + + public int nlist() { + int nlist = Integer.parseInt(values.get("nlist")); + checkArgument(nlist > 0, "Invalid nlist."); + return nlist; + } + + public List sourceRowRanges() { + List ranges = new ArrayList<>(); + String encoded = values.get("sourceRowRanges"); + checkArgument(encoded != null && !encoded.isEmpty(), "Missing source row coverage."); + for (String range : encoded.split(",")) { + String[] endpoints = range.split(":"); + checkArgument(endpoints.length == 2, "Invalid source row range."); + long from = Long.parseLong(endpoints[0]); + long to = Long.parseLong(endpoints[1]); + checkArgument(from >= 0 && to >= from, "Invalid source row range."); + ranges.add(new Range(from, to)); + } + return Range.sortAndMergeOverlap(ranges, true); + } + + public byte[] serialize() { + try { + return MAPPER.writeValueAsBytes(values); + } catch (IOException e) { + throw new IllegalStateException("Cannot serialize vector shard metadata", e); + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java new file mode 100644 index 000000000000..72c75c864762 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalVector; + +/** Materialization at the portable vector index boundary. */ +public final class VectorIndexUtils { + private VectorIndexUtils() {} + + public static boolean isVectorIndexType(String type) { + return "ivf-flat".equals(type) + || "ivf-pq".equals(type) + || "ivf-sq".equals(type) + || "ivf-rq".equals(type) + || "diskann".equals(type); + } + + public static float[] toFloatArray(Object value) { + if (value instanceof float[]) { + return ((float[]) value).clone(); + } + int size = + value instanceof InternalVector + ? ((InternalVector) value).size() + : ((InternalArray) value).size(); + float[] result = new float[size]; + for (int i = 0; i < size; i++) { + if (value instanceof InternalVector) { + result[i] = ((InternalVector) value).getFloat(i); + } else { + InternalArray array = (InternalArray) value; + if (array.isNullAt(i)) { + throw new IllegalArgumentException("Vector element cannot be null."); + } + result[i] = array.getFloat(i); + } + } + return result; + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModel.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModel.java new file mode 100644 index 000000000000..ae98ce99e38f --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModel.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; + +import java.io.IOException; + +/** Portable shared training state. Each restored instance owns its native resources. */ +public interface VectorTrainingModel extends AutoCloseable { + int nlist(); + + int assignCentroid(float[] vector); + + int[] nearestCentroids(float[] vector, int nprobe); + + byte[] serialize(); + + GlobalIndexSingleColumnWriter createCentroidWriter( + GlobalIndexFileWriter fileWriter, int centroid, String modelId); + + @Override + void close() throws IOException; +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java new file mode 100644 index 000000000000..752cab3b3ed1 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Versioned Paimon envelope around the independently versioned native training payload. */ +public final class VectorTrainingModelIO { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private VectorTrainingModelIO() {} + + public static void write(OutputStream out, byte[] payload, String modelId) throws IOException { + Map meta = new LinkedHashMap<>(); + meta.put("version", 1); + meta.put("shardMode", VectorIndexShardMeta.CENTROID_BASED); + meta.put("modelId", modelId); + byte[] header = MAPPER.writeValueAsBytes(meta); + DataOutputStream data = new DataOutputStream(out); + data.writeInt(header.length); + data.write(header); + data.write(payload); + } + + public static byte[] read(InputStream input, long fileSize, String modelId) throws IOException { + return read(input, fileSize, modelId, Integer.MAX_VALUE - 8); + } + + public static byte[] read(InputStream input, long fileSize, String modelId, int maxPayloadBytes) + throws IOException { + DataInputStream data = new DataInputStream(input); + int size = data.readInt(); + if (size <= 0 || size > 65536 || size > fileSize - 4) { + throw new IOException("Invalid routing model metadata length."); + } + long payloadSize = fileSize - 4 - size; + if (payloadSize <= 0 || payloadSize > maxPayloadBytes) { + throw new IOException("Invalid routing model payload length."); + } + byte[] header = new byte[size]; + data.readFully(header); + Map metadata = + MAPPER.readValue(header, new TypeReference>() {}); + if (!Integer.valueOf(1).equals(metadata.get("version")) + || !VectorIndexShardMeta.CENTROID_BASED.equals(metadata.get("shardMode")) + || !modelId.equals(metadata.get("modelId"))) { + throw new IOException("Unsupported or mismatched routing model envelope."); + } + byte[] payload = new byte[(int) payloadSize]; + data.readFully(payload); + return payload; + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java new file mode 100644 index 000000000000..043dd2b7c7b8 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Persisted envelope version, model identity and length validation. */ +class VectorTrainingModelIOTest { + @Test + void rejectsPayloadOverReaderLimitBeforeAllocation() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + VectorTrainingModelIO.write(out, new byte[] {1, 2}, "test"); + byte[] bytes = out.toByteArray(); + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(bytes), bytes.length, "test", 1)) + .isInstanceOf(IOException.class) + .hasMessageContaining("payload length"); + } + + @Test + void testPayloadRoundTripAndModelIdentity() throws Exception { + byte[] payload = {1, 2, 3}; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + VectorTrainingModelIO.write(out, payload, "model-a"); + byte[] bytes = out.toByteArray(); + assertThat( + VectorTrainingModelIO.read( + new ByteArrayInputStream(bytes), bytes.length, "model-a")) + .containsExactly(payload); + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(bytes), bytes.length, "model-b")) + .isInstanceOf(IOException.class) + .hasMessageContaining("mismatched"); + String encoded = new String(bytes, StandardCharsets.ISO_8859_1); + byte[] unsupported = + encoded.replace("\"version\":1", "\"version\":2") + .getBytes(StandardCharsets.ISO_8859_1); + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(unsupported), + unsupported.length, + "model-a")) + .isInstanceOf(IOException.class) + .hasMessageContaining("Unsupported"); + } + + @Test + void testInvalidLengthsFailBeforePayloadAllocation() throws Exception { + byte[] invalid = {-1, -1, -1, -1}; + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(invalid), 4, "model")) + .isInstanceOf(IOException.class) + .hasMessageContaining("length"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + VectorTrainingModelIO.write(out, new byte[] {1}, "model"); + byte[] bytes = out.toByteArray(); + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(bytes), + (long) Integer.MAX_VALUE + 65536, + "model")) + .isInstanceOf(IOException.class) + .hasMessageContaining("payload length"); + assertThatThrownBy( + () -> + VectorTrainingModelIO.read( + new ByteArrayInputStream(bytes), bytes.length - 1, "model")) + .isInstanceOf(IOException.class) + .hasMessageContaining("payload length"); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java new file mode 100644 index 000000000000..8e1a5cc688b8 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.utils.Range; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Validation and coverage of an atomically committed centroid model and its shards. */ +public final class CentroidVectorIndexUtils { + private CentroidVectorIndexUtils() {} + + public static boolean centroidBased(IndexFileMeta file) { + return VectorIndexUtils.isVectorIndexType(file.indexType()) + && file.globalIndexMeta() != null + && VectorIndexShardMeta.deserialize(file.globalIndexMeta().indexMeta()) + .centroidBased(); + } + + public static VectorIndexShardMeta metadata(IndexFileMeta file) { + return VectorIndexShardMeta.deserialize(file.globalIndexMeta().indexMeta()); + } + + public static IndexFileMeta validateModel(List files) { + IndexFileMeta routing = null; + for (IndexFileMeta file : files) { + checkArgument( + centroidBased(file), + "Cannot mix centroid-based and range-based vector shards."); + if (file.isRoutingModel()) { + checkArgument( + routing == null, + "Multiple routing models for the same partition and index."); + routing = file; + } + } + checkArgument(routing != null, "Missing centroid routing model."); + VectorIndexShardMeta model = metadata(routing); + Set centroids = new HashSet<>(); + for (IndexFileMeta file : files) { + checkArgument( + model.modelId().equals(metadata(file).modelId()), + "Centroid shards belong to different models."); + if (!file.isRoutingModel()) { + int centroid = metadata(file).centroid(); + checkArgument( + centroid < model.nlist() && centroids.add(centroid), + "Duplicate or invalid centroid shard."); + } + } + checkArgument(centroids.size() == model.nlist(), "Incomplete centroid model shard set."); + model.sourceRowRanges(); + return routing; + } + + public static List coverage(IndexFileMeta file) { + if (!centroidBased(file)) { + return java.util.Collections.singletonList(file.globalIndexMeta().rowRange()); + } + return file.isRoutingModel() + ? metadata(file).sourceRowRanges() + : java.util.Collections.emptyList(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java index 6ff33b520203..d63902eeca9f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java @@ -75,8 +75,9 @@ public DataEvolutionGlobalIndexCoverage( if (meta.extraFieldIds() != null && meta.extraFieldIds().length > 0) { continue; } - Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd()); - addCoverage(meta.indexFieldId(), range); + for (Range range : CentroidVectorIndexUtils.coverage(indexFile)) { + addCoverage(meta.indexFieldId(), range); + } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java index 34bcabf259dc..ca5421f0c69c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java @@ -516,16 +516,20 @@ private Collection createReaders( .collect(Collectors.toList()); futures.add( CompletableFuture.supplyAsync( - () -> - new OffsetGlobalIndexReader( - globalIndexer.createReader( - indexFileReadWrite, - globalMetas, - range.count(), - null, - executor), - range.from, - range.to), + () -> { + GlobalIndexReader reader = + globalIndexer.createReader( + indexFileReadWrite, + globalMetas, + range.count(), + null, + executor); + return CentroidVectorIndexUtils.centroidBased( + indexFileMetas.get(0)) + ? reader + : new OffsetGlobalIndexReader( + reader, range.from, range.to); + }, executor)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java index f3e6e6fdfbc8..d533e46cc216 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexBuilderUtils.java @@ -125,7 +125,7 @@ public static List unindexedRowRanges( for (IndexManifestEntry entry : currentIndexes) { GlobalIndexMeta meta = entry.indexFile().globalIndexMeta(); if (meta != null) { - indexedRanges.add(meta.rowRange()); + indexedRanges.addAll(CentroidVectorIndexUtils.coverage(entry.indexFile())); } } indexedRanges = Range.sortAndMergeOverlap(indexedRanges, true); diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java index cc77e30438f3..1aae2e641ba8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java @@ -41,6 +41,9 @@ @Public public class IndexFileMeta { + public static final String DATA_FILE = "INDEX"; + public static final String ROUTING_MODEL = "ROUTING_MODEL"; + public static final RowType SCHEMA = new RowType( false, @@ -54,12 +57,14 @@ public class IndexFileMeta { "_DELETIONS_VECTORS_RANGES", new ArrayType(true, DeletionVectorMeta.SCHEMA)), new DataField(5, "_EXTERNAL_PATH", newStringType(true)), - new DataField(6, "_GLOBAL_INDEX", GlobalIndexMeta.SCHEMA))); + new DataField(6, "_GLOBAL_INDEX", GlobalIndexMeta.SCHEMA), + new DataField(7, "_INDEX_FILE_KIND", newStringType(true)))); private final String indexType; private final String fileName; private final long fileSize; private final long rowCount; + private final String fileKind; @Nullable private final GlobalIndexMeta globalIndexMeta; @@ -89,6 +94,26 @@ public IndexFileMeta( @Nullable LinkedHashMap dvRanges, @Nullable String externalPath, @Nullable GlobalIndexMeta globalIndexMeta) { + this( + indexType, + fileName, + fileSize, + rowCount, + dvRanges, + externalPath, + globalIndexMeta, + DATA_FILE); + } + + public IndexFileMeta( + String indexType, + String fileName, + long fileSize, + long rowCount, + @Nullable LinkedHashMap dvRanges, + @Nullable String externalPath, + @Nullable GlobalIndexMeta globalIndexMeta, + @Nullable String fileKind) { this.indexType = indexType; this.fileName = fileName; this.fileSize = fileSize; @@ -96,6 +121,10 @@ public IndexFileMeta( this.dvRanges = dvRanges; this.externalPath = externalPath; this.globalIndexMeta = globalIndexMeta; + this.fileKind = fileKind == null ? DATA_FILE : fileKind; + if (!DATA_FILE.equals(this.fileKind) && !ROUTING_MODEL.equals(this.fileKind)) { + throw new IllegalArgumentException("Unknown index file kind: " + fileKind); + } } public IndexFileMeta( @@ -112,6 +141,14 @@ public String indexType() { return indexType; } + public String fileKind() { + return fileKind; + } + + public boolean isRoutingModel() { + return ROUTING_MODEL.equals(fileKind); + } + @Nullable public GlobalIndexMeta globalIndexMeta() { return globalIndexMeta; @@ -148,6 +185,7 @@ public boolean equals(Object o) { } IndexFileMeta that = (IndexFileMeta) o; return Objects.equals(indexType, that.indexType) + && Objects.equals(fileKind, that.fileKind) && Objects.equals(fileName, that.fileName) && fileSize == that.fileSize && rowCount == that.rowCount @@ -159,7 +197,14 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - indexType, fileName, fileSize, rowCount, dvRanges, externalPath, globalIndexMeta); + indexType, + fileName, + fileSize, + rowCount, + dvRanges, + externalPath, + globalIndexMeta, + fileKind); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java index 6e71c5f74a5b..82419c8fde51 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java @@ -58,7 +58,8 @@ public InternalRow toRow(IndexFileMeta record) { record.rowCount(), dvMetasToRowArrayData(record.dvRanges()), fromString(record.externalPath()), - globalIndexRow); + globalIndexRow, + record.isRoutingModel() ? fromString(record.fileKind()) : null); } @Override @@ -89,7 +90,8 @@ public IndexFileMeta fromRow(InternalRow row) { row.getLong(3), row.isNullAt(4) ? null : rowArrayDataToDvMetas(row.getArray(4)), row.isNullAt(5) ? null : row.getString(5).toString(), - globalIndexMeta); + globalIndexMeta, + row.getFieldCount() <= 7 || row.isNullAt(7) ? null : row.getString(7).toString()); } public static InternalArray dvMetasToRowArrayData( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java new file mode 100644 index 000000000000..a717ad72e38b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.ObjectSerializer; + +/** Reads the seven-field index metadata used by commit message versions 12 through 14. */ +public class IndexFileMetaV5Deserializer extends ObjectSerializer { + public IndexFileMetaV5Deserializer() { + super(new RowType(false, IndexFileMeta.SCHEMA.getFields().subList(0, 7))); + } + + @Override + public InternalRow toRow(IndexFileMeta record) { + throw new UnsupportedOperationException(); + } + + @Override + public IndexFileMeta fromRow(InternalRow row) { + return new IndexFileMetaSerializer().fromRow(row); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java index f69716d455e6..3fd90c68bdb0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java @@ -55,6 +55,7 @@ public class IndexManifestEntry { public static final String DELETION_VECTORS_RANGES = "_DELETIONS_VECTORS_RANGES"; public static final String EXTERNAL_PATH = "_EXTERNAL_PATH"; public static final String GLOBAL_INDEX = "_GLOBAL_INDEX"; + public static final String INDEX_FILE_KIND = "_INDEX_FILE_KIND"; public static final RowType SCHEMA = new RowType( @@ -72,7 +73,8 @@ public class IndexManifestEntry { DELETION_VECTORS_RANGES, new ArrayType(true, DeletionVectorMeta.SCHEMA)), new DataField(8, EXTERNAL_PATH, newStringType(true)), - new DataField(9, GLOBAL_INDEX, GlobalIndexMeta.SCHEMA))); + new DataField(9, GLOBAL_INDEX, GlobalIndexMeta.SCHEMA), + new DataField(10, INDEX_FILE_KIND, newStringType(true)))); public static final RowType MANIFEST_ROW_TYPE = ManifestSchemaUtils.withFormatIdentifier(SCHEMA); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java index c37bb77a0022..3f7912624648 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java @@ -76,7 +76,8 @@ public InternalRow toRow(IndexManifestEntry record) { indexFile.rowCount(), dvMetasToRowArrayData(indexFile.dvRanges()), fromString(indexFile.externalPath()), - globalIndexRow); + globalIndexRow, + indexFile.isRoutingModel() ? fromString(indexFile.fileKind()) : null); } @Override @@ -123,7 +124,10 @@ private IndexManifestEntry fromDataRow(InternalRow row) { row.getLong(6), row.isNullAt(7) ? null : rowArrayDataToDvMetas(row.getArray(7)), row.isNullAt(8) ? null : row.getString(8).toString(), - globalIndexMeta)); + globalIndexMeta, + row.getFieldCount() <= 10 || row.isNullAt(10) + ? null + : row.getString(10).toString())); } public static Function partitionGetter() { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java index 13d3feb6b239..606f286b566b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java @@ -19,6 +19,7 @@ package org.apache.paimon.manifest; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.GlobalIndexMeta; @@ -235,6 +236,39 @@ public List combine( for (IndexManifestEntry entry : added) { indexEntries.put(entry.indexFile().fileName(), entry); } + boolean checkCentroidModels = + prevIndexFiles.stream().anyMatch(entry -> entry.indexFile().isRoutingModel()) + || newIndexFiles.stream() + .anyMatch( + entry -> + entry.indexFile().isRoutingModel() + || CentroidVectorIndexUtils + .centroidBased( + entry.indexFile())); + if (!checkCentroidModels) { + return new ArrayList<>(indexEntries.values()); + } + Map>> models = new HashMap<>(); + for (IndexManifestEntry entry : indexEntries.values()) { + IndexFileMeta file = entry.indexFile(); + if (!org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType( + file.indexType()) + || file.globalIndexMeta() == null) { + continue; + } + models.computeIfAbsent(entry.partition(), p -> new HashMap<>()) + .computeIfAbsent( + file.indexType() + ":" + file.globalIndexMeta().indexFieldId(), + f -> new ArrayList<>()) + .add(file); + } + for (Map> partition : models.values()) { + for (List files : partition.values()) { + if (files.stream().anyMatch(CentroidVectorIndexUtils::centroidBased)) { + CentroidVectorIndexUtils.validateModel(files); + } + } + } return new ArrayList<>(indexEntries.values()); } @@ -251,6 +285,7 @@ private void validateRetainedIndexFiles( for (IndexManifestEntry added : addedIndexFiles) { GlobalIndexMeta addedMeta = added.indexFile().globalIndexMeta(); if (addedMeta == null + || !retained.partition().equals(added.partition()) || (retainedMeta.sourceMeta() != null && addedMeta.sourceMeta() != null && !DataEvolutionIndexSourceMeta.isDataEvolutionMeta( diff --git a/paimon-core/src/main/java/org/apache/paimon/metastore/GlobalIndexVisibilityChecker.java b/paimon-core/src/main/java/org/apache/paimon/metastore/GlobalIndexVisibilityChecker.java index 596bf15e91b1..ebeda8b2f669 100644 --- a/paimon-core/src/main/java/org/apache/paimon/metastore/GlobalIndexVisibilityChecker.java +++ b/paimon-core/src/main/java/org/apache/paimon/metastore/GlobalIndexVisibilityChecker.java @@ -20,6 +20,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileKind; @@ -94,7 +95,7 @@ boolean visibleIn(Snapshot snapshot) { indexedRangesByPartition .computeIfAbsent(entry.partition().copy(), k -> new HashMap<>()) .computeIfAbsent(identifier, k -> new ArrayList<>()) - .add(globalIndex.rowRange()); + .addAll(CentroidVectorIndexUtils.coverage(entry.indexFile())); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java index 45f8fcc7575e..9a59f97017aa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java @@ -145,6 +145,9 @@ private CommitChanges mapChanges(SerializationAssignment assignment) { indexes.add(entry); continue; } + checkState( + !org.apache.paimon.globalindex.CentroidVectorIndexUtils.centroidBased(file), + "Cannot reuse centroid indexes across row-ID reassignment; rebuild the index."); Range range = assignment.mapRowRange(entry.partition(), meta.rowRange()); GlobalIndexMeta mapped = new GlobalIndexMeta( @@ -166,7 +169,8 @@ private CommitChanges mapChanges(SerializationAssignment assignment) { file.rowCount(), file.dvRanges(), file.externalPath(), - mapped))); + mapped, + file.fileKind()))); } return new CommitChanges(files, changes.changelogFiles, indexes); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java index ae14f7a9d82d..07b1d200e034 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java @@ -22,6 +22,8 @@ import org.apache.paimon.Snapshot.CommitKind; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.errors.ErrorMessages; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; +import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; @@ -366,6 +368,11 @@ protected Optional checkTableSpecificConflicts( return exception; } + exception = checkCentroidSourceChanges(latestSnapshot, deltaIndexEntries); + if (exception.isPresent()) { + return exception; + } + return checkForRowIdFromSnapshot( latestSnapshot, deltaEntries, deltaIndexEntries, rowIdConflictChecker); } @@ -616,19 +623,55 @@ private Optional checkGlobalIndexRowIdExistence( for (IndexManifestEntry indexEntry : indexesToCheck) { GlobalIndexMeta globalIndex = indexEntry.indexFile().globalIndexMeta(); checkState(globalIndex != null, "Global index meta must not be null."); - Range indexRange = globalIndex.rowRange(); RowRangeIndex rowRangeIndex = rowRangeIndexes.get(Pair.of(indexEntry.partition(), indexEntry.bucket())); - if (rowRangeIndex == null || !rowRangeIndex.contains(indexRange)) { - return Optional.of( - new RuntimeException( - String.format( - "Global index row ID existence conflict: index file '%s' " - + "references row range %s, but this range " - + "is not fully covered by current data " - + "files. The referenced row IDs may have been " - + "reassigned or removed by a concurrent commit.", - indexEntry.indexFile().fileName(), indexRange))); + for (Range indexRange : CentroidVectorIndexUtils.coverage(indexEntry.indexFile())) { + if (rowRangeIndex == null || !rowRangeIndex.contains(indexRange)) { + return Optional.of( + new RuntimeException( + String.format( + "Global index row ID existence conflict: index file '%s' " + + "references row range %s, but this range " + + "is not fully covered by current data " + + "files. The referenced row IDs may have been " + + "reassigned or removed by a concurrent commit.", + indexEntry.indexFile().fileName(), indexRange))); + } + } + } + return Optional.empty(); + } + + private Optional checkCentroidSourceChanges( + Snapshot latestSnapshot, List indexEntries) { + for (IndexManifestEntry index : indexEntries) { + if (index.kind() != FileKind.ADD + || !index.indexFile().isRoutingModel() + || !CentroidVectorIndexUtils.centroidBased(index.indexFile())) { + continue; + } + long baseline = + DataEvolutionIndexSourceMeta.fromIndexFile(index.indexFile()).scanSnapshotId(); + List coverage = CentroidVectorIndexUtils.coverage(index.indexFile()); + for (long id = baseline + 1; id <= latestSnapshot.id(); id++) { + Snapshot snapshot = snapshotManager.snapshot(id); + for (ManifestEntry change : + commitScanner() + .readIncrementalEntries( + snapshot, Collections.singletonList(index.partition()))) { + if (change.file().firstRowId() == null) { + continue; + } + for (Range source : coverage) { + if (source.hasIntersection(change.file().nonNullRowIdRange())) { + return Optional.of( + new RuntimeException( + "Centroid index source rows changed after training snapshot " + + baseline + + "; rebuild the index against the latest snapshot.")); + } + } + } } } return Optional.empty(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java index 2a3e3ebd81bf..bdf385b05c2e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java @@ -26,6 +26,7 @@ import org.apache.paimon.index.IndexFileMetaV2Deserializer; import org.apache.paimon.index.IndexFileMetaV3Deserializer; import org.apache.paimon.index.IndexFileMetaV4Deserializer; +import org.apache.paimon.index.IndexFileMetaV5Deserializer; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMeta08Serializer; @@ -53,7 +54,7 @@ /** {@link VersionedSerializer} for {@link CommitMessage}. */ public class CommitMessageSerializer implements VersionedSerializer { - public static final int CURRENT_VERSION = 14; + public static final int CURRENT_VERSION = 15; private final DataFileMetaSerializer dataFileSerializer; private final IndexFileMetaSerializer indexEntrySerializer; @@ -68,6 +69,7 @@ public class CommitMessageSerializer implements VersionedSerializer> fileDeserializer( private IOExceptionSupplier> indexEntryDeserializer( int version, DataInputView view) { - if (version >= 12) { + if (version >= 15) { return () -> indexEntrySerializer.deserializeList(view); + } else if (version >= 12) { + if (indexEntryV5Deserializer == null) { + indexEntryV5Deserializer = new IndexFileMetaV5Deserializer(); + } + return () -> indexEntryV5Deserializer.deserializeList(view); } else if (version == 11) { if (indexEntryV4Deserializer == null) { indexEntryV4Deserializer = new IndexFileMetaV4Deserializer(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index e3206db3dec1..476df2d145a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.InternalVector; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner; import org.apache.paimon.globalindex.GlobalIndexEvaluator; import org.apache.paimon.globalindex.GlobalIndexIOMeta; @@ -340,7 +341,11 @@ protected CompletableFuture> eval( VectorSearch vectorSearch = new VectorSearch(vector, searchLimit, vectorColumn.name(), options) .withIncludeRowIds(includeRowIds); - return new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd) + GlobalIndexReader effectiveReader = + CentroidVectorIndexUtils.centroidBased(vectorIndexFiles.get(0)) + ? reader + : new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd); + return effectiveReader .visitVectorSearch(vectorSearch) .whenComplete((r, t) -> IOUtils.closeQuietly(reader)); } @@ -374,7 +379,11 @@ protected CompletableFuture>> evalBatch( BatchVectorSearch batchVectorSearch = new BatchVectorSearch(vectors, searchLimit, vectorColumn.name(), options) .withIncludeRowIds(includeRowIds); - return new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd) + GlobalIndexReader effectiveReader = + CentroidVectorIndexUtils.centroidBased(vectorIndexFiles.get(0)) + ? reader + : new OffsetGlobalIndexReader(reader, rowRangeStart, rowRangeEnd); + return effectiveReader .visitBatchVectorSearch(batchVectorSearch) .whenComplete((r, t) -> IOUtils.closeQuietly(reader)); } @@ -436,6 +445,7 @@ private List buildIOMetaList( new GlobalIndexIOMeta( indexPathFactory.toPath(indexFile), indexFile.fileSize(), + indexFile.rowCount(), meta.indexMeta())); } return indexIOMetaList; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index d1e7ba58a0b2..a277b322a2cd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -21,6 +21,8 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.CoreOptions.GlobalIndexSearchMode; import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.globalindex.DataEvolutionGlobalIndexCoverage; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; @@ -116,8 +118,9 @@ public Plan scan() { return false; }; + List indexEntries = indexFileHandler.scan(snapshot, indexFileFilter); List allIndexFiles = - indexFileHandler.scan(snapshot, indexFileFilter).stream() + indexEntries.stream() .map(IndexManifestEntry::indexFile) .collect(Collectors.toList()); String vectorIndexType = vectorIndexType(allIndexFiles); @@ -136,9 +139,12 @@ public Plan scan() { for (IndexFileMeta indexFile : allIndexFiles) { GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta()); if (isPrimaryColumn(meta, vectorColumn.id())) { + vectorIndexFiles.add(indexFile); + if (CentroidVectorIndexUtils.centroidBased(indexFile)) { + continue; + } Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd()); vectorByRange.computeIfAbsent(range, k -> new ArrayList<>()).add(indexFile); - vectorIndexFiles.add(indexFile); } } @@ -163,6 +169,30 @@ public Plan scan() { } CoreOptions effectiveOptions = effectiveCoreOptions(); + Map> partitionVectors = new HashMap<>(); + for (IndexManifestEntry entry : indexEntries) { + if (isPrimaryColumn( + checkNotNull(entry.indexFile().globalIndexMeta()), vectorColumn.id())) { + partitionVectors + .computeIfAbsent(entry.partition(), p -> new ArrayList<>()) + .add(entry.indexFile()); + } + } + for (List files : partitionVectors.values()) { + if (files.stream().noneMatch(CentroidVectorIndexUtils::centroidBased)) { + continue; + } + IndexFileMeta routing = CentroidVectorIndexUtils.validateModel(files); + List coverage = CentroidVectorIndexUtils.metadata(routing).sourceRowRanges(); + long from = coverage.get(0).from; + long to = coverage.get(coverage.size() - 1).to; + List scalarFiles = scalarIndexFiles(allIndexFiles, coverage); + splits.add(new IndexVectorSearchSplit(from, to, files, scalarFiles)); + // FAST scalar fallback is restricted to actual source coverage, not the envelope. + for (Range range : coverage) { + vectorByRange.putIfAbsent(range, Collections.emptyList()); + } + } GlobalIndexSearchMode vectorSearchMode = effectiveOptions.vectorIndexSearchMode(); List rawRowRanges = new DataEvolutionGlobalIndexCoverage( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java index 355b60d8b75e..9560c8d7abd3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java @@ -20,6 +20,7 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexFileMetaSerializer; +import org.apache.paimon.index.IndexFileMetaV5Deserializer; import org.apache.paimon.io.DataInputViewStreamWrapper; import org.apache.paimon.io.DataOutputViewStreamWrapper; @@ -34,7 +35,7 @@ public class IndexVectorSearchSplit extends VectorSearchSplit { private static final long serialVersionUID = 1L; - private static final int VERSION = 1; + private static final int VERSION = 2; private static final ThreadLocal INDEX_SERIALIZER = ThreadLocal.withInitial(IndexFileMetaSerializer::new); @@ -85,15 +86,21 @@ private void writeObject(ObjectOutputStream out) throws IOException { private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); int version = in.readInt(); - if (version != VERSION) { + if (version != 1 && version != VERSION) { throw new IOException("Unsupported IndexVectorSearchSplit version: " + version); } IndexFileMetaSerializer serializer = INDEX_SERIALIZER.get(); DataInputViewStreamWrapper view = new DataInputViewStreamWrapper(in); this.rowRangeStart = view.readLong(); this.rowRangeEnd = view.readLong(); - this.vectorIndexFiles = serializer.deserializeList(view); - this.scalarIndexFiles = serializer.deserializeList(view); + if (version == 1) { + IndexFileMetaV5Deserializer legacy = new IndexFileMetaV5Deserializer(); + this.vectorIndexFiles = legacy.deserializeList(view); + this.scalarIndexFiles = legacy.deserializeList(view); + } else { + this.vectorIndexFiles = serializer.deserializeList(view); + this.scalarIndexFiles = serializer.deserializeList(view); + } } @Override diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java new file mode 100644 index 000000000000..c7a66e21b6c7 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.utils.Range; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Source coverage belongs to complete model generations, never to sparse shard bounds. */ +class CentroidVectorIndexUtilsTest { + @Test + void testSparseShardsDoNotClaimEnvelopeCoverage() { + List files = model("a"); + assertThat(CentroidVectorIndexUtils.validateModel(files)).isEqualTo(files.get(0)); + assertThat(CentroidVectorIndexUtils.coverage(files.get(0))) + .containsExactly(new Range(10, 19), new Range(1000, 1009)); + assertThat(CentroidVectorIndexUtils.coverage(files.get(1))).isEmpty(); + assertThat(CentroidVectorIndexUtils.coverage(files.get(2))).isEmpty(); + assertThat(CentroidVectorIndexUtils.centroidBased(file("{}", false))).isFalse(); + } + + @Test + void testIncompleteMixedAndMismatchedModelsAreRejected() { + List complete = model("a"); + assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(complete.subList(0, 2))) + .hasMessageContaining("Incomplete"); + List mixed = new ArrayList<>(complete); + mixed.add(file("{}", false)); + assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mixed)) + .hasMessageContaining("mix"); + List mismatch = new ArrayList<>(complete); + mismatch.set(2, model("b").get(2)); + assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mismatch)) + .hasMessageContaining("different models"); + List duplicate = new ArrayList<>(complete); + duplicate.add(complete.get(1)); + assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(duplicate)) + .hasMessageContaining("Duplicate"); + } + + private static List model(String id) { + return Arrays.asList( + file( + VectorIndexShardMeta.routing( + id, + 2, + Arrays.asList(new Range(10, 19), new Range(1000, 1009))) + .serialize(), + true), + file(VectorIndexShardMeta.centroid(id, 0).serialize(), false), + file(VectorIndexShardMeta.centroid(id, 1).serialize(), false)); + } + + private static IndexFileMeta file(String json, boolean routing) { + return file(json.getBytes(java.nio.charset.StandardCharsets.UTF_8), routing); + } + + private static IndexFileMeta file(byte[] meta, boolean routing) { + return new IndexFileMeta( + "ivf-pq", + Arrays.toString(meta), + 100, + routing ? 0 : 10, + null, + null, + new GlobalIndexMeta(10, 1009, 7, null, meta), + routing ? IndexFileMeta.ROUTING_MODEL : IndexFileMeta.DATA_FILE); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java index 33373741c667..51e7a742e5bf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java @@ -32,6 +32,57 @@ /** Test for {@link org.apache.paimon.index.IndexFileMetaSerializer}. */ public class IndexFileMetaSerializerTest extends ObjectSerializerTestBase { + @Test + void testRoutingModelKindRoundTrip() throws Exception { + IndexFileMeta routing = + new IndexFileMeta( + "ivf-pq", + "model.index", + 100, + 0, + null, + null, + new GlobalIndexMeta(10, 1000, 7, null, new byte[] {1}), + IndexFileMeta.ROUTING_MODEL); + IndexFileMetaSerializer serializer = new IndexFileMetaSerializer(); + assertThat(serializer.fromRow(serializer.toRow(routing))).isEqualTo(routing); + assertThat(serializer.fromRow(serializer.toRow(routing)).isRoutingModel()).isTrue(); + + org.apache.paimon.manifest.IndexManifestEntry entry = + new org.apache.paimon.manifest.IndexManifestEntry( + org.apache.paimon.manifest.FileKind.ADD, + org.apache.paimon.data.BinaryRow.EMPTY_ROW, + 0, + routing); + org.apache.paimon.manifest.IndexManifestEntrySerializer manifest = + new org.apache.paimon.manifest.IndexManifestEntrySerializer(); + assertThat(manifest.fromRow(manifest.toRow(entry))).isEqualTo(entry); + } + + @Test + void testLegacySevenFieldMetadataDefaultsToDataFile() throws Exception { + IndexFileMeta old = + globalIndexFile( + new GlobalIndexMeta(10, 1000, 7, null, new byte[] {1}, new byte[] {2})); + IndexFileMetaSerializer serializer = new IndexFileMetaSerializer(); + org.apache.paimon.types.RowType oldSchema = + new org.apache.paimon.types.RowType( + false, IndexFileMeta.SCHEMA.getFields().subList(0, 7)); + org.apache.paimon.io.DataOutputSerializer output = + new org.apache.paimon.io.DataOutputSerializer(128); + output.writeInt(1); + org.apache.paimon.data.serializer.InternalSerializers.create(oldSchema) + .serialize( + new org.apache.paimon.utils.OffsetRow(7, 0).replace(serializer.toRow(old)), + output); + assertThat( + new IndexFileMetaV5Deserializer() + .deserializeList( + new org.apache.paimon.io.DataInputDeserializer( + output.getCopyOfBuffer()))) + .containsExactly(old); + } + @Test void testGlobalIndexSourceMetaRoundTrip() { IndexFileMetaSerializer serializer = new IndexFileMetaSerializer(); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java index 268d7113330d..b3d2bb3eb8ac 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java @@ -33,6 +33,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -41,6 +42,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -197,6 +199,58 @@ public void testGlobalIndexOverlappingRangeRejectedWhenPreviousFileKept() throws .hasMessageContaining("overlapping row range"); } + @ParameterizedTest + @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) + void testCentroidModelsMustBeCompleteAndReplacedTogether(String type) { + List first = centroidModel("first", type); + IndexManifestFileHandler.GlobalIndexCombiner combiner = + new IndexManifestFileHandler.GlobalIndexCombiner(); + assertThat(combiner.combine(Collections.emptyList(), first)) + .containsExactlyInAnyOrderElementsOf(first); + assertThatThrownBy(() -> combiner.combine(Collections.emptyList(), first.subList(0, 2))) + .hasMessageContaining("Incomplete"); + assertThatThrownBy( + () -> + combiner.combine( + first, + Collections.singletonList(first.get(1).toDeleteEntry()))) + .hasMessageContaining("Incomplete"); + List replacement = centroidModel("second", type); + assertThatThrownBy(() -> combiner.combine(first, replacement)) + .hasMessageContaining("previous file"); + List changes = new ArrayList<>(); + first.forEach(entry -> changes.add(entry.toDeleteEntry())); + changes.addAll(replacement); + assertThat(combiner.combine(first, changes)) + .containsExactlyInAnyOrderElementsOf(replacement); + } + + private static List centroidModel(String id, String type) { + List entries = new ArrayList<>(); + for (int centroid = -1; centroid < 2; centroid++) { + byte[] metadata = + centroid == -1 + ? org.apache.paimon.globalindex.VectorIndexShardMeta.routing( + id, 2, Collections.singletonList(new Range(10, 100))) + .serialize() + : org.apache.paimon.globalindex.VectorIndexShardMeta.centroid( + id, centroid) + .serialize(); + IndexFileMeta file = + new IndexFileMeta( + type, + id + "-" + centroid, + 100, + centroid == -1 ? 0 : 10, + null, + null, + new GlobalIndexMeta(10, 100, 1, null, metadata), + centroid == -1 ? IndexFileMeta.ROUTING_MODEL : IndexFileMeta.DATA_FILE); + entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, file)); + } + return entries; + } + @Test public void testGlobalIndexOverlappingRangeAllowedAfterDelete() throws Exception { TestAppendFileStore fileStore = diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 5eb4db4c799f..30b50cb66cfe 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -1734,6 +1734,94 @@ void testCheckGlobalIndexRowIdExistenceNoConflict() { assertThat(exception).isNotPresent(); } + @Test + void testCentroidIndexChecksSourceCoverageRatherThanSparseEnvelope() { + byte[] meta = + org.apache.paimon.globalindex.VectorIndexShardMeta.routing( + "model", + 2, + Arrays.asList( + new org.apache.paimon.utils.Range(10, 19), + new org.apache.paimon.utils.Range(100, 109))) + .serialize(); + IndexFileMeta file = + new IndexFileMeta( + "ivf-pq", + "routing", + 100, + 0, + null, + null, + new GlobalIndexMeta( + 10, + 109, + 1, + null, + meta, + new org.apache.paimon.index.DataEvolutionIndexSourceMeta(1) + .serialize()), + IndexFileMeta.ROUTING_MODEL); + IndexManifestEntry index = new IndexManifestEntry(ADD, BinaryRow.EMPTY_ROW, 0, file); + List data = + Arrays.asList( + createFileEntryWithRowId("a", ADD, 10L, 10L), + createFileEntryWithRowId("b", ADD, 100L, 10L)); + assertThat( + createConflictDetection() + .checkConflicts( + snapshot(1), + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isNotPresent(); + assertThat( + createConflictDetection() + .checkConflicts( + snapshot(1), + data.subList(0, 1), + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isPresent(); + + Snapshot concurrent = snapshot(2); + SnapshotManager snapshots = mock(SnapshotManager.class); + when(snapshots.snapshot(2)).thenReturn(concurrent); + CommitScanner scanner = mock(CommitScanner.class); + ManifestEntry updated = manifestEntry(ADD, "updated", 10L, new Range(10, 19)); + when(scanner.readIncrementalEntries( + concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) + .thenReturn(Collections.singletonList(updated)); + ConflictDetection detection = + createConflictDetection(scanner, true, false, false, snapshots); + assertThat( + detection.checkConflicts( + concurrent, + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .hasValueSatisfying( + error -> assertThat(error).hasMessageContaining("source rows changed")); + ManifestEntry appended = manifestEntry(ADD, "appended", 200L, new Range(200, 209)); + when(scanner.readIncrementalEntries( + concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) + .thenReturn(Collections.singletonList(appended)); + assertThat( + detection.checkConflicts( + concurrent, + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isNotPresent(); + } + @Test void testCheckGlobalIndexRowIdExistenceBaseFileRemoved() { DataEvolutionConflictDetection detection = createConflictDetection(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java new file mode 100644 index 000000000000..e53540688eb9 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.globalindex; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.globalindex.GlobalIndexBuilderUtils; +import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; +import org.apache.paimon.globalindex.GlobalIndexer; +import org.apache.paimon.globalindex.IndexedSplit; +import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.globalindex.VectorIndexShardMeta; +import org.apache.paimon.globalindex.VectorIndexUtils; +import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.globalindex.VectorTrainingModelIO; +import org.apache.paimon.index.DataEvolutionIndexSourceMeta; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.options.Options; +import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.Range; + +import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; + +import org.apache.spark.HashPartitioner; +import org.apache.spark.TaskContext; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation; +import org.apache.spark.util.TaskCompletionListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.UUID; +import java.util.stream.Collectors; + +import scala.Tuple2; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Single-partition full build: shared training, centroid shuffle, then one atomic index commit. */ +public class CentroidGlobalIndexTopoBuilder implements GlobalIndexTopologyBuilder { + private static final Logger LOG = LoggerFactory.getLogger(CentroidGlobalIndexTopoBuilder.class); + + @Override + public List buildIndex( + SparkSession spark, + DataSourceV2Relation relation, + PartitionPredicate partitionPredicate, + FileStoreTable table, + String indexType, + RowType readType, + DataField indexField, + Options options) + throws IOException { + checkArgument( + org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType(indexType), + "Unsupported centroid index type: %s", + indexType); + checkArgument( + table.coreOptions().dataEvolutionEnabled(), + "Centroid sharding requires a Data Evolution table."); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + if (snapshot == null) { + return Collections.emptyList(); + } + List entries = + table.store() + .newScan() + .withSnapshot(snapshot) + .withPartitionFilter(partitionPredicate) + .plan() + .files(); + List partitions = + entries.stream() + .map(ManifestEntry::partition) + .distinct() + .collect(Collectors.toList()); + checkArgument( + partitions.size() <= 1, + "Centroid sharding supports a full build of exactly one partition."); + if (partitions.isEmpty()) { + return Collections.emptyList(); + } + BinaryRow partition = partitions.get(0); + List splits = + GlobalIndexBuilderUtils.createShardIndexedSplits( + table, entries, DefaultGlobalIndexTopoBuilder.rowsPerShard(options)); + if (splits.isEmpty()) { + return Collections.emptyList(); + } + List ranges = new ArrayList<>(); + List serializedSplits = new ArrayList<>(); + for (IndexedSplit split : splits) { + ranges.addAll(split.rowRanges()); + serializedSplits.add(InstantiationUtil.serializeObject(split)); + } + ranges = Range.sortAndMergeOverlap(ranges, true); + BuildContext context = + new BuildContext( + table, + readType, + indexField, + options, + partition, + snapshot.id(), + new Range(ranges.get(0).from, ranges.get(ranges.size() - 1).to), + indexType); + byte[] contextBytes = InstantiationUtil.serializeObject(context); + JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + JavaRDD rows = + sc.parallelize( + serializedSplits, + DefaultGlobalIndexTopoBuilder.parallelism(splits.size(), options)) + .mapPartitions( + input -> new VectorRows(BuildContext.restore(contextBytes), input)); + VectorGlobalIndexer indexer = context.indexer(); + byte[] payload = train(sc, contextBytes, rows, indexer, context.getter()); + int nlist; + try (VectorTrainingModel model = indexer.restoreTrainingModel(payload)) { + nlist = model.nlist(); + } + String modelId = UUID.randomUUID().toString(); + Broadcast broadcast = sc.broadcast(payload); + try { + int parallelism = DefaultGlobalIndexTopoBuilder.parallelism(nlist, options); + JavaPairRDD assigned = + rows.mapPartitionsToPair( + input -> + route( + BuildContext.restore(contextBytes), + input, + broadcast.value())); + // Seed every centroid so empty lists also have an explicit, verifiable shard file. + List> emptyLists = new ArrayList<>(); + for (int c = 0; c < nlist; c++) { + emptyLists.add(new Tuple2<>(c, new RoutedVector(-1, null))); + } + List results = + assigned.union(sc.parallelizePairs(emptyLists, parallelism)) + .repartitionAndSortWithinPartitions(new HashPartitioner(parallelism)) + .mapPartitions( + input -> + writeShards( + BuildContext.restore(contextBytes), + input, + broadcast.value(), + modelId)) + .collect(); + List messages = CommitMessageSerializer.deserializeAll(results); + GlobalIndexFileReadWrite io = context.io(); + String modelFile = + "vector-" + indexType + "-global-index-training-model-" + modelId + ".index"; + try (PositionOutputStream out = io.newOutputStream(modelFile)) { + VectorTrainingModelIO.write(out, payload, modelId); + } + IndexFileMeta modelMeta = + GlobalIndexBuilderUtils.toIndexFileMetas( + table.fileIO(), + table.store().pathFactory().globalIndexFileFactory(), + table.coreOptions(), + context.envelope, + Collections.singletonList(indexField), + indexType, + Collections.singletonList( + new ResultEntry( + modelFile, + 0, + VectorIndexShardMeta.routing( + modelId, nlist, ranges) + .serialize())), + context.sourceMeta()) + .get(0); + modelMeta = + new IndexFileMeta( + modelMeta.indexType(), + modelMeta.fileName(), + modelMeta.fileSize(), + 0, + null, + modelMeta.externalPath(), + modelMeta.globalIndexMeta(), + IndexFileMeta.ROUTING_MODEL); + messages.add(context.message(Collections.singletonList(modelMeta))); + List old = + GlobalIndexBuilderUtils.currentIndexEntries( + table, + snapshot, + indexType, + Collections.singletonList(indexField), + partitionPredicate) + .stream() + .map(IndexManifestEntry::indexFile) + .collect(Collectors.toList()); + if (!old.isEmpty()) { + messages.add( + new CommitMessageImpl( + partition, + 0, + null, + DataIncrement.deleteIndexIncrement(old), + CompactIncrement.emptyIncrement()) + .withCheckFromSnapshot(snapshot.id())); + } + return messages; + } finally { + broadcast.destroy(); + } + } + + @Override + public List buildIndex( + SparkSession spark, + DataSourceV2Relation relation, + PartitionPredicate partitionPredicate, + FileStoreTable table, + String indexType, + RowType readType, + DataField indexField, + List extraFields, + Options options) + throws IOException { + checkArgument(extraFields.isEmpty(), "Centroid sharding supports one vector column."); + return buildIndex( + spark, + relation, + partitionPredicate, + table, + indexType, + readType, + indexField, + options); + } + + private static byte[] train( + JavaSparkContext sc, + byte[] contextBytes, + JavaRDD rows, + VectorGlobalIndexer indexer, + InternalRow.FieldGetter getter) { + // Count actual non-null rows at the pinned snapshot. Row-ID spans can contain + // gaps, nulls and deletions, so a split's planned row count is insufficient. + List counts = + rows.mapPartitions( + input -> { + long count = 0; + while (input.hasNext()) { + input.next(); + count++; + } + return Collections.singletonList(count).iterator(); + }) + .collect(); + long[] offsets = new long[counts.size() + 1]; + for (int i = 0; i < counts.size(); i++) { + offsets[i + 1] = Math.addExact(offsets[i], counts.get(i)); + } + long total = offsets[counts.size()]; + long[] positions = indexer.planGlobalTrainingSample(total); + int sampleSize = positions.length; + LOG.info( + "Sampling {} of {} non-null vectors across {} Spark input partitions for centroid training", + sampleSize, + total, + counts.size()); + Broadcast plan = + sc.broadcast(new CentroidTrainingSample(positions)); + try { + JavaRDD> samples = + rows.mapPartitionsWithIndex( + (partition, input) -> + plan.value() + .select( + input, + getter, + offsets[partition], + offsets[partition + 1]), + false); + return JavaPairRDD.fromJavaRDD(samples) + .repartitionAndSortWithinPartitions(new HashPartitioner(1)) + .values() + .mapPartitions( + input -> { + try (VectorTrainingModel model = + BuildContext.restore(contextBytes) + .indexer() + .trainGlobalModel( + total, + CentroidTrainingSample.batches( + input, sampleSize))) { + return Collections.singletonList(model.serialize()).iterator(); + } + }) + .collect() + .get(0); + } finally { + plan.destroy(); + } + } + + private static Iterator> route( + BuildContext context, Iterator input, byte[] payload) { + VectorTrainingModel model = context.indexer().restoreTrainingModel(payload); + closeOnTaskCompletion(model); + InternalRow.FieldGetter getter = context.getter(); + int idPosition = context.rowIdPosition(); + return Iterators.transform( + input, + row -> { + float[] vector = VectorIndexUtils.toFloatArray(getter.getFieldOrNull(row)); + return new Tuple2<>( + model.assignCentroid(vector), + new RoutedVector(row.getLong(idPosition), vector)); + }); + } + + private static void closeOnTaskCompletion(AutoCloseable resource) { + TaskContext.get() + .addTaskCompletionListener( + (TaskCompletionListener) + ignored -> { + try { + resource.close(); + } catch (Exception e) { + throw new RuntimeException( + "Could not close centroid build resource", e); + } + }); + } + + private static Iterator writeShards( + BuildContext context, + Iterator> rows, + byte[] payload, + String modelId) + throws Exception { + List commits = new ArrayList<>(); + try (VectorTrainingModel model = context.indexer().restoreTrainingModel(payload)) { + GlobalIndexSingleColumnWriter writer = null; + Integer centroid = null; + try { + while (rows.hasNext()) { + Tuple2 row = rows.next(); + if (!row._1().equals(centroid)) { + if (writer != null) { + commits.add(context.finish(writer)); + close(writer); + } + centroid = row._1(); + writer = model.createCentroidWriter(context.io(), centroid, modelId); + } + if (row._2().vector != null) { + writer.write(row._2().vector, row._2().id); + } + } + if (writer != null) { + commits.add(context.finish(writer)); + } + } finally { + close(writer); + } + } + return commits.iterator(); + } + + private static void close(Object resource) throws IOException { + if (resource instanceof Closeable) { + ((Closeable) resource).close(); + } + } + + private static final class RoutedVector implements Serializable { + private static final long serialVersionUID = 1L; + private final long id; + private final float[] vector; + + RoutedVector(long id, float[] vector) { + this.id = id; + this.vector = vector; + } + } + + private static final class BuildContext implements Serializable { + private static final long serialVersionUID = 1L; + private final FileStoreTable table; + private final RowType readType; + private final DataField field; + private final Options options; + private final BinaryRow partition; + private final long snapshotId; + private final Range envelope; + private final String indexType; + + BuildContext( + FileStoreTable table, + RowType readType, + DataField field, + Options options, + BinaryRow partition, + long snapshotId, + Range envelope, + String indexType) { + this.table = table; + this.readType = readType; + this.field = field; + this.options = options; + this.partition = partition; + this.snapshotId = snapshotId; + this.envelope = envelope; + this.indexType = indexType; + } + + static BuildContext restore(byte[] bytes) throws IOException, ClassNotFoundException { + return InstantiationUtil.deserializeObject( + bytes, CentroidGlobalIndexTopoBuilder.class.getClassLoader()); + } + + VectorGlobalIndexer indexer() { + return (VectorGlobalIndexer) + GlobalIndexer.create(indexType, Collections.singletonList(field), options); + } + + GlobalIndexFileReadWrite io() { + return new GlobalIndexFileReadWrite( + table.fileIO(), table.store().pathFactory().globalIndexFileFactory()); + } + + int rowIdPosition() { + return readType.getFieldIndex(SpecialFields.ROW_ID.name()); + } + + InternalRow.FieldGetter getter() { + return InternalRow.createFieldGetter( + field.type(), readType.getFieldIndex(field.name())); + } + + boolean contains(IndexedSplit split, long id) { + for (Range range : split.rowRanges()) { + if (id >= range.from && id <= range.to) { + return true; + } + } + return false; + } + + CloseableIterator rows(IndexedSplit split) throws IOException { + return table.newReadBuilder() + .withReadType(readType) + .newRead() + .createReader(split) + .toCloseableIterator(); + } + + byte[] sourceMeta() { + return new DataEvolutionIndexSourceMeta(snapshotId).serialize(); + } + + CommitMessageImpl message(List files) { + return new CommitMessageImpl( + partition, + 0, + null, + DataIncrement.indexIncrement(files), + CompactIncrement.emptyIncrement()) + .withCheckFromSnapshot(snapshotId); + } + + byte[] finish(GlobalIndexSingleColumnWriter writer) throws IOException { + List files = + GlobalIndexBuilderUtils.toIndexFileMetas( + table.fileIO(), + table.store().pathFactory().globalIndexFileFactory(), + table.coreOptions(), + envelope, + Collections.singletonList(field), + indexType, + writer.finish(), + sourceMeta()); + return new CommitMessageSerializer().serialize(message(files)); + } + } + + private static final class VectorRows implements Iterator, Closeable { + private final BuildContext context; + private final Iterator splits; + private final int vectorPosition; + private final int idPosition; + private CloseableIterator rows; + private IndexedSplit split; + private InternalRow next; + private boolean closed; + + VectorRows(BuildContext context, Iterator splits) { + this.context = context; + this.splits = splits; + this.vectorPosition = context.readType.getFieldIndex(context.field.name()); + this.idPosition = context.rowIdPosition(); + closeOnTaskCompletion(this); + } + + @Override + public boolean hasNext() { + if (next != null) { + return true; + } + if (closed) { + return false; + } + try { + while (true) { + if (rows == null || !rows.hasNext()) { + if (rows != null) { + rows.close(); + rows = null; + } + if (!splits.hasNext()) { + close(); + return false; + } + split = + InstantiationUtil.deserializeObject( + splits.next(), + CentroidGlobalIndexTopoBuilder.class.getClassLoader()); + rows = context.rows(split); + continue; + } + InternalRow row = rows.next(); + long id = row.getLong(idPosition); + if (!context.contains(split, id)) { + continue; + } + if (row.isNullAt(vectorPosition)) { + continue; + } + next = row; + return true; + } + } catch (Exception e) { + try { + close(); + } catch (IOException closeFailure) { + e.addSuppressed(closeFailure); + } + throw new RuntimeException("Cannot read centroid build input.", e); + } + } + + @Override + public InternalRow next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + InternalRow result = next; + next = null; + return result; + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + try { + if (rows != null) { + rows.close(); + } + } catch (Exception e) { + throw new IOException(e); + } + } + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java new file mode 100644 index 000000000000..b4f73b0ce4fa --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.globalindex; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; +import org.apache.paimon.globalindex.VectorIndexUtils; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import scala.Tuple2; + +/** Look up retained source rows during a sequential split scan and shuffle by reservoir slot. */ +final class CentroidTrainingSample implements Serializable { + private static final long serialVersionUID = 1L; + private final long[] positions; + private final int[] slots; + + CentroidTrainingSample(long[] positionsBySlot) { + positions = positionsBySlot.clone(); + Arrays.sort(positions); + slots = new int[positions.length]; + for (int slot = 0; slot < slots.length; slot++) { + slots[Arrays.binarySearch(positions, positionsBySlot[slot])] = slot; + } + } + + private int lowerBound(long position) { + int i = Arrays.binarySearch(positions, position); + return i < 0 ? -i - 1 : i; + } + + Iterator> select( + Iterator input, InternalRow.FieldGetter getter, long start, long end) { + return new Iterator>() { + private long row = start; + private int sample = lowerBound(start); + private final int limit = lowerBound(end); + + @Override + public boolean hasNext() { + return sample < limit; + } + + @Override + public Tuple2 next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + while (row < positions[sample]) { + input.next(); + row++; + } + row++; + return new Tuple2<>( + slots[sample++], + VectorIndexUtils.toFloatArray(getter.getFieldOrNull(input.next()))); + } + }; + } + + static Iterator batches(Iterator sorted, int sampleSize) { + return new Iterator() { + private int remaining = sampleSize; + + @Override + public boolean hasNext() { + return remaining > 0; + } + + @Override + public TrainingBatch next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + float[] first = sorted.next(); + int dimension = first.length; + if (dimension == 0) { + throw new IllegalArgumentException("Empty training vector"); + } + int count = Math.min(remaining, Math.max(1, (256 * 1024) / dimension)); + float[] batch = new float[count * dimension]; + System.arraycopy(first, 0, batch, 0, dimension); + for (int i = 1; i < count; i++) { + float[] vector = sorted.next(); + if (vector.length != dimension) { + throw new IllegalArgumentException( + "Inconsistent training vector dimensions."); + } + System.arraycopy(vector, 0, batch, i * dimension, dimension); + } + remaining -= count; + return new TrainingBatch(batch, count); + } + }; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java index 7e5a401df302..6d500d4cf57f 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java @@ -118,6 +118,13 @@ public List buildIndex( List rowRangesToBuild = new ArrayList<>( GlobalIndexBuilderUtils.unindexedRowRanges(snapshot, currentIndexes)); + checkArgument( + currentIndexes.stream() + .noneMatch( + entry -> + org.apache.paimon.globalindex.CentroidVectorIndexUtils + .centroidBased(entry.indexFile())), + "Cannot build range shards alongside a centroid model; drop the existing index or perform a full centroid rebuild."); byte[] sourceMeta = new DataEvolutionIndexSourceMeta(snapshot.id()).serialize(); boolean detectDataFileChange = new Options(table.options(), options.toMap()).get(GLOBAL_INDEX_COLUMN_UPDATE_ACTION) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java index 405438c5936c..0429be8485ec 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java @@ -18,6 +18,7 @@ package org.apache.paimon.spark.globalindex; +import org.apache.paimon.options.Options; import org.apache.paimon.spark.globalindex.sorted.SortedIndexTopoBuilder; /** Utility class for creating {@link GlobalIndexTopologyBuilder} implementations. */ @@ -29,4 +30,36 @@ public static GlobalIndexTopologyBuilder createTopoBuilder(String indexType) { } return new DefaultGlobalIndexTopoBuilder(); } + + public static GlobalIndexTopologyBuilder createTopoBuilder(String indexType, Options options) { + return createTopoBuilder(indexType, options, null); + } + + public static GlobalIndexTopologyBuilder createTopoBuilder( + String indexType, Options options, String fieldName) { + String key = indexType + ".shard"; + String fieldKey = "fields." + fieldName + ".shard"; + java.util.Map dynamic = options.dynamicOptions(); + java.util.Map stored = options.toMap(); + String shard = + dynamic.containsKey(fieldKey) + ? dynamic.get(fieldKey) + : dynamic.containsKey(key) + ? dynamic.get(key) + : stored.getOrDefault( + fieldKey, stored.getOrDefault(key, "range-based")); + if (org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType(indexType)) { + if ("centroid-based".equals(shard)) { + return new CentroidGlobalIndexTopoBuilder(); + } + if (!"range-based".equals(shard)) { + throw new IllegalArgumentException("Unknown vector shard mode: " + shard); + } + } + if ("centroid-based".equals(shard)) { + throw new IllegalArgumentException( + "Centroid sharding requires a supported vector index."); + } + return createTopoBuilder(indexType); + } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java index ed349c65340e..6858a312e3db 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java @@ -186,7 +186,8 @@ public InternalRow[] call(InternalRow args) { } GlobalIndexTopologyBuilder topoBuilder = - GlobalIndexTopologyBuilderUtils.createTopoBuilder(indexType); + GlobalIndexTopologyBuilderUtils.createTopoBuilder( + indexType, userOptions, indexFields.get(0).name()); List indexResults = topoBuilder.buildIndex( diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java new file mode 100644 index 000000000000..439cca56a05e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.globalindex; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class CentroidTrainingSampleTest { + @Test + void restoresSlotOrderAcrossEmptyAndUnevenSplits() { + long[] bySlot = {155, 0, 99, 98, 156, 10}; + CentroidTrainingSample plan = new CentroidTrainingSample(bySlot); + TreeMap shuffled = new TreeMap<>(); + int start = 0; + for (int end : new int[] {0, 1, 3, 98, 98, 156, 200}) { + Iterator rows = + IntStream.range(start, end) + .mapToObj(i -> GenericRow.of(new float[] {i})) + .iterator(); + plan.select(rows, row -> ((GenericRow) row).getField(0), start, end) + .forEachRemaining( + pair -> { + assertThat(shuffled.put(pair._1(), pair._2())).isNull(); + }); + start = end; + } + assertThat(shuffled.keySet()).containsExactly(0, 1, 2, 3, 4, 5); + Iterator batches = + CentroidTrainingSample.batches(shuffled.values().iterator(), bySlot.length); + assertThat(batches.next().vectors).containsExactly(155, 0, 99, 98, 156, 10); + assertThat(batches.hasNext()).isFalse(); + } + + @Test + void batchesCrossMapBoundariesWithoutChangingRowOrder() { + int count = 270007; + Iterator rows = + IntStream.range(0, count).mapToObj(i -> new float[] {i, -i}).iterator(); + Iterator batches = CentroidTrainingSample.batches(rows, count); + int ordinal = 0; + List sizes = new ArrayList<>(); + while (batches.hasNext()) { + TrainingBatch batch = batches.next(); + sizes.add(batch.count); + for (int i = 0; i < batch.count; i++, ordinal++) { + if (batch.vectors[2 * i] != ordinal || batch.vectors[2 * i + 1] != -ordinal) { + throw new AssertionError("Reordered sample at " + ordinal); + } + } + } + assertThat(sizes).containsExactly(131072, 131072, count - 262144); + assertThat(ordinal).isEqualTo(count); + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java index 0645119462c9..42d8f9fc637e 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -47,6 +48,36 @@ void testFMIndexUsesDefaultTopologyBuilder() { .isInstanceOf(DefaultGlobalIndexTopoBuilder.class); } + @Test + void testCentroidTopologyRespectsFieldAndExplicitOptionPrecedence() { + Map stored = new HashMap<>(); + stored.put("ivf-pq.shard", "range-based"); + stored.put("fields.v.shard", "centroid-based"); + assertThat( + GlobalIndexTopologyBuilderUtils.createTopoBuilder( + "ivf-pq", new Options(stored), "v")) + .isInstanceOf(CentroidGlobalIndexTopoBuilder.class); + Options overridden = + new Options(stored, Collections.singletonMap("ivf-pq.shard", "range-based")); + assertThat(GlobalIndexTopologyBuilderUtils.createTopoBuilder("ivf-pq", overridden, "v")) + .isInstanceOf(DefaultGlobalIndexTopoBuilder.class); + assertThat( + GlobalIndexTopologyBuilderUtils.createTopoBuilder( + "ivf-pq", new Options(stored), "other")) + .isInstanceOf(DefaultGlobalIndexTopoBuilder.class); + for (String type : Arrays.asList("ivf-flat", "ivf-sq", "ivf-rq", "diskann")) { + assertThat( + GlobalIndexTopologyBuilderUtils.createTopoBuilder( + type, new Options(stored), "v")) + .isInstanceOf(CentroidGlobalIndexTopoBuilder.class); + } + assertThatThrownBy( + () -> + GlobalIndexTopologyBuilderUtils.createTopoBuilder( + "btree", new Options(stored), "v")) + .hasMessageContaining("supported vector index"); + } + @Test void testRowsPerShardUsesMergedBuildOptions() { Map tableOptions = new HashMap<>(); diff --git a/paimon-spark/paimon-spark-ut/pom.xml b/paimon-spark/paimon-spark-ut/pom.xml index ec37a4dad4fb..313f91367b11 100644 --- a/paimon-spark/paimon-spark-ut/pom.xml +++ b/paimon-spark/paimon-spark-ut/pom.xml @@ -96,6 +96,13 @@ under the License. test + + org.apache.paimon + paimon-vector + ${project.version} + test + + org.mockito mockito-core diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala new file mode 100644 index 000000000000..ab30cae1837a --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.globalindex.{VectorIndexShardMeta, VectorTrainingModelIO} +import org.apache.paimon.index.IndexFileMeta +import org.apache.paimon.spark.PaimonSparkTestBase + +import scala.collection.JavaConverters._ + +/** End-to-end centroid build over nonzero, disjoint source row-ID ranges. */ +class CentroidVectorIndexTest extends PaimonSparkTestBase { + + private def build(): Unit = { + spark + .sql("""CALL sys.create_global_index( + |table => 'test.T', index_column => 'v', index_type => 'ivf-pq', + |partitions => 'p=0', + |options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=4,ivf-pq.pq.m=1')""".stripMargin) + .collect() + } + + private def nearest(id: Int): Int = { + spark + .sql(s"""SELECT id FROM vector_search( + |'T', 'v', array($id.0f, ${id % 17}.0f), 1, + |map('ivf.nprobe', '1'))""".stripMargin) + .collect() + .head + .getInt(0) + } + + test("centroid build routes absolute IDs, preserves raw coverage and atomically rebuilds") { + withTable("T") { + spark.sql("""CREATE TABLE T (id INT, v ARRAY, p INT) PARTITIONED BY (p) + |TBLPROPERTIES ( + |'bucket' = '-1', + |'row-tracking.enabled' = 'true', + |'data-evolution.enabled' = 'true', + |'deletion-vectors.enabled' = 'true', + |'vector-index.search-mode' = 'detail', + |'ivf-pq.dimension' = '2', + |'ivf-pq.metric' = 'l2', + |'global-index.row-count-per-shard' = '64', + |'global-index.build.max-parallelism' = '2')""".stripMargin) + def insert(from: Int, to: Int, partition: Int): Unit = { + spark.sql(s"""INSERT INTO T SELECT cast(id AS INT), + |array(cast(id AS FLOAT), cast(id % 17 AS FLOAT)), $partition + |FROM range($from, $to)""".stripMargin) + } + insert(-64, 0, 2) + insert(0, 128, 0) + insert(128, 256, 1) + insert(256, 384, 0) + build() + val table = loadTable("T") + def indexes: Seq[IndexFileMeta] = + table + .store() + .newIndexFileHandler() + .scan(table.snapshotManager().latestSnapshot(), "ivf-pq") + .asScala + .map(_.indexFile()) + .toSeq + + val firstFiles = indexes.map(_.fileName()).toSet + assert(indexes.size == 5) + assert(indexes.count(_.isRoutingModel()) == 1) + assert(nearest(300) == 300) + // Partition 1 lies inside the indexed partition's min/max envelope, but is unindexed. + assert(nearest(200) == 200) + + build() + assert(indexes.size == 5) + assert(indexes.map(_.fileName()).toSet.intersect(firstFiles).isEmpty) + assert(nearest(300) == 300) + assert(nearest(200) == 200) + + intercept[Exception] { + spark + .sql( + """CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => 'ivf-pq', + |options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=4,ivf-pq.pq.m=1')""".stripMargin) + .collect() + } + assert(indexes.size == 5) + + spark + .sql("CALL sys.create_global_index(table => 'test.T', index_column => 'id', index_type => 'btree')") + .collect() + val filtered = spark + .sql("""SELECT id FROM vector_search('T', 'v', array(300.0f, 11.0f), 1, + |map('ivf.nprobe', '4')) WHERE id = 300""".stripMargin) + .collect() + assert(filtered.map(_.getInt(0)).toSeq == Seq(300)) + spark.sql("DELETE FROM T WHERE p = 0 AND id = 300") + assert(nearest(300) != 300) + } + } + test("distributed training preserves the model across sparse splits and different task counts") { + withTable("T") { + spark.sql("""CREATE TABLE T (id INT, v ARRAY) TBLPROPERTIES ( + |'bucket'='-1', 'row-tracking.enabled'='true', 'data-evolution.enabled'='true', + |'deletion-vectors.enabled'='true', 'vector-index.search-mode'='fast', + |'ivf-flat.dimension'='2', 'ivf-flat.metric'='l2', + |'global-index.row-count-per-shard'='32', + |'global-index.build.max-parallelism'='4')""".stripMargin) + spark.sql("""INSERT INTO T SELECT cast(id AS INT), + |CASE WHEN id = 4 OR id >= 192 THEN array(cast(id AS FLOAT), 1.0f) + |ELSE cast(NULL AS ARRAY) END FROM range(256)""".stripMargin) + spark.sql("DELETE FROM T WHERE id = 200") + def buildModel(): Array[Byte] = { + spark.sql( + """CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => 'ivf-flat', + |options => 'ivf-flat.shard=centroid-based,ivf-flat.nlist=2,ivf-flat.train.sample-ratio=0.125')""".stripMargin) + val table = loadTable("T") + val model = table + .store() + .newIndexFileHandler() + .scan(table.snapshotManager().latestSnapshot(), "ivf-flat") + .asScala + .map(_.indexFile()) + .find(_.isRoutingModel()) + .get + val meta = VectorIndexShardMeta.deserialize(model.globalIndexMeta().indexMeta()) + val path = table.store().pathFactory().globalIndexFileFactory().toPath(model) + val in = table.fileIO().newInputStream(path) + try VectorTrainingModelIO.read(in, model.fileSize(), meta.modelId()) + finally in.close() + } + val fourTasks = buildModel() + spark.sql("ALTER TABLE T SET TBLPROPERTIES ('global-index.build.max-parallelism'='2')") + assert(fourTasks.sameElements(buildModel())) + def nearest(id: Int): Int = + spark + .sql(s"""SELECT id FROM vector_search('T','v',array($id.0f,1.0f),1, + |map('centroid.nprobe','2'))""".stripMargin) + .collect() + .head + .getInt(0) + assert(nearest(4) == 4) + assert(nearest(255) == 255) + assert(nearest(200) != 200) + } + } + + Seq("ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann").foreach { + kind => + test(s"$kind centroid SQL build, read, filter and range rebuild") { + withTable("T") { + spark.sql(s"""CREATE TABLE T (id INT, v ARRAY) + |TBLPROPERTIES ('bucket'='-1', 'row-tracking.enabled'='true', + |'data-evolution.enabled'='true', 'deletion-vectors.enabled'='true', + |'vector-index.search-mode'='fast', '$kind.dimension'='4', + |'$kind.metric'='l2', 'global-index.row-count-per-shard'='64', + |'global-index.build.max-parallelism'='2')""".stripMargin) + spark.sql( + """INSERT INTO T SELECT cast(id AS INT), + |array(cast(id AS FLOAT), cast(id % 17 AS FLOAT), + |cast(id % 23 AS FLOAT), cast(id % 31 AS FLOAT)) FROM range(0, 256)""".stripMargin) + val pq = if (kind == "ivf-pq" || kind == "diskann") s",$kind.pq.m=1" else "" + def create(mode: String): Unit = + spark + .sql(s"""CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => '$kind', + |options => '$kind.shard=$mode,$kind.nlist=4$pq')""".stripMargin) + .collect() + def nearest(mode: String): Array[Int] = { + val opts = + if (kind == "diskann" && mode == "centroid-based") + "map('centroid.nprobe','4','diskann.l_search','256')" + else if (kind == "diskann") "map('diskann.l_search','256')" + else if (mode == "centroid-based") "map('centroid.nprobe','2','ivf.nprobe','3')" + else "map('ivf.nprobe','4')" + spark + .sql(s"""SELECT id FROM vector_search('T','v',array(200.0f,13.0f,16.0f,14.0f),1, + |$opts) WHERE id >= 192""".stripMargin) + .collect() + .map(_.getInt(0)) + } + create("centroid-based") + val table = loadTable("T") + val files = table + .store() + .newIndexFileHandler() + .scan(table.snapshotManager().latestSnapshot(), kind) + .asScala + .map(_.indexFile()) + assert(files.size == 5 && files.count(_.isRoutingModel()) == 1) + assert(nearest("centroid-based").toSeq == Seq(200)) + val centroids = files + .filterNot(_.isRoutingModel()) + .map(f => VectorIndexShardMeta.deserialize(f.globalIndexMeta().indexMeta()).centroid()) + assert(centroids.toSet == Set(0, 1, 2, 3)) + spark + .sql(s"""CALL sys.drop_global_index(table => 'test.T', index_column => 'v', + |index_type => '$kind')""".stripMargin) + .collect() + create("range-based") + assert(nearest("range-based").toSeq == Seq(200)) + } + } + } + +} diff --git a/paimon-vector/pom.xml b/paimon-vector/pom.xml index 058c0c934dcc..760ec4a6695d 100644 --- a/paimon-vector/pom.xml +++ b/paimon-vector/pom.xml @@ -32,7 +32,7 @@ under the License. Paimon : Vector Index - 0.5.0 + 0.7.0-SNAPSHOT diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java new file mode 100644 index 000000000000..d77128a8a450 --- /dev/null +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.globalindex.AbstractVectorGlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.ScoredGlobalIndexResult; +import org.apache.paimon.globalindex.VectorIndexShardMeta; +import org.apache.paimon.globalindex.VectorTrainingModelIO; +import org.apache.paimon.globalindex.io.GlobalIndexFileReader; +import org.apache.paimon.index.vector.CentroidRouter; +import org.apache.paimon.predicate.BatchVectorSearch; +import org.apache.paimon.predicate.VectorSearch; +import org.apache.paimon.types.DataType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Routes before opening shard files; all filters and results use absolute Paimon row IDs. */ +final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexReader { + private final GlobalIndexFileReader files; + private final GlobalIndexIOMeta routing; + private final VectorIndexShardMeta metadata; + private final Map shards = new HashMap<>(); + private final Map readers = new HashMap<>(); + private final DataType fieldType; + private final ExecutorService executor; + private final long vectorCount; + private final String indexType; + private CentroidRouter model; + private boolean closed; + + NativeCentroidRoutedIndexReader( + GlobalIndexFileReader files, + List entries, + DataType fieldType, + ExecutorService executor, + String indexType) { + this.files = files; + this.fieldType = fieldType; + this.executor = executor; + this.indexType = indexType; + GlobalIndexIOMeta routing = null; + long count = 0; + for (GlobalIndexIOMeta file : entries) { + VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(file.metadata()); + checkArgument( + meta.centroidBased(), "Cannot mix centroid-based and range-based shards."); + if (meta.routingModel()) { + checkArgument(routing == null, "Multiple routing models."); + routing = file; + } else { + checkArgument( + file.rowCount() >= 0, "Centroid shards must record their vector count."); + checkArgument( + shards.put(meta.centroid(), file) == null, "Duplicate centroid shard."); + count = Math.addExact(count, file.rowCount()); + } + } + checkArgument(routing != null, "Missing routing model."); + this.routing = routing; + this.metadata = VectorIndexShardMeta.deserialize(routing.metadata()); + checkArgument(shards.size() == metadata.nlist(), "Incomplete centroid shard set."); + for (Map.Entry shard : shards.entrySet()) { + VectorIndexShardMeta data = + VectorIndexShardMeta.deserialize(shard.getValue().metadata()); + checkArgument( + shard.getKey() < metadata.nlist() && metadata.modelId().equals(data.modelId()), + "Mismatched centroid model."); + } + this.vectorCount = count; + } + + private synchronized CentroidRouter model() { + checkArgument(!closed, "Centroid reader is closed."); + if (model == null) { + try (SeekableInputStream in = files.getInputStream(routing)) { + CentroidRouter loaded = + CentroidRouter.deserialize( + VectorTrainingModelIO.read( + in, + routing.fileSize(), + metadata.modelId(), + CentroidRouter.maxSerializedBytes())); + if (loaded.nlist() != metadata.nlist()) { + loaded.close(); + throw new IOException("Routing nlist differs from manifest metadata."); + } + model = loaded; + } catch (IOException e) { + throw new RuntimeException("Cannot read centroid routing model.", e); + } + } + return model; + } + + private synchronized NativeVectorGlobalIndexReader reader(int centroid) { + checkArgument(!closed, "Centroid reader is closed."); + return readers.computeIfAbsent( + centroid, + c -> + new NativeVectorGlobalIndexReader( + files, + Collections.singletonList(shards.get(c)), + fieldType, + executor, + indexType)); + } + + private int initialWidth(VectorSearch query) { + String width = query.options().get("centroid.nprobe"); + if ("diskann".equals(indexType) && query.options().containsKey("ivf.nprobe")) { + throw new IllegalArgumentException( + "Use centroid.nprobe for DiskANN outer routing and diskann.l_search for graph search"); + } + org.apache.paimon.index.vector.VectorSearchParams params = + width == null + ? org.apache.paimon.index.vector.VectorSearchParams.automatic(query.limit()) + : org.apache.paimon.index.vector.VectorSearchParams.ivf( + query.limit(), Integer.parseInt(width)); + return model().resolveNprobe( + params, + vectorCount, + query.includeRowIds() == null + ? -1 + : query.includeRowIds().getLongCardinality()); + } + + private boolean automatic(VectorSearch query) { + return !query.options().containsKey("centroid.nprobe"); + } + + @Override + public CompletableFuture> visitVectorSearch( + VectorSearch query) { + if (query.includeRowIds() != null && query.includeRowIds().isEmpty()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return CompletableFuture.supplyAsync(() -> initialWidth(query), executor) + .thenCompose( + width -> + search( + query, + width, + Collections.emptyList(), + Collections.emptyList())); + } + + private CompletableFuture> search( + VectorSearch query, + int width, + List visited, + List previous) { + int[] selected = model().findNearestCentroids(query.vector(), width); + List nextVisited = new ArrayList<>(visited); + List>> pending = new ArrayList<>(); + for (int centroid : selected) { + if (!visited.contains(centroid)) { + nextVisited.add(centroid); + pending.add(reader(centroid).visitVectorSearch(query)); + } + } + return CompletableFuture.allOf(pending.toArray(new CompletableFuture[0])) + .thenCompose( + ignored -> { + List results = new ArrayList<>(previous); + for (CompletableFuture> future : + pending) { + future.join().ifPresent(results::add); + } + ScoredGlobalIndexResult merged = + ScoredGlobalIndexResult.merge(results).topK(query.limit()); + if (automatic(query) + && merged.results().getLongCardinality() < query.limit() + && width < metadata.nlist()) { + return search( + query, + (int) Math.min(metadata.nlist(), (long) width * 2), + nextVisited, + results); + } + return CompletableFuture.completedFuture( + merged.results().isEmpty() + ? Optional.empty() + : Optional.of(merged)); + }); + } + + @Override + public CompletableFuture>> visitBatchVectorSearch( + BatchVectorSearch batch) { + int count = batch.vectorCount(); + checkArgument(count > 0, "Batch vector count must be positive."); + if (batch.includeRowIds() != null && batch.includeRowIds().isEmpty()) { + return CompletableFuture.completedFuture(Collections.nCopies(count, Optional.empty())); + } + return CompletableFuture.supplyAsync( + () -> { + Map> groups = new LinkedHashMap<>(); + for (int i = 0; i < count; i++) { + VectorSearch query = batch.forIndex(i); + for (int centroid : + model().findNearestCentroids( + query.vector(), initialWidth(query))) { + groups.computeIfAbsent(centroid, c -> new ArrayList<>()).add(i); + } + } + return groups; + }, + executor) + .thenCompose( + groups -> { + List> results = new ArrayList<>(); + for (int i = 0; i < count; i++) { + results.add(new ArrayList<>()); + } + List> pending = new ArrayList<>(); + for (Map.Entry> group : groups.entrySet()) { + float[][] queries = new float[group.getValue().size()][]; + for (int i = 0; i < queries.length; i++) { + queries[i] = batch.vectors()[group.getValue().get(i)]; + } + BatchVectorSearch subset = + new BatchVectorSearch( + queries, + batch.limit(), + batch.fieldName(), + batch.options()) + .withIncludeRowIds(batch.includeRowIds()); + pending.add( + reader(group.getKey()) + .visitBatchVectorSearch(subset) + .thenAccept( + found -> { + synchronized (results) { + for (int i = 0; + i < found.size(); + i++) { + final int inputIndex = + group.getValue().get(i); + found.get(i) + .ifPresent( + value -> + results.get( + inputIndex) + .add( + value)); + } + } + })); + } + return CompletableFuture.allOf( + pending.toArray(new CompletableFuture[0])) + .thenCompose( + ignored -> { + List< + CompletableFuture< + Optional< + ScoredGlobalIndexResult>>> + merged = new ArrayList<>(); + for (int i = 0; i < count; i++) { + VectorSearch query = batch.forIndex(i); + ScoredGlobalIndexResult result = + ScoredGlobalIndexResult.merge( + results.get(i)) + .topK(batch.limit()); + if (automatic(query) + && result.results().getLongCardinality() + < batch.limit()) { + merged.add(visitVectorSearch(query)); + } else { + merged.add( + CompletableFuture.completedFuture( + result.results().isEmpty() + ? Optional.empty() + : Optional.of( + result))); + } + } + return CompletableFuture.allOf( + merged.toArray( + new CompletableFuture[0])) + .thenApply( + done -> { + List< + Optional< + ScoredGlobalIndexResult>> + output = + new ArrayList<>(); + for (CompletableFuture< + Optional< + ScoredGlobalIndexResult>> + result : merged) { + output.add(result.join()); + } + return output; + }); + }); + }); + } + + @Override + public synchronized void close() throws IOException { + closed = true; + IOException failure = null; + for (NativeVectorGlobalIndexReader reader : readers.values()) { + try { + reader.close(); + } catch (IOException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (model != null) { + model.close(); + model = null; + } + readers.clear(); + if (failure != null) { + throw failure; + } + } +} diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java index 7b9c2d7872b3..a6219b6c0a55 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java @@ -22,9 +22,8 @@ import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadUtils; import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.globalindex.AbstractVectorGlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexIOMeta; -import org.apache.paimon.globalindex.GlobalIndexReader; -import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.index.vector.VectorIndexInput; @@ -34,7 +33,6 @@ import org.apache.paimon.index.vector.VectorSearchParams; import org.apache.paimon.index.vector.VectorSearchResult; import org.apache.paimon.predicate.BatchVectorSearch; -import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.VectorSearch; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; @@ -45,6 +43,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -61,7 +60,7 @@ *

Each shard has exactly one vector index file. The reader lazily opens the index and performs * vector similarity search. */ -public class NativeVectorGlobalIndexReader implements GlobalIndexReader { +public class NativeVectorGlobalIndexReader extends AbstractVectorGlobalIndexReader { private static final String NPROBE_PARAMETER = "ivf.nprobe"; private static final String L_SEARCH_PARAMETER = "diskann.l_search"; @@ -77,6 +76,7 @@ public class NativeVectorGlobalIndexReader implements GlobalIndexReader { private final GlobalIndexFileReader fileReader; private final DataType fieldType; private final ExecutorService executor; + private final String expectedIndexType; private volatile VectorIndexMetadata nativeMeta; private volatile VectorIndexReader vectorReader; @@ -87,16 +87,29 @@ public NativeVectorGlobalIndexReader( List ioMetas, DataType fieldType, ExecutorService executor) { + this(fileReader, ioMetas, fieldType, executor, null); + } + + NativeVectorGlobalIndexReader( + GlobalIndexFileReader fileReader, + List ioMetas, + DataType fieldType, + ExecutorService executor, + String expectedIndexType) { checkArgument(ioMetas.size() == 1, "Expected exactly one index file per shard"); this.executor = executor; this.fileReader = fileReader; this.ioMeta = ioMetas.get(0); this.fieldType = fieldType; + this.expectedIndexType = expectedIndexType; } @Override public CompletableFuture> visitVectorSearch( VectorSearch vectorSearch) { + if (ioMeta.rowCount() == 0) { + return CompletableFuture.completedFuture(Optional.empty()); + } return CompletableFuture.supplyAsync( () -> { try { @@ -116,6 +129,10 @@ public CompletableFuture> visitVectorSearch( @Override public CompletableFuture>> visitBatchVectorSearch( BatchVectorSearch batchVectorSearch) { + if (ioMeta.rowCount() == 0) { + return CompletableFuture.completedFuture( + Collections.nCopies(batchVectorSearch.vectorCount(), Optional.empty())); + } return CompletableFuture.supplyAsync( () -> { try { @@ -377,8 +394,16 @@ private void ensureLoaded() throws IOException { SeekableInputStream in = fileReader.getInputStream(ioMeta); VectorIndexReader reader = null; try { - reader = new VectorIndexReader(new SeekableStreamVectorIndexInput(in)); + VectorIndexInput input = new SeekableStreamVectorIndexInput(in); + reader = new VectorIndexReader(input); nativeMeta = reader.metadata(); + if (expectedIndexType != null + && !expectedIndexType.equals( + nativeMeta.indexType().replace('_', '-'))) { + throw new IOException( + "Index file type differs from manifest index type: " + + ioMeta.filePath()); + } reader.optimizeForSearch(); vectorReader = reader; openStream = in; @@ -530,88 +555,4 @@ private static boolean areRangesNonOverlapping(long[] positions, byte[][] buffer return true; } } - - // =================== unsupported ===================== - - @Override - public CompletableFuture> visitIsNotNull(FieldRef fieldRef) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitIsNull(FieldRef fieldRef) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitStartsWith( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitEndsWith( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitContains( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitLike( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitLessThan( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitGreaterOrEqual( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitNotEqual( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitLessOrEqual( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitEqual( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitGreaterThan( - FieldRef fieldRef, Object literal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitIn( - FieldRef fieldRef, List literals) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @Override - public CompletableFuture> visitNotIn( - FieldRef fieldRef, List literals) { - return CompletableFuture.completedFuture(Optional.empty()); - } } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java index 5a7f27ee7103..28fd77094e9a 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java @@ -24,7 +24,9 @@ import org.apache.paimon.fs.PositionOutputStreamWrapper; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.vector.CentroidRouter; import org.apache.paimon.index.vector.VectorIndexTrainer; import org.apache.paimon.index.vector.VectorIndexTraining; import org.apache.paimon.index.vector.VectorIndexWriter; @@ -40,6 +42,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; @@ -76,6 +79,7 @@ public class NativeVectorGlobalIndexWriter implements GlobalIndexSingleColumnWri private final Map nativeOptions; private final int dim; private final double trainSampleRatio; + private final VectorIndexShardMeta shardMeta; private File tempVectorFile; private FileChannel writeChannel; @@ -107,6 +111,17 @@ public NativeVectorGlobalIndexWriter( Map options, String identifier, double trainSampleRatio) { + this(fileWriter, fieldType, options, identifier, trainSampleRatio, null); + } + + NativeVectorGlobalIndexWriter( + GlobalIndexFileWriter fileWriter, + DataType fieldType, + Map options, + String identifier, + double trainSampleRatio, + VectorIndexShardMeta shardMeta) { + this.shardMeta = shardMeta; this.fileWriter = fileWriter; this.identifier = identifier; validateFieldType(fieldType); @@ -161,18 +176,22 @@ private void validateFieldType(DataType dataType) { } @Override - public void write(Object fieldData, long relativeRowId) { + public void write(Object fieldData, long rowId) { if (fieldData == null) { rowCount++; return; } - float[] src = materializeAndValidate(fieldData, relativeRowId); + if (shardMeta != null && rowId < 0) { + throw new IllegalArgumentException( + "Centroid shards require nonnegative absolute row IDs."); + } + float[] src = materializeAndValidate(fieldData, rowId); if (writeBuf.remaining() < recordSizeInBytes) { flushWriteBuffer(); } - writeBuf.putLong(relativeRowId); + writeBuf.putLong(rowId); for (int i = 0; i < dim; i++) { writeBuf.putFloat(src[i]); } @@ -234,7 +253,14 @@ public List finish() { writeChannel.close(); writeChannel = null; writeBuf = null; - return Collections.emptyList(); + if (shardMeta == null) { + return Collections.emptyList(); + } + String name = fileWriter.newFileName(fileNamePrefix()); + try (PositionOutputStream out = fileWriter.newOutputStream(name)) { + out.write(new byte[] {'P', 'C', 'E', 'M', 'P', 'T', 'Y', '1'}); + } + return Collections.singletonList(new ResultEntry(name, 0, shardMeta.serialize())); } flushWriteBuffer(); writeChannel.close(); @@ -309,8 +335,10 @@ public void write(byte[] b) throws IOException { identifier, System.currentTimeMillis() - buildStart); - VectorIndexMeta meta = new VectorIndexMeta(); - return new ResultEntry(fileName, rowCount, meta.serialize()); + return new ResultEntry( + fileName, + shardMeta == null ? rowCount : count, + shardMeta == null ? new VectorIndexMeta().serialize() : shardMeta.serialize()); } } @@ -319,13 +347,37 @@ private String fileNamePrefix() { } private VectorIndexTraining trainFromTempFile() throws IOException { - int trainCount = trainingVectorCount(count, trainSampleRatio); - int trainBatchSize = vectorBatchSize(TRAIN_BATCH_SIZE, dim); + Map options = trainingOptions(nativeOptions, count); + if (shardMeta != null) { + options.put("expected-vector-count", Long.toString(count)); + String nlist = options.get("nlist"); + if (nlist != null && !"auto".equals(nlist)) { + options.put( + "nlist", + Integer.toString( + Math.min( + trainingVectorCount(count, trainSampleRatio), + Integer.parseInt(nlist)))); + } + } + if ("diskann".equals(identifier)) { + // nlist belongs to the global router, not an independent DiskANN graph. + options.remove("nlist"); + } + try (VectorIndexTrainer trainer = VectorIndexTrainer.create(options)) { + int trainCount = trainingVectorCount(count, trainSampleRatio); + logTrainingMemoryEstimate(trainCount); + addTrainingBatches(trainer, trainCount); + return trainer.finishTraining(); + } + } + + private void addTrainingBatches(VectorIndexTrainer trainer, long trainCount) + throws IOException { + int trainBatchSize = Math.min(TRAIN_BATCH_SIZE, IO_BUFFER_SIZE / recordSizeInBytes); float[] batchVectors = new float[trainBatchSize * dim]; - logTrainingMemoryEstimate(trainCount); - try (VectorIndexTrainer trainer = VectorIndexTrainer.create(trainingOptions()); - RandomAccessFile raf = new RandomAccessFile(tempVectorFile, "r"); + try (RandomAccessFile raf = new RandomAccessFile(tempVectorFile, "r"); FileChannel channel = raf.getChannel()) { ByteBuffer readBuf = ByteBuffer.allocateDirect(IO_BUFFER_SIZE); readBuf.order(ByteOrder.nativeOrder()); @@ -369,13 +421,27 @@ private VectorIndexTraining trainFromTempFile() throws IOException { + " training vectors, but selected " + selected); } + } + } - return trainer.finishTraining(); + CentroidRouter finishRoutingModel(NativeVectorGlobalIndexer indexer) throws IOException { + if (count == 0) { + throw new IllegalArgumentException( + "Cannot train a centroid model without non-null vectors."); + } + flushWriteBuffer(); + try (VectorIndexTrainer trainer = indexer.createCoarseTrainer(count)) { + addTrainingBatches(trainer, trainingCandidateCount(count, trainSampleRatio)); + try (VectorIndexTraining coarse = trainer.finishTraining()) { + return CentroidRouter.fromTraining(coarse); + } } } - private Map trainingOptions() { + private static Map trainingOptions( + Map nativeOptions, long count) { Map options = new LinkedHashMap<>(nativeOptions); + options.remove("centroid.local-nlist"); String indexType = options.get("index.type"); String nlist = options.get("nlist"); if (indexType != null @@ -482,6 +548,17 @@ private static int checkedRecordSize(int dim, int bufferCapacity) { } static int trainingVectorCount(long vectorCount, double trainSampleRatio) { + long trainCount = trainingCandidateCount(vectorCount, trainSampleRatio); + if (trainCount > Integer.MAX_VALUE) { + throw new IllegalStateException( + "Training vector count " + + trainCount + + " exceeds Java integer capacity. Reduce train.sample-ratio."); + } + return (int) trainCount; + } + + static long trainingCandidateCount(long vectorCount, double trainSampleRatio) { if (vectorCount <= 0) { return 0; } @@ -494,14 +571,7 @@ static int trainingVectorCount(long vectorCount, double trainSampleRatio) { + trainSampleRatio); } long trainCount = (long) Math.ceil(vectorCount * trainSampleRatio); - trainCount = Math.max(1L, Math.min(vectorCount, trainCount)); - if (trainCount > Integer.MAX_VALUE) { - throw new IllegalStateException( - "Training vector count " - + trainCount - + " exceeds Java integer capacity. Reduce train.sample-ratio."); - } - return (int) trainCount; + return Math.max(1L, Math.min(vectorCount, trainCount)); } static int vectorBatchSize(int requestedBatchSize, int dim) { @@ -554,9 +624,15 @@ private static long saturatedMultiply(long left, long right) { return left * right; } - static long sampleIndex(long sampleOrdinal, long vectorCount, int trainCount) { + static long sampleIndex(long sampleOrdinal, long vectorCount, long trainCount) { long quotient = vectorCount / trainCount; long remainder = vectorCount % trainCount; + if (sampleOrdinal != 0 && remainder > Long.MAX_VALUE / sampleOrdinal) { + return BigInteger.valueOf(sampleOrdinal) + .multiply(BigInteger.valueOf(vectorCount)) + .divide(BigInteger.valueOf(trainCount)) + .longValueExact(); + } return sampleOrdinal * quotient + sampleOrdinal * remainder / trainCount; } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java index 8d7c3e4ab924..b5c180d1115b 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java @@ -20,13 +20,22 @@ import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; import org.apache.paimon.globalindex.VectorGlobalIndexer; +import org.apache.paimon.globalindex.VectorGlobalModelTrainer; +import org.apache.paimon.globalindex.VectorIndexShardMeta; +import org.apache.paimon.globalindex.VectorTrainingModel; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.vector.CentroidRouter; +import org.apache.paimon.index.vector.VectorIndexTrainer; +import org.apache.paimon.index.vector.VectorIndexTraining; import org.apache.paimon.types.DataType; import org.apache.paimon.utils.Range; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -40,6 +49,7 @@ public class NativeVectorGlobalIndexer implements VectorGlobalIndexer { private final DataType fieldType; private final Map options; private final String identifier; + private final String localNlist; private final double trainSampleRatio; public NativeVectorGlobalIndexer( @@ -59,6 +69,13 @@ public NativeVectorGlobalIndexer( this.fieldType = fieldType; this.options = Objects.requireNonNull(options, "options must not be null"); this.identifier = Objects.requireNonNull(identifier, "identifier must not be null"); + this.localNlist = this.options.get("centroid.local-nlist"); + if (localNlist != null + && ("diskann".equals(identifier) + || (!"auto".equals(localNlist) && Integer.parseInt(localNlist) <= 0))) { + throw new IllegalArgumentException( + "centroid.local-nlist requires local IVF and a positive count or auto."); + } if (Double.isNaN(trainSampleRatio) || Double.isInfinite(trainSampleRatio) || trainSampleRatio <= 0 @@ -83,7 +100,103 @@ public GlobalIndexReader createReader( long totalRowCount, List rowRanges, ExecutorService executor) { - return new NativeVectorGlobalIndexReader(fileReader, files, fieldType, executor); + if (files.stream() + .anyMatch( + file -> + VectorIndexShardMeta.deserialize(file.metadata()) + .centroidBased())) { + return new NativeCentroidRoutedIndexReader( + fileReader, files, fieldType, executor, identifier); + } + return new NativeVectorGlobalIndexReader( + fileReader, files, fieldType, executor, identifier); + } + + @Override + public long[] planGlobalTrainingSample(long vectorCount) { + long candidates = + NativeVectorGlobalIndexWriter.trainingCandidateCount(vectorCount, trainSampleRatio); + if (candidates == 0) { + throw new IllegalArgumentException( + "Cannot train a centroid model without non-null vectors."); + } + try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { + long[] positions = trainer.planTrainingSample(candidates); + for (int i = 0; i < positions.length; i++) { + positions[i] = + NativeVectorGlobalIndexWriter.sampleIndex( + positions[i], vectorCount, candidates); + } + return positions; + } + } + + VectorIndexTrainer createCoarseTrainer(long vectorCount) { + Map coarse = new LinkedHashMap<>(); + coarse.put("index.type", "ivf_flat"); + coarse.put("dimension", options.get("dimension")); + coarse.put("metric", options.get("metric")); + coarse.put("ivf.coarse-assignment", "exact"); + String nlist = options.getOrDefault("nlist", "auto"); + if ("diskann".equals(identifier) && "auto".equals(nlist.trim())) { + nlist = "64"; + } + coarse.put("nlist", nlist); + coarse.put( + "expected-vector-count", + options.getOrDefault("expected-vector-count", Long.toString(vectorCount))); + if (options.containsKey("ivf.train.max-points-per-centroid")) { + coarse.put( + "ivf.train.max-points-per-centroid", + options.get("ivf.train.max-points-per-centroid")); + } + return VectorIndexTrainer.create(coarse); + } + + GlobalIndexSingleColumnWriter createLocalWriter( + GlobalIndexFileWriter fileWriter, int centroid, String modelId) { + Map local = new LinkedHashMap<>(options); + if ("diskann".equals(identifier)) { + local.remove("nlist"); + } else { + local.put("nlist", localNlist == null ? "auto" : localNlist); + } + local.remove("expected-vector-count"); + return new NativeVectorGlobalIndexWriter( + fileWriter, + fieldType, + local, + identifier, + trainSampleRatio, + VectorIndexShardMeta.centroid(modelId, centroid)); + } + + @Override + public VectorTrainingModel trainGlobalModel(long vectorCount, Iterator batches) { + try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { + while (batches.hasNext()) { + TrainingBatch batch = batches.next(); + trainer.addTrainingVectors(batch.vectors, batch.count); + } + try (VectorIndexTraining coarse = trainer.finishTraining()) { + return wrap(CentroidRouter.fromTraining(coarse)); + } + } + } + + @Override + public VectorGlobalModelTrainer createGlobalModelTrainer() { + return new NativeVectorGlobalModelTrainer( + fieldType, options, trainSampleRatio, identifier, this); + } + + @Override + public VectorTrainingModel restoreTrainingModel(byte[] payload) { + return wrap(CentroidRouter.deserialize(payload)); + } + + NativeVectorTrainingModel wrap(CentroidRouter model) { + return new NativeVectorTrainingModel(model, this); } @Override diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java index 01b3a71a1cf8..82e4af55a420 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java @@ -182,6 +182,7 @@ private static String nativeOptionKey(String optionKey) { case "distance.metric": case "metric": return "metric"; + case "centroid.local-nlist": case "nlist": case "expected-vector-count": case "ivf.coarse-assignment": diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java new file mode 100644 index 000000000000..cbfd09b0efe9 --- /dev/null +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.globalindex.VectorGlobalModelTrainer; +import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.types.DataType; + +import java.io.IOException; +import java.util.Map; + +/** Reuses bounded heap ingestion and deterministic disk-backed sampling from the range writer. */ +final class NativeVectorGlobalModelTrainer implements VectorGlobalModelTrainer { + private final NativeVectorGlobalIndexWriter samples; + private final NativeVectorGlobalIndexer indexer; + private boolean finished; + + NativeVectorGlobalModelTrainer( + DataType fieldType, + Map options, + double ratio, + String identifier, + NativeVectorGlobalIndexer indexer) { + this.indexer = indexer; + this.samples = + new NativeVectorGlobalIndexWriter(null, fieldType, options, identifier, ratio); + } + + @Override + public void write(Object value, long absoluteRowId) { + if (finished) { + throw new IllegalStateException("Model trainer is finished."); + } + samples.write(value, absoluteRowId); + } + + @Override + public VectorTrainingModel finishTraining() throws IOException { + if (finished) { + throw new IllegalStateException("Model trainer is finished."); + } + finished = true; + try { + return indexer.wrap(samples.finishRoutingModel(indexer)); + } finally { + samples.close(); + } + } + + @Override + public void close() { + finished = true; + samples.close(); + } +} diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java new file mode 100644 index 000000000000..50d4b9126cc3 --- /dev/null +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; +import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.vector.CentroidRouter; + +/** JNI-backed portable partition model and independent centroid writers. */ +final class NativeVectorTrainingModel implements VectorTrainingModel { + private final CentroidRouter training; + + private final NativeVectorGlobalIndexer localIndexer; + + NativeVectorTrainingModel(CentroidRouter training, NativeVectorGlobalIndexer localIndexer) { + this.localIndexer = localIndexer; + this.training = training; + } + + @Override + public int nlist() { + return training.nlist(); + } + + @Override + public int assignCentroid(float[] vector) { + return training.assignCentroid(vector); + } + + @Override + public int[] nearestCentroids(float[] vector, int nprobe) { + return training.findNearestCentroids(vector, nprobe); + } + + @Override + public byte[] serialize() { + return training.serialize(); + } + + @Override + public void close() { + training.close(); + } + + @Override + public GlobalIndexSingleColumnWriter createCentroidWriter( + GlobalIndexFileWriter fileWriter, int centroid, String modelId) { + if (centroid < 0 || centroid >= nlist()) { + throw new IllegalArgumentException("Invalid routing centroid."); + } + return localIndexer.createLocalWriter(fileWriter, centroid, modelId); + } +} diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java new file mode 100644 index 000000000000..e4948476b1df --- /dev/null +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; +import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.globalindex.ScoredGlobalIndexResult; +import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; +import org.apache.paimon.globalindex.VectorGlobalModelTrainer; +import org.apache.paimon.globalindex.VectorIndexShardMeta; +import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.globalindex.VectorTrainingModelIO; +import org.apache.paimon.globalindex.io.GlobalIndexFileReader; +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.vector.CentroidRouter; +import org.apache.paimon.index.vector.VectorIndexReader; +import org.apache.paimon.index.vector.VectorIndexTrainer; +import org.apache.paimon.index.vector.VectorIndexTraining; +import org.apache.paimon.predicate.BatchVectorSearch; +import org.apache.paimon.predicate.VectorSearch; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RoaringNavigableMap64; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Real native routing, file pruning, absolute row-ID filters and batch dispatch. */ +class CentroidVectorGlobalIndexTest { + @TempDir java.nio.file.Path temp; + private static final long BASE = 1L << 35; + + @Test + void distributedSampleBudgetUsesCorpusSizeAndConfiguredRatio() { + Map options = new HashMap<>(); + options.put("index.type", "ivf_flat"); + options.put("metric", "l2"); + options.put("dimension", "2"); + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer( + new ArrayType(new FloatType()), options, "ivf-flat", 0.25); + assertThat(indexer.planGlobalTrainingSample(5)).containsExactly(0L, 2L); + assertThat(indexer.planGlobalTrainingSample(1000000)).hasSize(65536); + assertThatThrownBy(() -> indexer.planGlobalTrainingSample(0)) + .hasMessageContaining("without non-null vectors"); + // Explicit expected size still controls automatic nlist on a sampled build. + options.put("expected-vector-count", "16000000"); + assertThat(indexer.planGlobalTrainingSample(2000000)).hasSize(4096 * 64); + options.put("nlist", "2048"); + assertThat(indexer.planGlobalTrainingSample(1000000)).hasSize(2048 * 64); + } + + @Test + void diskAnnLocalSettingsDoNotChangeTheOuterSampleOrCenters() throws Exception { + Map coarse = new HashMap<>(); + coarse.put("index.type", "ivf_flat"); + coarse.put("dimension", "128"); + coarse.put("metric", "l2"); + coarse.put("nlist", "64"); + long[] expected; + try (VectorIndexTrainer trainer = VectorIndexTrainer.create(coarse)) { + expected = trainer.planTrainingSample(100000); + } + assertThat(expected).hasSize(65536); + float[] vectors = new float[256 * 128]; + for (int i = 0; i < vectors.length; i++) { + vectors[i] = (i * 19) % 401; + } + Map disk = new HashMap<>(coarse); + disk.put("index.type", "diskann"); + disk.put("nlist", "auto"); + try (VectorIndexTraining training = VectorIndexTrainer.train(coarse, vectors, 256); + CentroidRouter reference = CentroidRouter.fromTraining(training)) { + for (int seed : new int[] {42, 93}) { + disk.put("diskann.seed", Integer.toString(seed)); + disk.put("diskann.memory-budget-bytes", seed == 42 ? "8388608" : "4194304"); + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer( + new ArrayType(new FloatType()), disk, "diskann", 1.0); + assertThat(indexer.planGlobalTrainingSample(100000)).isEqualTo(expected); + try (VectorTrainingModel model = + indexer.trainGlobalModel( + 100000, + Collections.singletonList(new TrainingBatch(vectors, 256)) + .iterator())) { + assertThat(model.serialize()).isEqualTo(reference.serialize()); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) + void plannedRowsReproduceStreamingModel(String type) throws Exception { + int rows = 142003; + Map options = new HashMap<>(); + options.put("index.type", type.replace('-', '_')); + options.put("dimension", "2"); + options.put("nlist", "4"); + options.put("metric", "l2"); + if (type.equals("diskann") || type.equals("ivf-pq")) { + options.put("pq.m", "1"); + } + if (type.equals("diskann")) { + options.put("diskann.seed", "93"); + } + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer(new ArrayType(new FloatType()), options, type, 0.5); + long[] positions = indexer.planGlobalTrainingSample(rows); + assertThat(positions.length).isLessThan((rows + 1) / 2); + float[] selected = new float[2 * positions.length]; + for (int i = 0; i < positions.length; i++) { + System.arraycopy(testVector((int) positions[i]), 0, selected, 2 * i, 2); + } + try (VectorGlobalModelTrainer streaming = indexer.createGlobalModelTrainer()) { + for (int i = 0; i < rows; i++) { + streaming.write(testVector(i), BASE + i); + } + try (VectorTrainingModel expected = streaming.finishTraining(); + VectorTrainingModel actual = + indexer.trainGlobalModel( + rows, + Collections.singletonList( + new TrainingBatch(selected, positions.length)) + .iterator())) { + assertThat(actual.serialize()).isEqualTo(expected.serialize()); + try (CentroidRouter routing = CentroidRouter.deserialize(actual.serialize())) { + assertThat(routing.nlist()).isEqualTo(4); + } + } + } + } + + private static float[] testVector(int i) { + return new float[] {(i * 47) % 4001, (i * 31) % 1471}; + } + + @Test + void distributedBatchesTrainOnceUsingTheFullCorpusShape() throws Exception { + Map options = new HashMap<>(); + options.put("index.type", "ivf_flat"); + options.put("metric", "l2"); + options.put("dimension", "2"); + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer( + new ArrayType(new FloatType()), options, "ivf-flat", 0.0625); + float[] samples = new float[128]; + for (int i = 0; i < samples.length; i++) { + samples[i] = (i * 19) % 73; + } + Map reference = new HashMap<>(options); + reference.put("expected-vector-count", "1024"); + List batches = + Arrays.asList( + new TrainingBatch(Arrays.copyOfRange(samples, 0, 50), 25), + new TrainingBatch(Arrays.copyOfRange(samples, 50, samples.length), 39)); + try (VectorTrainingModel actual = indexer.trainGlobalModel(1024, batches.iterator()); + VectorIndexTraining coarse = VectorIndexTrainer.train(reference, samples, 64); + CentroidRouter expected = CentroidRouter.fromTraining(coarse)) { + assertThat(actual.nlist()).isEqualTo(16); + assertThat(actual.serialize()).isEqualTo(expected.serialize()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) + void testRoutingPrunesFilesAndPreservesAbsoluteIds(String type) throws Exception { + FileIO io = new LocalFileIO(); + Path directory = new Path(temp.toString()); + boolean diskann = "diskann".equals(type); + Map options = new HashMap<>(); + options.put("dimension", "2"); + options.put("index.type", type.replace('-', '_')); + options.put("metric", "l2"); + options.put("nlist", "4"); + if (!diskann) { + options.put("centroid.local-nlist", "8"); + } + if ("ivf-pq".equals(type) || diskann) { + options.put("pq.m", "1"); + } + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer(new ArrayType(new FloatType()), options, type); + GlobalIndexFileWriter files = + new GlobalIndexFileWriter() { + @Override + public String newFileName(String prefix) { + return prefix + "-" + UUID.randomUUID() + ".index"; + } + + @Override + public PositionOutputStream newOutputStream(String name) + throws java.io.IOException { + return io.newOutputStream(new Path(directory, name), false); + } + }; + float[][] vectors = new float[256][]; + for (int i = 0; i < vectors.length; i++) { + vectors[i] = new float[] {(i / 64) * 100f + (i % 64) * 0.1f, i % 11}; + } + List metas = new ArrayList<>(); + int[] centroidByRow = new int[256]; + GlobalIndexIOMeta emptyMeta; + try (VectorGlobalModelTrainer trainer = indexer.createGlobalModelTrainer()) { + for (int i = 0; i < vectors.length; i++) { + trainer.write(vectors[i], BASE + 101L * i); + } + try (VectorTrainingModel model = trainer.finishTraining()) { + assertThatThrownBy(() -> trainer.write(vectors[0], BASE)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("finished"); + assertThatThrownBy(trainer::finishTraining) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("finished"); + String id = "test-model"; + String name = files.newFileName("routing"); + try (PositionOutputStream out = files.newOutputStream(name)) { + VectorTrainingModelIO.write(out, model.serialize(), id); + } + Path modelPath = new Path(directory, name); + metas.add( + new GlobalIndexIOMeta( + modelPath, + io.getFileSize(modelPath), + 0, + VectorIndexShardMeta.routing( + id, + model.nlist(), + Collections.singletonList( + new Range(BASE, BASE + 101L * 255))) + .serialize())); + GlobalIndexSingleColumnWriter empty = model.createCentroidWriter(files, 0, id); + try { + ResultEntry result = empty.finish().get(0); + Path path = new Path(directory, result.fileName()); + emptyMeta = new GlobalIndexIOMeta(path, io.getFileSize(path), 0, result.meta()); + assertThat(io.getFileSize(path)).isEqualTo(8); + } finally { + ((Closeable) empty).close(); + } + for (int c = 0; c < model.nlist(); c++) { + GlobalIndexSingleColumnWriter writer = model.createCentroidWriter(files, c, id); + try { + assertThatThrownBy(() -> writer.write(vectors[0], -1L)) + .isInstanceOf(IllegalArgumentException.class); + for (int i = 0; i < vectors.length; i++) { + centroidByRow[i] = model.assignCentroid(vectors[i]); + if (centroidByRow[i] == c) { + writer.write(vectors[i], BASE + 101L * i); + } + } + ResultEntry result = writer.finish().get(0); + Path path = new Path(directory, result.fileName()); + if (result.rowCount() > 0) { + assertThat(VectorIndexShardMeta.deserialize(result.meta()).centroid()) + .isEqualTo(c); + try (SeekableInputStream in = io.newInputStream(path); + VectorIndexReader local = + new VectorIndexReader( + new NativeVectorGlobalIndexReader + .SeekableStreamVectorIndexInput(in))) { + assertThat(local.metadata().indexType()) + .isEqualTo(type.replace('-', '_')); + if (!diskann) { + assertThat(local.metadata().nlist()).isEqualTo(8); + } + } + } + metas.add( + new GlobalIndexIOMeta( + path, + io.getFileSize(path), + result.rowCount(), + result.meta())); + } finally { + ((Closeable) writer).close(); + } + } + } + } + // A single worker also guards against waiting for child futures on the same executor. + ExecutorService executor = Executors.newSingleThreadExecutor(); + Map opened = new ConcurrentHashMap<>(); + GlobalIndexFileReader fileReader = + meta -> { + opened.computeIfAbsent(meta.filePath(), p -> new AtomicInteger()) + .incrementAndGet(); + return io.newInputStream(meta.filePath()); + }; + try (GlobalIndexReader reader = + indexer.createReader(fileReader, metas, 256, null, executor)) { + try (GlobalIndexReader empty = + new NativeVectorGlobalIndexReader( + fileReader, + Collections.singletonList(emptyMeta), + new ArrayType(new FloatType()), + executor, + type)) { + assertThat(empty.visitVectorSearch(new VectorSearch(vectors[0], 1, "v")).get()) + .isEmpty(); + assertThat( + empty.visitBatchVectorSearch( + new BatchVectorSearch( + new float[][] {vectors[0], vectors[1]}, + 1, + "v")) + .get()) + .allSatisfy(result -> assertThat(result).isEmpty()); + assertThat(opened).isEmpty(); + } + Map search = new HashMap<>(); + search.put("centroid.nprobe", "1"); + if (diskann) { + search.put("diskann.l_search", "256"); + } else { + search.put("ivf.nprobe", "2"); + } + List> empty = + reader.visitBatchVectorSearch( + new BatchVectorSearch( + new float[][] {vectors[0], vectors[200]}, + 1, + "v", + search) + .withIncludeRowIds(new RoaringNavigableMap64())) + .get(30, TimeUnit.SECONDS); + assertThat(empty).allSatisfy(result -> assertThat(result).isEmpty()); + assertThat(opened).isEmpty(); + ScoredGlobalIndexResult one = + reader.visitVectorSearch(new VectorSearch(vectors[0], 1, "v", search)) + .get(30, TimeUnit.SECONDS) + .get(); + assertThat(one.results().contains(BASE)).isTrue(); + assertThat(opened).hasSize(2); // routing model and the selected centroid only + for (GlobalIndexIOMeta file : metas.subList(1, metas.size())) { + boolean selected = + VectorIndexShardMeta.deserialize(file.metadata()).centroid() + == centroidByRow[0]; + assertThat(opened.containsKey(file.filePath())).isEqualTo(selected); + } + float[][] queries = {vectors[0], vectors[200], vectors[0]}; + List> batch = + reader.visitBatchVectorSearch(new BatchVectorSearch(queries, 1, "v", search)) + .get(30, TimeUnit.SECONDS); + assertThat(batch).hasSize(3); + assertThat(batch.get(0).get().results().contains(BASE)).isTrue(); + assertThat(batch.get(1).get().results().contains(BASE + 101L * 200)).isTrue(); + assertThat(batch.get(2).get().results().contains(BASE)).isTrue(); + assertThat(opened.values()).allSatisfy(count -> assertThat(count.get()).isEqualTo(1)); + + RoaringNavigableMap64 include = new RoaringNavigableMap64(); + include.add(BASE + 101L * 200); + ScoredGlobalIndexResult filtered = + reader.visitVectorSearch( + new VectorSearch(vectors[0], 1, "v").withIncludeRowIds(include)) + .get(30, TimeUnit.SECONDS) + .get(); + assertThat(filtered.results().getLongCardinality()).isEqualTo(1); + assertThat(filtered.results().contains(BASE + 101L * 200)).isTrue(); + + assertThat( + reader.visitVectorSearch( + new VectorSearch(vectors[0], 1, "v", search) + .withIncludeRowIds(new RoaringNavigableMap64())) + .get(30, TimeUnit.SECONDS)) + .isEmpty(); + } finally { + executor.shutdownNow(); + } + + List incomplete = new ArrayList<>(metas); + incomplete.remove(incomplete.size() - 1); + assertThatThrownBy(() -> indexer.createReader(fileReader, incomplete, 256, null, executor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Incomplete"); + } +} diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java index 6d0180b1611f..c876e3167919 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java @@ -183,6 +183,12 @@ public void testAllNullReturnsEmpty() { @Test public void testTrainingVectorCountUsesConfiguredSampleRatio() { + assertThat(NativeVectorGlobalIndexWriter.trainingCandidateCount(10000000000L, 0.6)) + .isEqualTo(6000000000L); + assertThat( + NativeVectorGlobalIndexWriter.sampleIndex( + 5999999999L, 10000000000L, 6000000000L)) + .isEqualTo(9999999998L); assertThat(NativeVectorGlobalIndexWriter.trainingVectorCount(10_000L, 1.0)) .isEqualTo(10_000); assertThat(NativeVectorGlobalIndexWriter.trainingVectorCount(10_000L, 0.25)) From c6963d53ded3348d77741d56fd7659d1d665e260 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 18 Nov 2026 19:15:49 +0800 Subject: [PATCH 2/7] [test] Benchmark range and centroid vector sharding in Spark --- .../benchmark/VectorShardingBenchmark.java | 868 ++++++++++++++++++ 1 file changed, 868 insertions(+) create mode 100644 paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java new file mode 100644 index 000000000000..9f3d288ceff0 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java @@ -0,0 +1,868 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.benchmark; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.OffsetGlobalIndexReader; +import org.apache.paimon.globalindex.ScoredGlobalIndexResult; +import org.apache.paimon.globalindex.VectorIndexUtils; +import org.apache.paimon.globalindex.io.GlobalIndexFileReader; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.predicate.VectorSearch; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.spark.SparkCatalog; +import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.vector.index.NativeVectorGlobalIndexer; + +import org.apache.spark.sql.SparkSession; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; + +/** + * Explicit opt-in benchmark of the actual Spark build and native Paimon index readers. + * + *

Arguments: data-directory, NEW output-directory, row-count, measured-query-count, repetitions, + * nlist, rows-per-range-shard, threads, comma-separated nprobes. The input directory contains + * base.fvecs, query.fvecs and groundtruth.ivecs. Ground truth must describe exactly the requested + * base rows; no prefix ground truth is inferred. + * + *

Import and row-ID mapping are excluded from build time. Query time includes fresh reader + * creation, routing/model loads, all shard searches, top-K merge and close. It excludes manifest + * planning, source-row fetch and SQL scheduling. Every configuration uses the same held-out query + * IDs, top-10, warm OS page cache and no refinement. I/O counters measure logical bytes actually + * returned by Paimon streams, including metadata and routing, and preserve vectored reads. + * + *

Run serially, after compilation, with RAYON_NUM_THREADS set to the thread argument. Output + * contains every query and every build; there are no timing assertions or automatic test runs. + * + *

System properties: {@code benchmark.range-based.options} and {@code + * benchmark.centroid-based.options} add comma-separated procedure options for each mode. {@code + * benchmark.range-based.nprobes} and {@code benchmark.centroid-based.nprobes} override the probe + * sweep separately, allowing comparisons at a common recall target. {@code benchmark.query-offset} + * skips that many entries in the fixed shuffled query order before warmup, allowing disjoint + * calibration and validation sets. {@code benchmark.source-warehouse} reuses this benchmark's + * existing source table; index generations are replaced but source rows are unchanged. {@code + * benchmark.modes} selects one or both modes. With one mode, {@code benchmark.skip-build=true} + * reads the current generation and records build time as -1 (unmeasured). {@code + * benchmark.index-type} selects any native vector index family. {@code benchmark.read-snapshot} + * selects a retained generation for query-only sweeps. DiskANN independently sweeps {@code + * benchmark..l-searches} and outer centroid probes; range mode has no outer probe. {@code + * benchmark.reuse-readers=true} is exclusively for recall calibration: it records I/O as -1 and its + * query timing must not be compared with fresh-reader measurements. + */ +public final class VectorShardingBenchmark { + private static final int TOP_K = 10; + private static final int WARMUP = 16; + private static final long SEED = 20261117L; + + private final java.nio.file.Path data; + private final java.nio.file.Path output; + private final int rows; + private final int queryCount; + private final int repetitions; + private final int nlist; + private final int rangeRows; + private final int threads; + private final List probes; + private final List configuredModes; + private final int queryOffset = + Integer.parseInt(System.getProperty("benchmark.query-offset", "0")); + private final String sourceWarehouse = System.getProperty("benchmark.source-warehouse"); + private final boolean skipBuild = + Boolean.parseBoolean(System.getProperty("benchmark.skip-build", "false")); + private final String indexType = System.getProperty("benchmark.index-type", "ivf-pq"); + private final boolean reuseReaders = + Boolean.parseBoolean(System.getProperty("benchmark.reuse-readers", "false")); + private final String readSnapshot = System.getProperty("benchmark.read-snapshot"); + private final LocalFileIO io = new LocalFileIO(); + private SparkSession spark; + private Path tablePath; + private int dimension; + private long[] logicalIds; + private float[][] queries; + private int[][] truth; + private List queryIds; + private ExecutorService executor; + + private VectorShardingBenchmark(String[] args) { + if (args.length != 9) { + throw new IllegalArgumentException("Expected 9 arguments; see class documentation."); + } + data = java.nio.file.Paths.get(args[0]); + output = java.nio.file.Paths.get(args[1]); + rows = Integer.parseInt(args[2]); + queryCount = Integer.parseInt(args[3]); + repetitions = Integer.parseInt(args[4]); + nlist = Integer.parseInt(args[5]); + rangeRows = Integer.parseInt(args[6]); + threads = Integer.parseInt(args[7]); + configuredModes = + Arrays.asList( + System.getProperty("benchmark.modes", "range-based,centroid-based") + .split(",")); + if (!VectorIndexUtils.isVectorIndexType(indexType) + || (readSnapshot != null && !skipBuild) + || configuredModes.isEmpty() + || new HashSet<>(configuredModes).size() != configuredModes.size() + || !Arrays.asList("range-based", "centroid-based").containsAll(configuredModes) + || queryOffset < 0 + || (skipBuild && (sourceWarehouse == null || configuredModes.size() != 1))) { + throw new IllegalArgumentException( + "Invalid modes, query offset or existing-index configuration"); + } + probes = new ArrayList<>(); + for (String value : args[8].split(",")) { + int probe = Integer.parseInt(value); + if (probe < 1 || probe > nlist) { + throw new IllegalArgumentException("nprobe must be in [1, nlist]"); + } + probes.add(probe); + } + if (rows < TOP_K || queryCount < 1 || repetitions < 1 || threads < 1 || rangeRows < 1) { + throw new IllegalArgumentException("Invalid workload size"); + } + if (!String.valueOf(threads).equals(System.getenv("RAYON_NUM_THREADS"))) { + throw new IllegalArgumentException("Set RAYON_NUM_THREADS=" + threads); + } + } + + public static void main(String[] args) throws Exception { + Locale.setDefault(Locale.ROOT); + new VectorShardingBenchmark(args).run(); + } + + private void run() throws Exception { + Files.createDirectory(output); + try (DataInputStream in = input("base.fvecs")) { + dimension = littleInt(in); + } + if (dimension < 4 + || dimension % 4 != 0 + || Files.size(data.resolve("base.fvecs")) != (long) rows * (dimension + 1) * 4) { + throw new IllegalArgumentException("Base shape differs from requested rows/dimension"); + } + queries = loadQueries(); + truth = loadTruth(); + if (truth.length != queries.length + || (long) queries.length < (long) queryOffset + queryCount + WARMUP) { + throw new IllegalArgumentException( + "Need aligned ground truth and enough held-out queries"); + } + queryIds = new ArrayList<>(); + for (int i = 0; i < queries.length; i++) { + queryIds.add(i); + } + Collections.shuffle(queryIds, new Random(SEED)); + String excludedIds = System.getProperty("benchmark.exclude-query-ids"); + if (excludedIds != null) { + Set excluded = new HashSet<>(); + for (String id : Files.readAllLines(java.nio.file.Paths.get(excludedIds))) { + excluded.add(Integer.parseInt(id)); + } + queryIds.removeIf(excluded::contains); + if (queryIds.size() < queryOffset + queryCount + WARMUP) { + throw new IllegalArgumentException("Too few queries after exclusions"); + } + } + queryIds = new ArrayList<>(queryIds.subList(queryOffset, queryIds.size())); + Files.write( + output.resolve("workload.txt"), + Arrays.asList( + "indexType=" + indexType, + "rows=" + rows, + "dimension=" + dimension, + "topK=" + TOP_K, + "queries=" + queryCount, + "warmup=" + WARMUP, + "repetitions=" + repetitions, + "nlist=" + nlist, + "defaultPQm=" + (dimension / 4), + "metric=l2", + "train.sample-ratio=1.0", + "rangeRows=" + rangeRows, + "threads=" + threads, + "nprobes=" + probes, + "rangeNprobes=" + probesForMode("range-based"), + "centroidNprobes=" + probesForMode("centroid-based"), + "seed=" + SEED, + "queryOffset=" + queryOffset, + "buildSkipped=" + skipBuild, + "sourceWarehouse=" + sourceWarehouse, + "rangeOptions=" + buildOptions("range-based"), + "centroidOptions=" + buildOptions("centroid-based"), + "readSnapshot=" + readSnapshot, + "rangeLSearches=" + lSearchesForMode("range-based"), + "centroidLSearches=" + lSearchesForMode("centroid-based"), + "readerLifetime=" + + (reuseReaders ? "quality-only-reused" : "fresh-per-query"), + "pageCache=warm", + "refinement=false", + "java=" + System.getProperty("java.version"), + "os=" + System.getProperty("os.name") + " " + System.getProperty("os.arch"), + "queryIds=" + queryIds.subList(WARMUP, WARMUP + queryCount))); + executor = Executors.newFixedThreadPool(threads); + spark = + SparkSession.builder() + .master("local[" + threads + "]") + .appName("VectorShardingBenchmark") + .config("spark.ui.enabled", false) + .config("spark.sql.catalog.paimon", SparkCatalog.class.getName()) + .config( + "spark.sql.catalog.paimon.warehouse", + sourceWarehouse == null + ? output.resolve("warehouse").toString() + : sourceWarehouse) + .config("spark.sql.defaultCatalog", "paimon") + .config( + "spark.sql.extensions", + PaimonSparkSessionExtensions.class.getName()) + .config("spark.sql.shuffle.partitions", threads) + .config("spark.default.parallelism", threads) + .getOrCreate(); + spark.sparkContext().setLogLevel("WARN"); + try (PrintWriter builds = + csv( + "builds.csv", + "repetition,mode,rows,build_ns,index_files,index_bytes,snapshot_id,index_type"); + PrintWriter measurements = + csv( + "queries.csv", + "repetition,mode,nprobe,query_id,latency_ns,hits_at_10,result_count,files,opens,read_calls,read_bytes,routing_bytes,l_search,index_type")) { + spark.sql("CREATE DATABASE IF NOT EXISTS bench").collect(); + spark.sql("USE bench").collect(); + tablePath = + new Path( + (sourceWarehouse == null + ? output.resolve("warehouse") + : java.nio.file.Paths.get(sourceWarehouse)) + .resolve("bench.db/vectors") + .toString()); + if (sourceWarehouse == null) { + spark.sql( + "CREATE TABLE vectors (id INT, v ARRAY) TBLPROPERTIES (" + + "'bucket'='-1','row-tracking.enabled'='true','data-evolution.enabled'='true'," + + "'" + + indexType + + ".dimension'='" + + dimension + + "','" + + indexType + + ".metric'='l2'," + + "'global-index.row-count-per-shard'='" + + rangeRows + + "'," + + "'global-index.build.max-parallelism'='" + + threads + + "')") + .collect(); + importSource(); + } + mapRowIds(); + for (int repeat = 0; repeat < repetitions; repeat++) { + List modes = new ArrayList<>(configuredModes); + if (repeat % 2 != 0) { + Collections.reverse(modes); + } + for (String mode : modes) { + // Every family can reuse the immutable source table. + spark.sql( + "ALTER TABLE vectors SET TBLPROPERTIES ('" + + indexType + + ".dimension'='" + + dimension + + "','" + + indexType + + ".metric'='l2')") + .collect(); + long elapsed = -1; + if (!skipBuild) { + Map options = buildOptions(mode); + List pairs = new ArrayList<>(); + options.forEach((key, value) -> pairs.add(key + "=" + value)); + for (String type : + Arrays.asList( + "ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann")) { + spark.sql( + "CALL sys.drop_global_index(table=>'bench.vectors'," + + "index_column=>'v',index_type=>'" + + type + + "')") + .collect(); + } + long started = System.nanoTime(); + spark.sql( + "CALL sys.create_global_index(table=>'bench.vectors'," + + "index_column=>'v',index_type=>'" + + indexType + + "',options=>'" + + String.join(",", pairs) + + "')") + .collect(); + elapsed = System.nanoTime() - started; + } + FileStoreTable table = table(); + Snapshot snapshot = + readSnapshot == null + ? table.snapshotManager().latestSnapshot() + : table.snapshotManager() + .snapshot(Long.parseLong(readSnapshot)); + List files = new ArrayList<>(); + for (IndexManifestEntry entry : + table.store().newIndexFileHandler().scan(snapshot, indexType)) { + files.add(entry.indexFile()); + } + if (files.isEmpty()) { + throw new IllegalStateException("Build produced no index"); + } + boolean centroid = files.stream().anyMatch(IndexFileMeta::isRoutingModel); + if (centroid != "centroid-based".equals(mode)) { + throw new IllegalStateException( + "Existing index does not match the requested mode"); + } + long bytes = files.stream().mapToLong(IndexFileMeta::fileSize).sum(); + builds.printf( + "%d,%s,%d,%d,%d,%d,%d,%s%n", + repeat, + mode, + rows, + elapsed, + files.size(), + bytes, + snapshot.id(), + indexType); + builds.flush(); + System.out.printf( + "BUILD repeat=%d mode=%s seconds=%.3f files=%d MiB=%.2f%n", + repeat, mode, elapsed / 1e9, files.size(), bytes / 1048576.0); + // Retain metadata for reproducibility even after the next index generation + // replaces it. + List fileNames = new ArrayList<>(); + List metas = new ArrayList<>(); + byte[] warm = new byte[1024 * 1024]; + for (IndexFileMeta file : files) { + Path path = + table.store().pathFactory().globalIndexFileFactory().toPath(file); + fileNames.add(path + "," + file.fileSize() + "," + file.rowCount()); + metas.add( + new GlobalIndexIOMeta( + path, + file.fileSize(), + file.rowCount(), + file.globalIndexMeta().indexMeta())); + try (SeekableInputStream in = io.newInputStream(path)) { + while (in.read(warm) >= 0) { + // Warm every index file before measuring either mode. + } + } + } + Files.write(output.resolve("files-" + repeat + "-" + mode + ".csv"), fileNames); + List widths = probesForMode(mode); + Collections.shuffle(widths, new Random(SEED + repeat)); + List cached = + reuseReaders + ? openReaders( + mode, + files, + metas, + meta -> io.newInputStream(meta.filePath())) + : null; + try { + for (int probe : widths) { + for (int lSearch : lSearchesForMode(mode)) { + for (int i = 0; i < WARMUP + queryCount; i++) { + int id = queryIds.get(i); + Observation observation = + query( + mode, + probe, + lSearch, + queries[id], + files, + metas, + cached); + int hits = 0; + for (long rowId : observation.result.results()) { + if (rowId < 0 + || rowId >= logicalIds.length + || logicalIds[(int) rowId] < 0) { + throw new IllegalStateException( + "Unknown absolute row ID " + rowId); + } + long logical = logicalIds[(int) rowId]; + for (int k = 0; k < TOP_K; k++) { + if (logical == truth[id][k]) { + hits++; + break; + } + } + } + if (observation.result.results().getLongCardinality() + != TOP_K) { + throw new IllegalStateException("Incomplete top-K"); + } + if (i >= WARMUP) { + Stats stats = observation.stats; + measurements.printf( + "%d,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%s%n", + repeat, + mode, + probe, + id, + observation.nanos, + hits, + TOP_K, + reuseReaders ? -1 : stats.files.size(), + reuseReaders ? -1 : stats.opens.sum(), + reuseReaders ? -1 : stats.calls.sum(), + reuseReaders ? -1 : stats.bytes.sum(), + reuseReaders ? -1 : stats.routingBytes.sum(), + lSearch, + indexType); + } + } + measurements.flush(); + System.out.printf( + "QUERIES repeat=%d mode=%s nprobe=%d l_search=%d count=%d%n", + repeat, mode, probe, lSearch, queryCount); + } + } + } finally { + if (cached != null) { + for (GlobalIndexReader reader : cached) { + reader.close(); + } + } + } + } + } + } finally { + spark.stop(); + executor.shutdownNow(); + } + } + + private FileStoreTable table() { + return FileStoreTableFactory.create(io, tablePath); + } + + private Map buildOptions(String mode) { + Map options = new LinkedHashMap<>(); + options.put(indexType + ".shard", mode); + if (!"diskann".equals(indexType) || "centroid-based".equals(mode)) { + options.put(indexType + ".nlist", String.valueOf(nlist)); + } + if ("ivf-pq".equals(indexType) || "diskann".equals(indexType)) { + options.put(indexType + ".pq.m", String.valueOf(dimension / 4)); + } + options.put(indexType + ".train.sample-ratio", "1.0"); + String extra = System.getProperty("benchmark." + mode + ".options", ""); + if (!extra.isEmpty()) { + for (String pair : extra.split(",")) { + String[] entry = pair.split("=", 2); + if (entry.length != 2 + || !entry[0].startsWith(indexType + ".") + || entry[1].isEmpty() + || (indexType + ".shard").equals(entry[0])) { + throw new IllegalArgumentException("Invalid benchmark option: " + pair); + } + options.put(entry[0], entry[1]); + } + } + return options; + } + + private List probesForMode(String mode) { + if ("diskann".equals(indexType) && "range-based".equals(mode)) { + return new ArrayList<>(Collections.singletonList(0)); + } + String override = System.getProperty("benchmark." + mode + ".nprobes"); + if (override == null) { + return new ArrayList<>(probes); + } + List widths = new ArrayList<>(); + int modeNlist = Integer.parseInt(buildOptions(mode).get(indexType + ".nlist")); + for (String value : override.split(",")) { + int width = Integer.parseInt(value); + if (width < 1 || width > modeNlist || widths.contains(width)) { + throw new IllegalArgumentException("Invalid nprobe override: " + override); + } + widths.add(width); + } + return widths; + } + + private void importSource() throws Exception { + BatchWriteBuilder builder = table().newBatchWriteBuilder(); + try (BatchTableWrite writer = builder.newWrite(); + BatchTableCommit commit = builder.newCommit(); + DataInputStream in = input("base.fvecs")) { + for (int id = 0; id < rows; id++) { + writer.write(GenericRow.of(id, new GenericArray(vector(in)))); + } + commit.commit(writer.prepareCommit()); + } + System.out.println("Imported source rows=" + rows); + } + + private void mapRowIds() throws Exception { + FileStoreTable table = table(); + ReadBuilder builder = + table.newReadBuilder() + .withReadType( + new RowType( + true, + Arrays.asList( + table.rowType().getField("id"), + SpecialFields.ROW_ID))); + logicalIds = new long[rows]; + Arrays.fill(logicalIds, -1); + Set ids = new HashSet<>(); + try (RecordReader reader = + builder.newRead().createReader(builder.newScan().plan())) { + reader.forEachRemaining( + row -> { + int id = row.getInt(0); + long rowId = row.getLong(1); + if (rowId < 0 + || rowId >= rows + || logicalIds[(int) rowId] != -1 + || !ids.add(id)) { + throw new IllegalStateException("Non-bijective source ID mapping"); + } + logicalIds[(int) rowId] = id; + }); + } + if (ids.size() != rows) { + throw new IllegalStateException("Incomplete source mapping"); + } + System.out.println("Validated absolute row-ID mapping rows=" + ids.size()); + } + + private List lSearchesForMode(String mode) { + if (!"diskann".equals(indexType)) { + return Collections.singletonList(0); + } + List result = new ArrayList<>(); + for (String value : + System.getProperty("benchmark." + mode + ".l-searches", "64").split(",")) { + int width = Integer.parseInt(value); + if (width < TOP_K || result.contains(width)) { + throw new IllegalArgumentException("Invalid DiskANN search width: " + value); + } + result.add(width); + } + return result; + } + + private Observation query( + String mode, + int probe, + int lSearch, + float[] vector, + List files, + List metas, + List cached) + throws Exception { + Stats stats = new Stats(); + GlobalIndexFileReader input = + meta -> { + stats.opens.increment(); + stats.files.add(meta.filePath().toString()); + SeekableInputStream delegate = io.newInputStream(meta.filePath()); + if (!(delegate instanceof VectoredReadable)) { + delegate.close(); + throw new IOException("Local benchmark requires vectored reads"); + } + return new CountingInput( + delegate, stats, meta.filePath().getName().contains("training-model")); + }; + Map options = new LinkedHashMap<>(); + if (probe > 0) { + options.put( + "centroid-based".equals(mode) ? "centroid.nprobe" : "ivf.nprobe", + String.valueOf(probe)); + } + String inner = System.getProperty("benchmark.local-nprobe"); + if (inner != null && "centroid-based".equals(mode)) { + options.put("ivf.nprobe", inner); + } + if (lSearch > 0) { + options.put("diskann.l_search", String.valueOf(lSearch)); + } + VectorSearch search = new VectorSearch(vector, TOP_K, "v", options); + long started = System.nanoTime(); + List readers = + cached == null ? openReaders(mode, files, metas, input) : cached; + ScoredGlobalIndexResult result; + try { + List>> futures = new ArrayList<>(); + for (GlobalIndexReader reader : readers) { + futures.add(reader.visitVectorSearch(search)); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .get(120, TimeUnit.SECONDS); + List results = new ArrayList<>(); + for (CompletableFuture> future : futures) { + future.get().ifPresent(results::add); + } + result = ScoredGlobalIndexResult.merge(results).topK(TOP_K); + } finally { + if (cached == null) { + for (GlobalIndexReader reader : readers) { + reader.close(); + } + } + } + return new Observation(result, System.nanoTime() - started, stats); + } + + private List openReaders( + String mode, + List files, + List metas, + GlobalIndexFileReader input) + throws IOException { + NativeVectorGlobalIndexer indexer = + new NativeVectorGlobalIndexer( + new ArrayType(new FloatType()), Collections.emptyMap(), indexType); + List readers = new ArrayList<>(); + try { + if ("centroid-based".equals(mode)) { + readers.add(indexer.createReader(input, metas, rows, null, executor)); + } else { + for (int i = 0; i < metas.size(); i++) { + IndexFileMeta file = files.get(i); + readers.add( + new OffsetGlobalIndexReader( + indexer.createReader( + input, + Collections.singletonList(metas.get(i)), + file.rowCount(), + null, + executor), + file.globalIndexMeta().rowRangeStart(), + file.globalIndexMeta().rowRangeEnd())); + } + } + return readers; + } catch (RuntimeException e) { + for (GlobalIndexReader reader : readers) { + try { + reader.close(); + } catch (IOException close) { + e.addSuppressed(close); + } + } + throw e; + } + } + + private DataInputStream input(String name) throws IOException { + return new DataInputStream( + new BufferedInputStream(Files.newInputStream(data.resolve(name)), 1 << 20)); + } + + private static int littleInt(DataInputStream in) throws IOException { + return Integer.reverseBytes(in.readInt()); + } + + private float[] vector(DataInputStream in) throws IOException { + if (littleInt(in) != dimension) { + throw new IOException("Inconsistent vector dimension"); + } + float[] vector = new float[dimension]; + for (int i = 0; i < dimension; i++) { + vector[i] = Float.intBitsToFloat(littleInt(in)); + if (!Float.isFinite(vector[i])) { + throw new IOException("Non-finite vector"); + } + } + return vector; + } + + private float[][] loadQueries() throws IOException { + int count = + Math.toIntExact(Files.size(data.resolve("query.fvecs")) / ((dimension + 1L) * 4)); + float[][] vectors = new float[count][]; + try (DataInputStream in = input("query.fvecs")) { + for (int i = 0; i < count; i++) { + vectors[i] = vector(in); + } + } + return vectors; + } + + private int[][] loadTruth() throws IOException { + int width; + try (DataInputStream in = input("groundtruth.ivecs")) { + width = littleInt(in); + } + if (width < TOP_K) { + throw new IOException("Ground truth is shorter than top-K"); + } + int count = + Math.toIntExact(Files.size(data.resolve("groundtruth.ivecs")) / ((width + 1L) * 4)); + int[][] neighbors = new int[count][TOP_K]; + try (DataInputStream in = input("groundtruth.ivecs")) { + for (int i = 0; i < count; i++) { + if (littleInt(in) != width) { + throw new IOException("Inconsistent ground truth width"); + } + for (int j = 0; j < width; j++) { + int id = littleInt(in); + if (id < 0 || id >= rows) { + throw new IOException("Ground truth must refer to the exact source corpus"); + } + if (j < TOP_K) { + neighbors[i][j] = id; + } + } + } + } + return neighbors; + } + + private PrintWriter csv(String name, String header) throws IOException { + PrintWriter writer = + new PrintWriter( + Files.newBufferedWriter( + output.resolve(name), StandardOpenOption.CREATE_NEW)); + writer.println(header); + writer.flush(); + return writer; + } + + private static class Observation { + final ScoredGlobalIndexResult result; + final long nanos; + final Stats stats; + + Observation(ScoredGlobalIndexResult result, long nanos, Stats stats) { + this.result = result; + this.nanos = nanos; + this.stats = stats; + } + } + + private static class Stats { + final Set files = ConcurrentHashMap.newKeySet(); + final LongAdder opens = new LongAdder(); + final LongAdder calls = new LongAdder(); + final LongAdder bytes = new LongAdder(); + final LongAdder routingBytes = new LongAdder(); + + int record(int count, boolean routing) { + calls.increment(); + if (count > 0) { + bytes.add(count); + if (routing) { + routingBytes.add(count); + } + } + return count; + } + } + + private static class CountingInput extends SeekableInputStream implements VectoredReadable { + private final SeekableInputStream delegate; + private final Stats stats; + private final boolean routing; + + CountingInput(SeekableInputStream delegate, Stats stats, boolean routing) { + this.delegate = delegate; + this.stats = stats; + this.routing = routing; + } + + @Override + public int read() throws IOException { + int value = delegate.read(); + stats.record(value < 0 ? 0 : 1, routing); + return value; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + return stats.record(delegate.read(buffer, offset, length), routing); + } + + @Override + public int pread(long position, byte[] buffer, int offset, int length) throws IOException { + return stats.record( + ((VectoredReadable) delegate).pread(position, buffer, offset, length), routing); + } + + @Override + public void seek(long desired) throws IOException { + delegate.seek(desired); + } + + @Override + public long getPos() throws IOException { + return delegate.getPos(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } +} From 77dc702e85177810eb9aa0b4114857b3673b29bd Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 18 Nov 2026 20:03:53 +0800 Subject: [PATCH 3/7] [core] Wrap centroid routing in global index metadata --- .../multimodal-table/global-index/vector.mdx | 7 +- .../CentroidVectorGlobalIndexer.java | 48 ++++++++ .../globalindex/VectorGlobalIndexer.java | 36 ------ .../globalindex/VectorIndexShardMeta.java | 106 +++++++++++++++--- .../paimon/globalindex/VectorIndexUtils.java | 8 -- .../globalindex/CentroidVectorIndexUtils.java | 52 +++++---- .../apache/paimon/index/IndexFileMeta.java | 49 +------- .../paimon/index/IndexFileMetaSerializer.java | 6 +- .../index/IndexFileMetaV5Deserializer.java | 40 ------- .../paimon/manifest/IndexManifestEntry.java | 4 +- .../IndexManifestEntrySerializer.java | 8 +- .../manifest/IndexManifestFileHandler.java | 17 +-- .../commit/DataEvolutionChangesProvider.java | 6 +- .../DataEvolutionConflictDetection.java | 3 +- .../table/sink/CommitMessageSerializer.java | 11 +- .../table/source/IndexVectorSearchSplit.java | 15 +-- .../CentroidVectorIndexUtilsTest.java | 26 ++++- .../index/IndexFileMetaSerializerTest.java | 43 +++---- .../IndexManifestFileHandlerTest.java | 3 +- .../commit/ConflictDetectionTest.java | 3 +- .../sink/CommitMessageSerializerTest.java | 25 +++++ .../CentroidGlobalIndexTopoBuilder.java | 36 +++--- .../globalindex/CentroidTrainingSample.java | 2 +- .../GlobalIndexTopologyBuilderUtils.java | 9 +- .../CentroidTrainingSampleTest.java | 2 +- .../DefaultGlobalIndexTopoBuilderTest.java | 13 +++ .../benchmark/VectorShardingBenchmark.java | 7 +- .../spark/sql/CentroidVectorIndexTest.scala | 10 +- .../NativeCentroidRoutedIndexReader.java | 31 +++-- .../index/NativeVectorGlobalIndexWriter.java | 3 +- .../index/NativeVectorGlobalIndexer.java | 10 +- .../index/CentroidVectorGlobalIndexTest.java | 9 +- 32 files changed, 337 insertions(+), 311 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java delete mode 100644 paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java diff --git a/docs/docs/multimodal-table/global-index/vector.mdx b/docs/docs/multimodal-table/global-index/vector.mdx index d924cf63b114..c7c085e8e014 100644 --- a/docs/docs/multimodal-table/global-index/vector.mdx +++ b/docs/docs/multimodal-table/global-index/vector.mdx @@ -489,7 +489,12 @@ Queries load the routing model, select centroids with `centroid.nprobe`, and ope only their shard files. Inside each selected shard, `ivf.nprobe` controls how many local IVF lists to search. For DiskANN, `diskann.l_search` controls the graph search. Automatic probe sizing and filtered search remain available. -Centroid shards store absolute row IDs. The model records the exact source row +The trained outer centers live in the routing model file. In the manifest, each +centroid index entry stores its routing descriptor in `GlobalIndexMeta.indexMeta`. +For a shard, this descriptor identifies the model and centroid and wraps the local +index writer’s metadata. The local reader receives those original bytes. + +Centroid shards store absolute row IDs. The model entry records the exact source row ranges so gaps belonging to other partitions remain eligible for raw search in `vector-index.search-mode=detail` or `chunk`. The default `fast` mode searches only indexed rows. diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java new file mode 100644 index 000000000000..33b7dadc045f --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import java.io.Serializable; +import java.util.Iterator; + +/** A vector indexer that can train and restore a router for independently built shards. */ +public interface CentroidVectorGlobalIndexer extends VectorGlobalIndexer { + + /** Return non-null input ordinals in the order required for training. */ + long[] planGlobalTrainingSample(long vectorCount); + + /** Train on ordered, preselected batches using the full non-null corpus size. */ + VectorTrainingModel trainGlobalModel(long vectorCount, Iterator batches); + + VectorGlobalModelTrainer createGlobalModelTrainer(); + + VectorTrainingModel restoreTrainingModel(byte[] payload); + + /** A row-major batch selected by the distributed sampler. */ + final class TrainingBatch implements Serializable { + private static final long serialVersionUID = 1L; + public final float[] vectors; + public final int count; + + public TrainingBatch(float[] vectors, int count) { + this.vectors = vectors; + this.count = count; + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java index ccba5595f268..63166ef7c1f1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorGlobalIndexer.java @@ -18,45 +18,9 @@ package org.apache.paimon.globalindex; -import java.io.Serializable; -import java.util.Iterator; - /** A {@link GlobalIndexer} that supports vector similarity search. */ public interface VectorGlobalIndexer extends GlobalIndexer { /** Returns the metric name used to convert vector distances to comparable scores. */ String metric(); - - /** Zero-based ordinals in the non-null input stream, in the order required for training. */ - default long[] planGlobalTrainingSample(long vectorCount) { - throw new UnsupportedOperationException("Centroid training is not supported."); - } - - /** Train using ordered, preselected batches and the full non-null corpus size. */ - default VectorTrainingModel trainGlobalModel( - long vectorCount, Iterator batches) { - throw new UnsupportedOperationException("Centroid training is not supported."); - } - - /** A row-major batch selected by the distributed sampler. */ - final class TrainingBatch implements Serializable { - private static final long serialVersionUID = 1L; - public final float[] vectors; - public final int count; - - public TrainingBatch(float[] vectors, int count) { - this.vectors = vectors; - this.count = count; - } - } - - default VectorGlobalModelTrainer createGlobalModelTrainer() { - throw new UnsupportedOperationException( - "Centroid sharding is not supported by this vector index."); - } - - default VectorTrainingModel restoreTrainingModel(byte[] payload) { - throw new UnsupportedOperationException( - "Centroid sharding is not supported by this vector index."); - } } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java index 483b11160d26..8b8f195898b6 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java @@ -26,8 +26,9 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -37,23 +38,77 @@ /** Manifest metadata shared by vector implementations and engine-independent planning. */ public final class VectorIndexShardMeta { - public static final String CENTROID_BASED = "centroid-based"; - public static final String ABSOLUTE_ROW_ID = "ABSOLUTE_ROW_ID"; + private static final byte[] MAGIC = {'P', 'C', 'S', 'M'}; + private static final byte VERSION = 1; + // The preview writer emitted this key first in a LinkedHashMap. + private static final byte[] PREVIEW_PREFIX = + "{\"shardMode\":\"centroid-based\",".getBytes(StandardCharsets.UTF_8); + private static final String ABSOLUTE_ROW_ID = "ABSOLUTE_ROW_ID"; private static final ObjectMapper MAPPER = new ObjectMapper(); private static final TypeReference> TYPE = new TypeReference>() {}; private final Map values; + @Nullable private final byte[] payloadMeta; - private VectorIndexShardMeta(Map values) { + private VectorIndexShardMeta(Map values, @Nullable byte[] payloadMeta) { this.values = values; + this.payloadMeta = payloadMeta; } - public static VectorIndexShardMeta deserialize(@Nullable byte[] bytes) { + /** Ordinary index metadata is opaque and can use any encoding. */ + public static boolean isWrapped(@Nullable byte[] bytes) { + return startsWith(bytes, MAGIC) || startsWith(bytes, PREVIEW_PREFIX); + } + + private static boolean startsWith(@Nullable byte[] bytes, byte[] prefix) { + if (bytes == null || bytes.length < prefix.length) { + return false; + } + for (int i = 0; i < prefix.length; i++) { + if (bytes[i] != prefix[i]) { + return false; + } + } + return true; + } + + public static VectorIndexShardMeta deserialize(byte[] bytes) { try { - return new VectorIndexShardMeta( - bytes == null ? Collections.emptyMap() : MAPPER.readValue(bytes, TYPE)); + Map descriptor; + byte[] payload = null; + if (startsWith(bytes, PREVIEW_PREFIX)) { + descriptor = MAPPER.readValue(bytes, TYPE); + } else { + checkArgument( + startsWith(bytes, MAGIC) + && bytes.length >= MAGIC.length + 1 + 2 * Integer.BYTES, + "Invalid centroid metadata header."); + ByteBuffer input = ByteBuffer.wrap(bytes); + input.position(MAGIC.length); + checkArgument(input.get() == VERSION, "Unsupported centroid metadata version."); + int descriptorSize = input.getInt(); + int payloadSize = input.getInt(); + checkArgument( + descriptorSize > 0 + && descriptorSize <= input.remaining() + && payloadSize >= -1 + && input.remaining() - descriptorSize == Math.max(0, payloadSize), + "Invalid centroid metadata lengths."); + descriptor = MAPPER.readValue(bytes, input.position(), descriptorSize, TYPE); + if (payloadSize >= 0) { + input.position(input.position() + descriptorSize); + payload = new byte[payloadSize]; + input.get(payload); + } + } + checkArgument( + descriptor != null + && descriptor.containsKey("nlist") + != descriptor.containsKey("centroid"), + "Expected a routing model or a centroid shard."); + return new VectorIndexShardMeta(descriptor, payload); } catch (IOException e) { - throw new IllegalArgumentException("Invalid vector shard metadata", e); + throw new IllegalArgumentException("Invalid centroid metadata", e); } } @@ -68,7 +123,7 @@ public static VectorIndexShardMeta routing(String modelId, int nlist, List r.from + ":" + r.to) .collect(Collectors.joining(","))); - return new VectorIndexShardMeta(values); + return new VectorIndexShardMeta(values, null); } public static VectorIndexShardMeta centroid(String modelId, int centroid) { @@ -76,23 +131,23 @@ public static VectorIndexShardMeta centroid(String modelId, int centroid) { Map values = base(modelId); values.put("centroid", Integer.toString(centroid)); values.put("rowIdEncoding", ABSOLUTE_ROW_ID); - return new VectorIndexShardMeta(values); + return new VectorIndexShardMeta(values, null); } private static Map base(String modelId) { checkArgument(modelId != null && !modelId.isEmpty(), "Model id is required."); Map values = new LinkedHashMap<>(); - values.put("shardMode", CENTROID_BASED); values.put("modelId", modelId); return values; } - public boolean centroidBased() { - return CENTROID_BASED.equals(values.get("shardMode")); + public boolean routingModel() { + return values.containsKey("nlist"); } - public boolean routingModel() { - return centroidBased() && values.containsKey("nlist"); + @Nullable + public byte[] payloadMeta() { + return payloadMeta; } public String modelId() { @@ -132,10 +187,27 @@ public List sourceRowRanges() { } public byte[] serialize() { + return wrap(payloadMeta); + } + + /** Store routing information and the underlying writer's opaque metadata in indexMeta. */ + public byte[] wrap(@Nullable byte[] payload) { try { - return MAPPER.writeValueAsBytes(values); + byte[] descriptor = MAPPER.writeValueAsBytes(values); + int size = Math.addExact(MAGIC.length + 1 + 2 * Integer.BYTES, descriptor.length); + ByteBuffer output = + ByteBuffer.allocate(Math.addExact(size, payload == null ? 0 : payload.length)); + output.put(MAGIC) + .put(VERSION) + .putInt(descriptor.length) + .putInt(payload == null ? -1 : payload.length) + .put(descriptor); + if (payload != null) { + output.put(payload); + } + return output.array(); } catch (IOException e) { - throw new IllegalStateException("Cannot serialize vector shard metadata", e); + throw new IllegalStateException("Cannot serialize centroid metadata", e); } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java index 72c75c864762..3a332cb3d660 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexUtils.java @@ -25,14 +25,6 @@ public final class VectorIndexUtils { private VectorIndexUtils() {} - public static boolean isVectorIndexType(String type) { - return "ivf-flat".equals(type) - || "ivf-pq".equals(type) - || "ivf-sq".equals(type) - || "ivf-rq".equals(type) - || "diskann".equals(type); - } - public static float[] toFloatArray(Object value) { if (value instanceof float[]) { return ((float[]) value).clone(); diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java index 8e1a5cc688b8..b5fd84c87281 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java @@ -18,9 +18,11 @@ package org.apache.paimon.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.utils.Range; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -32,10 +34,13 @@ public final class CentroidVectorIndexUtils { private CentroidVectorIndexUtils() {} public static boolean centroidBased(IndexFileMeta file) { - return VectorIndexUtils.isVectorIndexType(file.indexType()) - && file.globalIndexMeta() != null - && VectorIndexShardMeta.deserialize(file.globalIndexMeta().indexMeta()) - .centroidBased(); + return file.globalIndexMeta() != null + && VectorIndexShardMeta.isWrapped(file.globalIndexMeta().indexMeta()) + && CoreOptions.isVectorIndexType(file.indexType()); + } + + public static boolean isRoutingModel(IndexFileMeta file) { + return centroidBased(file) && metadata(file).routingModel(); } public static VectorIndexShardMeta metadata(IndexFileMeta file) { @@ -44,42 +49,43 @@ public static VectorIndexShardMeta metadata(IndexFileMeta file) { public static IndexFileMeta validateModel(List files) { IndexFileMeta routing = null; + String modelId = null; + int nlist = 0; + Set centroids = new HashSet<>(); for (IndexFileMeta file : files) { checkArgument( centroidBased(file), "Cannot mix centroid-based and range-based vector shards."); - if (file.isRoutingModel()) { + VectorIndexShardMeta meta = metadata(file); + String id = meta.modelId(); + checkArgument( + modelId == null || modelId.equals(id), + "Centroid shards belong to different models."); + modelId = id; + if (meta.routingModel()) { checkArgument( routing == null, "Multiple routing models for the same partition and index."); routing = file; + nlist = meta.nlist(); + meta.sourceRowRanges(); + } else { + checkArgument(centroids.add(meta.centroid()), "Duplicate centroid shard."); } } checkArgument(routing != null, "Missing centroid routing model."); - VectorIndexShardMeta model = metadata(routing); - Set centroids = new HashSet<>(); - for (IndexFileMeta file : files) { - checkArgument( - model.modelId().equals(metadata(file).modelId()), - "Centroid shards belong to different models."); - if (!file.isRoutingModel()) { - int centroid = metadata(file).centroid(); - checkArgument( - centroid < model.nlist() && centroids.add(centroid), - "Duplicate or invalid centroid shard."); - } + for (int centroid : centroids) { + checkArgument(centroid < nlist, "Invalid centroid shard."); } - checkArgument(centroids.size() == model.nlist(), "Incomplete centroid model shard set."); - model.sourceRowRanges(); + checkArgument(centroids.size() == nlist, "Incomplete centroid model shard set."); return routing; } public static List coverage(IndexFileMeta file) { if (!centroidBased(file)) { - return java.util.Collections.singletonList(file.globalIndexMeta().rowRange()); + return Collections.singletonList(file.globalIndexMeta().rowRange()); } - return file.isRoutingModel() - ? metadata(file).sourceRowRanges() - : java.util.Collections.emptyList(); + VectorIndexShardMeta meta = metadata(file); + return meta.routingModel() ? meta.sourceRowRanges() : Collections.emptyList(); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java index 1aae2e641ba8..cc77e30438f3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java @@ -41,9 +41,6 @@ @Public public class IndexFileMeta { - public static final String DATA_FILE = "INDEX"; - public static final String ROUTING_MODEL = "ROUTING_MODEL"; - public static final RowType SCHEMA = new RowType( false, @@ -57,14 +54,12 @@ public class IndexFileMeta { "_DELETIONS_VECTORS_RANGES", new ArrayType(true, DeletionVectorMeta.SCHEMA)), new DataField(5, "_EXTERNAL_PATH", newStringType(true)), - new DataField(6, "_GLOBAL_INDEX", GlobalIndexMeta.SCHEMA), - new DataField(7, "_INDEX_FILE_KIND", newStringType(true)))); + new DataField(6, "_GLOBAL_INDEX", GlobalIndexMeta.SCHEMA))); private final String indexType; private final String fileName; private final long fileSize; private final long rowCount; - private final String fileKind; @Nullable private final GlobalIndexMeta globalIndexMeta; @@ -94,26 +89,6 @@ public IndexFileMeta( @Nullable LinkedHashMap dvRanges, @Nullable String externalPath, @Nullable GlobalIndexMeta globalIndexMeta) { - this( - indexType, - fileName, - fileSize, - rowCount, - dvRanges, - externalPath, - globalIndexMeta, - DATA_FILE); - } - - public IndexFileMeta( - String indexType, - String fileName, - long fileSize, - long rowCount, - @Nullable LinkedHashMap dvRanges, - @Nullable String externalPath, - @Nullable GlobalIndexMeta globalIndexMeta, - @Nullable String fileKind) { this.indexType = indexType; this.fileName = fileName; this.fileSize = fileSize; @@ -121,10 +96,6 @@ public IndexFileMeta( this.dvRanges = dvRanges; this.externalPath = externalPath; this.globalIndexMeta = globalIndexMeta; - this.fileKind = fileKind == null ? DATA_FILE : fileKind; - if (!DATA_FILE.equals(this.fileKind) && !ROUTING_MODEL.equals(this.fileKind)) { - throw new IllegalArgumentException("Unknown index file kind: " + fileKind); - } } public IndexFileMeta( @@ -141,14 +112,6 @@ public String indexType() { return indexType; } - public String fileKind() { - return fileKind; - } - - public boolean isRoutingModel() { - return ROUTING_MODEL.equals(fileKind); - } - @Nullable public GlobalIndexMeta globalIndexMeta() { return globalIndexMeta; @@ -185,7 +148,6 @@ public boolean equals(Object o) { } IndexFileMeta that = (IndexFileMeta) o; return Objects.equals(indexType, that.indexType) - && Objects.equals(fileKind, that.fileKind) && Objects.equals(fileName, that.fileName) && fileSize == that.fileSize && rowCount == that.rowCount @@ -197,14 +159,7 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - indexType, - fileName, - fileSize, - rowCount, - dvRanges, - externalPath, - globalIndexMeta, - fileKind); + indexType, fileName, fileSize, rowCount, dvRanges, externalPath, globalIndexMeta); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java index 82419c8fde51..6e71c5f74a5b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java @@ -58,8 +58,7 @@ public InternalRow toRow(IndexFileMeta record) { record.rowCount(), dvMetasToRowArrayData(record.dvRanges()), fromString(record.externalPath()), - globalIndexRow, - record.isRoutingModel() ? fromString(record.fileKind()) : null); + globalIndexRow); } @Override @@ -90,8 +89,7 @@ public IndexFileMeta fromRow(InternalRow row) { row.getLong(3), row.isNullAt(4) ? null : rowArrayDataToDvMetas(row.getArray(4)), row.isNullAt(5) ? null : row.getString(5).toString(), - globalIndexMeta, - row.getFieldCount() <= 7 || row.isNullAt(7) ? null : row.getString(7).toString()); + globalIndexMeta); } public static InternalArray dvMetasToRowArrayData( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java deleted file mode 100644 index a717ad72e38b..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV5Deserializer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.index; - -import org.apache.paimon.data.InternalRow; -import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.ObjectSerializer; - -/** Reads the seven-field index metadata used by commit message versions 12 through 14. */ -public class IndexFileMetaV5Deserializer extends ObjectSerializer { - public IndexFileMetaV5Deserializer() { - super(new RowType(false, IndexFileMeta.SCHEMA.getFields().subList(0, 7))); - } - - @Override - public InternalRow toRow(IndexFileMeta record) { - throw new UnsupportedOperationException(); - } - - @Override - public IndexFileMeta fromRow(InternalRow row) { - return new IndexFileMetaSerializer().fromRow(row); - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java index 3fd90c68bdb0..f69716d455e6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java @@ -55,7 +55,6 @@ public class IndexManifestEntry { public static final String DELETION_VECTORS_RANGES = "_DELETIONS_VECTORS_RANGES"; public static final String EXTERNAL_PATH = "_EXTERNAL_PATH"; public static final String GLOBAL_INDEX = "_GLOBAL_INDEX"; - public static final String INDEX_FILE_KIND = "_INDEX_FILE_KIND"; public static final RowType SCHEMA = new RowType( @@ -73,8 +72,7 @@ public class IndexManifestEntry { DELETION_VECTORS_RANGES, new ArrayType(true, DeletionVectorMeta.SCHEMA)), new DataField(8, EXTERNAL_PATH, newStringType(true)), - new DataField(9, GLOBAL_INDEX, GlobalIndexMeta.SCHEMA), - new DataField(10, INDEX_FILE_KIND, newStringType(true)))); + new DataField(9, GLOBAL_INDEX, GlobalIndexMeta.SCHEMA))); public static final RowType MANIFEST_ROW_TYPE = ManifestSchemaUtils.withFormatIdentifier(SCHEMA); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java index 3f7912624648..c37bb77a0022 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java @@ -76,8 +76,7 @@ public InternalRow toRow(IndexManifestEntry record) { indexFile.rowCount(), dvMetasToRowArrayData(indexFile.dvRanges()), fromString(indexFile.externalPath()), - globalIndexRow, - indexFile.isRoutingModel() ? fromString(indexFile.fileKind()) : null); + globalIndexRow); } @Override @@ -124,10 +123,7 @@ private IndexManifestEntry fromDataRow(InternalRow row) { row.getLong(6), row.isNullAt(7) ? null : rowArrayDataToDvMetas(row.getArray(7)), row.isNullAt(8) ? null : row.getString(8).toString(), - globalIndexMeta, - row.getFieldCount() <= 10 || row.isNullAt(10) - ? null - : row.getString(10).toString())); + globalIndexMeta)); } public static Function partitionGetter() { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java index 606f286b566b..dd0b00483af1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java @@ -236,24 +236,15 @@ public List combine( for (IndexManifestEntry entry : added) { indexEntries.put(entry.indexFile().fileName(), entry); } - boolean checkCentroidModels = - prevIndexFiles.stream().anyMatch(entry -> entry.indexFile().isRoutingModel()) - || newIndexFiles.stream() - .anyMatch( - entry -> - entry.indexFile().isRoutingModel() - || CentroidVectorIndexUtils - .centroidBased( - entry.indexFile())); - if (!checkCentroidModels) { + if (indexEntries.values().stream() + .noneMatch( + entry -> CentroidVectorIndexUtils.centroidBased(entry.indexFile()))) { return new ArrayList<>(indexEntries.values()); } Map>> models = new HashMap<>(); for (IndexManifestEntry entry : indexEntries.values()) { IndexFileMeta file = entry.indexFile(); - if (!org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType( - file.indexType()) - || file.globalIndexMeta() == null) { + if (file.globalIndexMeta() == null) { continue; } models.computeIfAbsent(entry.partition(), p -> new HashMap<>()) diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java index 9a59f97017aa..81715cb6a2cd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionChangesProvider.java @@ -21,6 +21,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.append.dataevolution.SerializationAssignment; import org.apache.paimon.fs.FileIO; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; @@ -146,7 +147,7 @@ private CommitChanges mapChanges(SerializationAssignment assignment) { continue; } checkState( - !org.apache.paimon.globalindex.CentroidVectorIndexUtils.centroidBased(file), + !CentroidVectorIndexUtils.centroidBased(file), "Cannot reuse centroid indexes across row-ID reassignment; rebuild the index."); Range range = assignment.mapRowRange(entry.partition(), meta.rowRange()); GlobalIndexMeta mapped = @@ -169,8 +170,7 @@ private CommitChanges mapChanges(SerializationAssignment assignment) { file.rowCount(), file.dvRanges(), file.externalPath(), - mapped, - file.fileKind()))); + mapped))); } return new CommitChanges(files, changes.changelogFiles, indexes); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java index 07b1d200e034..7ce22a924241 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java @@ -646,8 +646,7 @@ private Optional checkCentroidSourceChanges( Snapshot latestSnapshot, List indexEntries) { for (IndexManifestEntry index : indexEntries) { if (index.kind() != FileKind.ADD - || !index.indexFile().isRoutingModel() - || !CentroidVectorIndexUtils.centroidBased(index.indexFile())) { + || !CentroidVectorIndexUtils.isRoutingModel(index.indexFile())) { continue; } long baseline = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java index bdf385b05c2e..2a3e3ebd81bf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java @@ -26,7 +26,6 @@ import org.apache.paimon.index.IndexFileMetaV2Deserializer; import org.apache.paimon.index.IndexFileMetaV3Deserializer; import org.apache.paimon.index.IndexFileMetaV4Deserializer; -import org.apache.paimon.index.IndexFileMetaV5Deserializer; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMeta08Serializer; @@ -54,7 +53,7 @@ /** {@link VersionedSerializer} for {@link CommitMessage}. */ public class CommitMessageSerializer implements VersionedSerializer { - public static final int CURRENT_VERSION = 15; + public static final int CURRENT_VERSION = 14; private final DataFileMetaSerializer dataFileSerializer; private final IndexFileMetaSerializer indexEntrySerializer; @@ -69,7 +68,6 @@ public class CommitMessageSerializer implements VersionedSerializer> fileDeserializer( private IOExceptionSupplier> indexEntryDeserializer( int version, DataInputView view) { - if (version >= 15) { + if (version >= 12) { return () -> indexEntrySerializer.deserializeList(view); - } else if (version >= 12) { - if (indexEntryV5Deserializer == null) { - indexEntryV5Deserializer = new IndexFileMetaV5Deserializer(); - } - return () -> indexEntryV5Deserializer.deserializeList(view); } else if (version == 11) { if (indexEntryV4Deserializer == null) { indexEntryV4Deserializer = new IndexFileMetaV4Deserializer(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java index 9560c8d7abd3..355b60d8b75e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexVectorSearchSplit.java @@ -20,7 +20,6 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexFileMetaSerializer; -import org.apache.paimon.index.IndexFileMetaV5Deserializer; import org.apache.paimon.io.DataInputViewStreamWrapper; import org.apache.paimon.io.DataOutputViewStreamWrapper; @@ -35,7 +34,7 @@ public class IndexVectorSearchSplit extends VectorSearchSplit { private static final long serialVersionUID = 1L; - private static final int VERSION = 2; + private static final int VERSION = 1; private static final ThreadLocal INDEX_SERIALIZER = ThreadLocal.withInitial(IndexFileMetaSerializer::new); @@ -86,21 +85,15 @@ private void writeObject(ObjectOutputStream out) throws IOException { private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); int version = in.readInt(); - if (version != 1 && version != VERSION) { + if (version != VERSION) { throw new IOException("Unsupported IndexVectorSearchSplit version: " + version); } IndexFileMetaSerializer serializer = INDEX_SERIALIZER.get(); DataInputViewStreamWrapper view = new DataInputViewStreamWrapper(in); this.rowRangeStart = view.readLong(); this.rowRangeEnd = view.readLong(); - if (version == 1) { - IndexFileMetaV5Deserializer legacy = new IndexFileMetaV5Deserializer(); - this.vectorIndexFiles = legacy.deserializeList(view); - this.scalarIndexFiles = legacy.deserializeList(view); - } else { - this.vectorIndexFiles = serializer.deserializeList(view); - this.scalarIndexFiles = serializer.deserializeList(view); - } + this.vectorIndexFiles = serializer.deserializeList(view); + this.scalarIndexFiles = serializer.deserializeList(view); } @Override diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java index c7a66e21b6c7..5a9bef4dd605 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java @@ -33,6 +33,29 @@ /** Source coverage belongs to complete model generations, never to sparse shard bounds. */ class CentroidVectorIndexUtilsTest { + @Test + void onlyWrappedMetadataAffectsPlanning() { + byte[] opaque = {(byte) 0xff, 0, 7, 42}; + assertThat(CentroidVectorIndexUtils.centroidBased(file(opaque, false))).isFalse(); + VectorIndexShardMeta shard = VectorIndexShardMeta.centroid("g", 7); + byte[] wrapped = shard.wrap(opaque); + VectorIndexShardMeta decoded = VectorIndexShardMeta.deserialize(wrapped); + assertThat(decoded.payloadMeta()).containsExactly(opaque); + assertThat(decoded.centroid()).isEqualTo(7); + assertThat(decoded.modelId()).isEqualTo("g"); + assertThat(VectorIndexShardMeta.deserialize(shard.wrap(new byte[0])).payloadMeta()) + .isEmpty(); + assertThat(VectorIndexShardMeta.deserialize(shard.serialize()).payloadMeta()).isNull(); + assertThatThrownBy( + () -> + VectorIndexShardMeta.deserialize( + Arrays.copyOf(wrapped, wrapped.length - 1))) + .isInstanceOf(IllegalArgumentException.class); + wrapped[4]++; + assertThatThrownBy(() -> VectorIndexShardMeta.deserialize(wrapped)) + .hasMessageContaining("version"); + } + @Test void testSparseShardsDoNotClaimEnvelopeCoverage() { List files = model("a"); @@ -88,7 +111,6 @@ private static IndexFileMeta file(byte[] meta, boolean routing) { routing ? 0 : 10, null, null, - new GlobalIndexMeta(10, 1009, 7, null, meta), - routing ? IndexFileMeta.ROUTING_MODEL : IndexFileMeta.DATA_FILE); + new GlobalIndexMeta(10, 1009, 7, null, meta)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java index 51e7a742e5bf..4451d7997a62 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java @@ -19,11 +19,15 @@ package org.apache.paimon.index; import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; +import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; +import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Random; @@ -34,19 +38,24 @@ public class IndexFileMetaSerializerTest extends ObjectSerializerTestBase centroidModel(String id, String type) { centroid == -1 ? 0 : 10, null, null, - new GlobalIndexMeta(10, 100, 1, null, metadata), - centroid == -1 ? IndexFileMeta.ROUTING_MODEL : IndexFileMeta.DATA_FILE); + new GlobalIndexMeta(10, 100, 1, null, metadata)); entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, file)); } return entries; diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 30b50cb66cfe..459e0e5239b4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -1759,8 +1759,7 @@ void testCentroidIndexChecksSourceCoverageRatherThanSparseEnvelope() { null, meta, new org.apache.paimon.index.DataEvolutionIndexSourceMeta(1) - .serialize()), - IndexFileMeta.ROUTING_MODEL); + .serialize())); IndexManifestEntry index = new IndexManifestEntry(ADD, BinaryRow.EMPTY_ROW, 0, file); List data = Arrays.asList( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java index 418ea005d3c8..bb955b57822b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java @@ -18,8 +18,12 @@ package org.apache.paimon.table.sink; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; +import org.apache.paimon.index.DataEvolutionIndexSourceMeta; +import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; @@ -29,6 +33,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.Arrays; +import java.util.Base64; import static org.apache.paimon.index.IndexFileMetaSerializerTest.randomIndexFile; import static org.apache.paimon.manifest.ManifestCommittableSerializerTest.randomCompactIncrement; @@ -39,6 +44,26 @@ /** Test for {@link CommitMessageSerializer}. */ public class CommitMessageSerializerTest { + @Test + void readsPreviewRoutingCommit() throws IOException { + // Produced by the eight-field v15 writer. indexMeta contains its original JSON descriptor. + byte[] bytes = + Base64.getDecoder() + .decode( + "AAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAQAAMAAAAAAAAAgAAABIAAAAbW9k" + + "ZWwAAIUMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABQAAAADQAAAPAAAABpdmYtZmxh" + + "dAAIAAAAAAAADAAAAAAAAAATAAAAAAAAAAIAAAAAAAAAAAAAAAAAAABSAAAAOAAAABAAAACQAAAAeyJz" + + "aGFyZE1vZGUiOiJjZW50cm9pZC1iYXNlZCIsIm1vZGVsSWQiOiJnIiwibmxpc3QiOiIxIiwic291cmNl" + + "Um93UmFuZ2VzIjoiMTI6MTkifQAAAAAAAERFSVgAAAABAAAAAAAAAAVST1VUSU5HX01PREVMAAAAAAAA" + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAF"); + CommitMessageImpl message = + (CommitMessageImpl) new CommitMessageSerializer().deserialize(15, bytes); + IndexFileMeta model = message.newFilesIncrement().newIndexFiles().get(0); + assertThat(CentroidVectorIndexUtils.isRoutingModel(model)).isTrue(); + assertThat(CentroidVectorIndexUtils.coverage(model)).containsExactly(new Range(12, 19)); + assertThat(DataEvolutionIndexSourceMeta.fromIndexFile(model).scanSnapshotId()).isEqualTo(5); + } + @Test public void test() throws IOException { CommitMessageSerializer serializer = new CommitMessageSerializer(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java index e53540688eb9..563283f3a341 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java @@ -22,13 +22,13 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexBuilderUtils; import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.IndexedSplit; import org.apache.paimon.globalindex.ResultEntry; -import org.apache.paimon.globalindex.VectorGlobalIndexer; import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.VectorIndexUtils; import org.apache.paimon.globalindex.VectorTrainingModel; @@ -96,10 +96,7 @@ public List buildIndex( DataField indexField, Options options) throws IOException { - checkArgument( - org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType(indexType), - "Unsupported centroid index type: %s", - indexType); + CentroidVectorGlobalIndexer indexer = createIndexer(indexType, indexField, options); checkArgument( table.coreOptions().dataEvolutionEnabled(), "Centroid sharding requires a Data Evolution table."); @@ -157,7 +154,6 @@ public List buildIndex( DefaultGlobalIndexTopoBuilder.parallelism(splits.size(), options)) .mapPartitions( input -> new VectorRows(BuildContext.restore(contextBytes), input)); - VectorGlobalIndexer indexer = context.indexer(); byte[] payload = train(sc, contextBytes, rows, indexer, context.getter()); int nlist; try (VectorTrainingModel model = indexer.restoreTrainingModel(payload)) { @@ -214,16 +210,6 @@ public List buildIndex( .serialize())), context.sourceMeta()) .get(0); - modelMeta = - new IndexFileMeta( - modelMeta.indexType(), - modelMeta.fileName(), - modelMeta.fileSize(), - 0, - null, - modelMeta.externalPath(), - modelMeta.globalIndexMeta(), - IndexFileMeta.ROUTING_MODEL); messages.add(context.message(Collections.singletonList(modelMeta))); List old = GlobalIndexBuilderUtils.currentIndexEntries( @@ -279,7 +265,7 @@ private static byte[] train( JavaSparkContext sc, byte[] contextBytes, JavaRDD rows, - VectorGlobalIndexer indexer, + CentroidVectorGlobalIndexer indexer, InternalRow.FieldGetter getter) { // Count actual non-null rows at the pinned snapshot. Row-ID spans can contain // gaps, nulls and deletions, so a split's planned row count is insufficient. @@ -423,6 +409,17 @@ private static final class RoutedVector implements Serializable { } } + static CentroidVectorGlobalIndexer createIndexer( + String indexType, DataField field, Options options) { + GlobalIndexer indexer = + GlobalIndexer.create(indexType, Collections.singletonList(field), options); + checkArgument( + indexer instanceof CentroidVectorGlobalIndexer, + "Index '%s' does not support centroid sharding.", + indexType); + return (CentroidVectorGlobalIndexer) indexer; + } + private static final class BuildContext implements Serializable { private static final long serialVersionUID = 1L; private final FileStoreTable table; @@ -458,9 +455,8 @@ static BuildContext restore(byte[] bytes) throws IOException, ClassNotFoundExcep bytes, CentroidGlobalIndexTopoBuilder.class.getClassLoader()); } - VectorGlobalIndexer indexer() { - return (VectorGlobalIndexer) - GlobalIndexer.create(indexType, Collections.singletonList(field), options); + CentroidVectorGlobalIndexer indexer() { + return createIndexer(indexType, field, options); } GlobalIndexFileReadWrite io() { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java index b4f73b0ce4fa..2776679f0860 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidTrainingSample.java @@ -19,7 +19,7 @@ package org.apache.paimon.spark.globalindex; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer.TrainingBatch; import org.apache.paimon.globalindex.VectorIndexUtils; import java.io.Serializable; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java index 0429be8485ec..aca61d8e50df 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java @@ -18,9 +18,12 @@ package org.apache.paimon.spark.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.options.Options; import org.apache.paimon.spark.globalindex.sorted.SortedIndexTopoBuilder; +import java.util.Map; + /** Utility class for creating {@link GlobalIndexTopologyBuilder} implementations. */ public class GlobalIndexTopologyBuilderUtils { @@ -39,8 +42,8 @@ public static GlobalIndexTopologyBuilder createTopoBuilder( String indexType, Options options, String fieldName) { String key = indexType + ".shard"; String fieldKey = "fields." + fieldName + ".shard"; - java.util.Map dynamic = options.dynamicOptions(); - java.util.Map stored = options.toMap(); + Map dynamic = options.dynamicOptions(); + Map stored = options.toMap(); String shard = dynamic.containsKey(fieldKey) ? dynamic.get(fieldKey) @@ -48,7 +51,7 @@ public static GlobalIndexTopologyBuilder createTopoBuilder( ? dynamic.get(key) : stored.getOrDefault( fieldKey, stored.getOrDefault(key, "range-based")); - if (org.apache.paimon.globalindex.VectorIndexUtils.isVectorIndexType(indexType)) { + if (CoreOptions.isVectorIndexType(indexType)) { if ("centroid-based".equals(shard)) { return new CentroidGlobalIndexTopoBuilder(); } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java index 439cca56a05e..7603641dfc5e 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/CentroidTrainingSampleTest.java @@ -20,7 +20,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer.TrainingBatch; import org.junit.jupiter.api.Test; diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java index 42d8f9fc637e..fa12c11d62dc 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java @@ -20,6 +20,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.spark.globalindex.sorted.SortedIndexTopoBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.Test; @@ -36,6 +38,17 @@ /** Tests for {@link DefaultGlobalIndexTopoBuilder}. */ public class DefaultGlobalIndexTopoBuilderTest { + @Test + void centroidBuildChecksIndexerCapability() { + DataField field = new DataField(0, "v", DataTypes.ARRAY(DataTypes.FLOAT())); + assertThatThrownBy( + () -> + CentroidGlobalIndexTopoBuilder.createIndexer( + "test-vector-ann", field, new Options())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not support centroid sharding"); + } + @Test void testBitmapUsesSortedTopologyBuilder() { assertThat(GlobalIndexTopologyBuilderUtils.createTopoBuilder("bitmap")) diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java index 9f3d288ceff0..22d37d565dd9 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java @@ -26,11 +26,11 @@ import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.VectoredReadable; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; import org.apache.paimon.globalindex.OffsetGlobalIndexReader; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; -import org.apache.paimon.globalindex.VectorIndexUtils; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.IndexManifestEntry; @@ -158,7 +158,7 @@ private VectorShardingBenchmark(String[] args) { Arrays.asList( System.getProperty("benchmark.modes", "range-based,centroid-based") .split(",")); - if (!VectorIndexUtils.isVectorIndexType(indexType) + if (!Arrays.asList("ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann").contains(indexType) || (readSnapshot != null && !skipBuild) || configuredModes.isEmpty() || new HashSet<>(configuredModes).size() != configuredModes.size() @@ -372,7 +372,8 @@ private void run() throws Exception { if (files.isEmpty()) { throw new IllegalStateException("Build produced no index"); } - boolean centroid = files.stream().anyMatch(IndexFileMeta::isRoutingModel); + boolean centroid = + files.stream().anyMatch(CentroidVectorIndexUtils::isRoutingModel); if (centroid != "centroid-based".equals(mode)) { throw new IllegalStateException( "Existing index does not match the requested mode"); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala index ab30cae1837a..5215936db8c2 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala @@ -19,6 +19,7 @@ package org.apache.paimon.spark.sql import org.apache.paimon.globalindex.{VectorIndexShardMeta, VectorTrainingModelIO} +import org.apache.paimon.globalindex.CentroidVectorIndexUtils import org.apache.paimon.index.IndexFileMeta import org.apache.paimon.spark.PaimonSparkTestBase @@ -81,7 +82,7 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { val firstFiles = indexes.map(_.fileName()).toSet assert(indexes.size == 5) - assert(indexes.count(_.isRoutingModel()) == 1) + assert(indexes.count(f => CentroidVectorIndexUtils.isRoutingModel(f)) == 1) assert(nearest(300) == 300) // Partition 1 lies inside the indexed partition's min/max envelope, but is unindexed. assert(nearest(200) == 200) @@ -138,7 +139,7 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { .scan(table.snapshotManager().latestSnapshot(), "ivf-flat") .asScala .map(_.indexFile()) - .find(_.isRoutingModel()) + .find(f => CentroidVectorIndexUtils.isRoutingModel(f)) .get val meta = VectorIndexShardMeta.deserialize(model.globalIndexMeta().indexMeta()) val path = table.store().pathFactory().globalIndexFileFactory().toPath(model) @@ -204,10 +205,11 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { .scan(table.snapshotManager().latestSnapshot(), kind) .asScala .map(_.indexFile()) - assert(files.size == 5 && files.count(_.isRoutingModel()) == 1) + assert( + files.size == 5 && files.count(f => CentroidVectorIndexUtils.isRoutingModel(f)) == 1) assert(nearest("centroid-based").toSeq == Seq(200)) val centroids = files - .filterNot(_.isRoutingModel()) + .filterNot(f => CentroidVectorIndexUtils.isRoutingModel(f)) .map(f => VectorIndexShardMeta.deserialize(f.globalIndexMeta().indexMeta()).centroid()) assert(centroids.toSet == Set(0, 1, 2, 3)) spark diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java index d77128a8a450..48d291844d65 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java @@ -68,32 +68,43 @@ final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexRea this.executor = executor; this.indexType = indexType; GlobalIndexIOMeta routing = null; + VectorIndexShardMeta modelMeta = null; + String modelId = null; long count = 0; for (GlobalIndexIOMeta file : entries) { - VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(file.metadata()); checkArgument( - meta.centroidBased(), "Cannot mix centroid-based and range-based shards."); + VectorIndexShardMeta.isWrapped(file.metadata()), + "Cannot mix centroid-based and range-based shards."); + VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(file.metadata()); + String id = meta.modelId(); + checkArgument(modelId == null || modelId.equals(id), "Mismatched centroid model."); + modelId = id; if (meta.routingModel()) { + modelMeta = meta; checkArgument(routing == null, "Multiple routing models."); routing = file; } else { checkArgument( file.rowCount() >= 0, "Centroid shards must record their vector count."); checkArgument( - shards.put(meta.centroid(), file) == null, "Duplicate centroid shard."); + shards.put( + meta.centroid(), + new GlobalIndexIOMeta( + file.filePath(), + file.fileSize(), + file.rowCount(), + meta.payloadMeta())) + == null, + "Duplicate centroid shard."); count = Math.addExact(count, file.rowCount()); } } checkArgument(routing != null, "Missing routing model."); this.routing = routing; - this.metadata = VectorIndexShardMeta.deserialize(routing.metadata()); + this.metadata = modelMeta; checkArgument(shards.size() == metadata.nlist(), "Incomplete centroid shard set."); - for (Map.Entry shard : shards.entrySet()) { - VectorIndexShardMeta data = - VectorIndexShardMeta.deserialize(shard.getValue().metadata()); - checkArgument( - shard.getKey() < metadata.nlist() && metadata.modelId().equals(data.modelId()), - "Mismatched centroid model."); + for (int id : shards.keySet()) { + checkArgument(id < metadata.nlist(), "Invalid centroid ID."); } this.vectorCount = count; } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java index 28fd77094e9a..b81a3be7d0e5 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java @@ -335,10 +335,11 @@ public void write(byte[] b) throws IOException { identifier, System.currentTimeMillis() - buildStart); + byte[] payloadMeta = new VectorIndexMeta().serialize(); return new ResultEntry( fileName, shardMeta == null ? rowCount : count, - shardMeta == null ? new VectorIndexMeta().serialize() : shardMeta.serialize()); + shardMeta == null ? payloadMeta : shardMeta.wrap(payloadMeta)); } } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java index b5c180d1115b..ac302af4f397 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java @@ -18,11 +18,11 @@ package org.apache.paimon.vector.index; +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; -import org.apache.paimon.globalindex.VectorGlobalIndexer; import org.apache.paimon.globalindex.VectorGlobalModelTrainer; import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.VectorTrainingModel; @@ -42,7 +42,7 @@ import java.util.concurrent.ExecutorService; /** Native vector global indexer backed by paimon-vector-index-java. */ -public class NativeVectorGlobalIndexer implements VectorGlobalIndexer { +public class NativeVectorGlobalIndexer implements CentroidVectorGlobalIndexer { static final String DEFAULT_METRIC = "inner_product"; @@ -100,11 +100,7 @@ public GlobalIndexReader createReader( long totalRowCount, List rowRanges, ExecutorService executor) { - if (files.stream() - .anyMatch( - file -> - VectorIndexShardMeta.deserialize(file.metadata()) - .centroidBased())) { + if (files.stream().anyMatch(file -> VectorIndexShardMeta.isWrapped(file.metadata()))) { return new NativeCentroidRoutedIndexReader( fileReader, files, fieldType, executor, identifier); } diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java index e4948476b1df..9cb83c64cf4b 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java @@ -23,12 +23,12 @@ import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer.TrainingBatch; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; -import org.apache.paimon.globalindex.VectorGlobalIndexer.TrainingBatch; import org.apache.paimon.globalindex.VectorGlobalModelTrainer; import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.VectorTrainingModel; @@ -296,8 +296,11 @@ public PositionOutputStream newOutputStream(String name) ResultEntry result = writer.finish().get(0); Path path = new Path(directory, result.fileName()); if (result.rowCount() > 0) { - assertThat(VectorIndexShardMeta.deserialize(result.meta()).centroid()) - .isEqualTo(c); + VectorIndexShardMeta descriptor = + VectorIndexShardMeta.deserialize(result.meta()); + assertThat(descriptor.centroid()).isEqualTo(c); + assertThat(descriptor.payloadMeta()) + .isEqualTo(new VectorIndexMeta().serialize()); try (SeekableInputStream in = io.newInputStream(path); VectorIndexReader local = new VectorIndexReader( From 9f052b51350d9dee277a5b7c8cccbad6ecc65294 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 18 Nov 2026 21:29:50 +0800 Subject: [PATCH 4/7] [core] Keep the centroid model file mode in its envelope codec --- .../org/apache/paimon/globalindex/VectorTrainingModelIO.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java index 752cab3b3ed1..e326582d7229 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java @@ -32,13 +32,14 @@ /** Versioned Paimon envelope around the independently versioned native training payload. */ public final class VectorTrainingModelIO { private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String SHARD_MODE = "centroid-based"; private VectorTrainingModelIO() {} public static void write(OutputStream out, byte[] payload, String modelId) throws IOException { Map meta = new LinkedHashMap<>(); meta.put("version", 1); - meta.put("shardMode", VectorIndexShardMeta.CENTROID_BASED); + meta.put("shardMode", SHARD_MODE); meta.put("modelId", modelId); byte[] header = MAPPER.writeValueAsBytes(meta); DataOutputStream data = new DataOutputStream(out); @@ -67,7 +68,7 @@ public static byte[] read(InputStream input, long fileSize, String modelId, int Map metadata = MAPPER.readValue(header, new TypeReference>() {}); if (!Integer.valueOf(1).equals(metadata.get("version")) - || !VectorIndexShardMeta.CENTROID_BASED.equals(metadata.get("shardMode")) + || !SHARD_MODE.equals(metadata.get("shardMode")) || !modelId.equals(metadata.get("modelId"))) { throw new IOException("Unsupported or mismatched routing model envelope."); } From e6ff1bc7b9656d9f3288e3e52c74cb421ceba9ab Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 19 Nov 2026 10:40:27 +0800 Subject: [PATCH 5/7] [vector] Register centroid layouts as dedicated index types --- .../multimodal-table/global-index/vector.mdx | 41 ++-- .../CentroidVectorGlobalIndexer.java | 15 ++ .../CentroidVectorIndexFactory.java | 36 ++++ .../GlobalIndexerFactoryUtils.java | 14 ++ .../globalindex/CentroidVectorIndexUtils.java | 58 ++---- .../manifest/IndexManifestFileHandler.java | 63 +++--- .../DataEvolutionConflictDetection.java | 7 +- .../table/source/DataEvolutionVectorScan.java | 3 +- .../index/IndexFileMetaSerializerTest.java | 18 +- .../IndexManifestFileHandlerTest.java | 53 ----- .../commit/ConflictDetectionTest.java | 172 ++++++++-------- .../sink/CommitMessageSerializerTest.java | 25 --- .../CentroidGlobalIndexTopoBuilder.java | 16 +- .../DefaultGlobalIndexTopoBuilder.java | 7 - .../GlobalIndexTopologyBuilderUtils.java | 40 +--- .../procedure/CreateGlobalIndexProcedure.java | 3 +- .../DefaultGlobalIndexTopoBuilderTest.java | 29 +-- .../benchmark/VectorShardingBenchmark.java | 67 ++++--- .../spark/sql/CentroidVectorIndexTest.scala | 51 ++--- .../NativeCentroidRoutedIndexReader.java | 5 - .../NativeCentroidVectorGlobalIndexer.java | 184 ++++++++++++++++++ ...iveCentroidVectorGlobalIndexerFactory.java | 129 ++++++++++++ .../index/NativeVectorGlobalIndexWriter.java | 4 +- .../index/NativeVectorGlobalIndexer.java | 120 +----------- .../NativeVectorGlobalIndexerFactory.java | 55 ++++-- .../index/NativeVectorGlobalModelTrainer.java | 4 +- .../index/NativeVectorTrainingModel.java | 5 +- .../vector/index}/VectorIndexShardMeta.java | 70 +++---- .../vector/index}/VectorTrainingModelIO.java | 2 +- ...he.paimon.globalindex.GlobalIndexerFactory | 5 + .../manifest/CentroidIndexManifestTest.java | 115 +++++++++++ .../index/CentroidVectorGlobalIndexTest.java | 24 +-- .../index}/CentroidVectorIndexUtilsTest.java | 26 ++- .../NativeVectorGlobalIndexerFactoryTest.java | 33 ++++ .../index}/VectorTrainingModelIOTest.java | 2 +- 35 files changed, 891 insertions(+), 610 deletions(-) create mode 100644 paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java create mode 100644 paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java create mode 100644 paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java rename {paimon-common/src/main/java/org/apache/paimon/globalindex => paimon-vector/src/main/java/org/apache/paimon/vector/index}/VectorIndexShardMeta.java (74%) rename {paimon-common/src/main/java/org/apache/paimon/globalindex => paimon-vector/src/main/java/org/apache/paimon/vector/index}/VectorTrainingModelIO.java (98%) create mode 100644 paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java rename {paimon-core/src/test/java/org/apache/paimon/globalindex => paimon-vector/src/test/java/org/apache/paimon/vector/index}/CentroidVectorIndexUtilsTest.java (81%) rename {paimon-common/src/test/java/org/apache/paimon/globalindex => paimon-vector/src/test/java/org/apache/paimon/vector/index}/VectorTrainingModelIOTest.java (99%) diff --git a/docs/docs/multimodal-table/global-index/vector.mdx b/docs/docs/multimodal-table/global-index/vector.mdx index 0994b23248d3..b0ffa49d05d9 100644 --- a/docs/docs/multimodal-table/global-index/vector.mdx +++ b/docs/docs/multimodal-table/global-index/vector.mdx @@ -466,28 +466,30 @@ Use full 768-dimensional query vectors when searching either of these indexes. ## Centroid-Based Sharding in Spark -For a Data Evolution append table, Spark supports centroid-based sharding for -`ivf-flat`, `ivf-pq`, `ivf-sq`, `ivf-rq`, and `diskann`. It trains a partition-level -routing model and writes one file per centroid. The table must enable row tracking and +Spark can build `ivf-flat-centroid`, `ivf-pq-centroid`, `ivf-sq-centroid`, `ivf-rq-centroid` +and `diskann-centroid` on Data Evolution append tables. Each build writes one routing model +for the partition and one file per centroid. The table must enable row tracking and Data Evolution, have no primary key, and use `bucket=-1`. ```sql ALTER TABLE db.model_embeddings SET TBLPROPERTIES ( - 'ivf-pq.dimension' = '768', - 'ivf-pq.distance.metric' = 'cosine' + 'ivf-pq-centroid.dimension' = '768', + 'ivf-pq-centroid.distance.metric' = 'cosine' ); CALL sys.create_global_index( table => 'db.model_embeddings', index_column => 'embedding', - index_type => 'ivf-pq', + index_type => 'ivf-pq-centroid', partitions => 'dt="2026-11-17"', - options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=256' + options => 'ivf-pq-centroid.nlist=256' ); ``` -Set `.shard=centroid-based` and `.nlist` for any of these -types. The default `.shard=range-based` uses the existing build path. A centroid +Use the registered index type in both `index_type` and the option prefix. +The ordinary names (`ivf-pq`, for example) select range-based sharding. +Drop the previous index on the vector column before switching types, including +when different partitions are built in separate calls. A centroid build pins one snapshot. Spark map tasks count non-null vectors in their input splits. Using those counts and the configured training ratio, the driver plans the exact row ordinals that the outer IVF-FLAT trainer would retain. Each @@ -495,9 +497,8 @@ map task reads its own splits and shuffles selected vectors by reservoir slot. A single task trains the routing model on those vectors in the original training order. The driver broadcasts the model for the subsequent centroid shuffle. Sampling requires at least one -non-null vector. Build one partition per call; a non-partitioned table counts -as one partition. Partial builds and mixing range and centroid shards in the -same partition are unsupported. Repeating the call atomically replaces the +non-null vector. Each call builds a complete partition. An unpartitioned table +counts as one partition. Repeating the call atomically replaces the partition's existing model and shards of the requested type. Queries load the routing model, select centroids with `centroid.nprobe`, and open @@ -525,7 +526,7 @@ For IVF-PQ, this gives each shard its own inner centroids and PQ codebook. The o defaults to automatic sizing from that shard's vector count. For example, build with -`ivf-pq.shard=centroid-based,ivf-pq.nlist=64,ivf-pq.centroid.local-nlist=32`. +`ivf-pq-centroid.nlist=64,ivf-pq-centroid.centroid.local-nlist=32`. A query with `map('centroid.nprobe','4','ivf.nprobe','8')` selects four shard files and searches eight local lists in each populated shard. Both levels support automatic probe sizing. Empty local shards have a manifest row count @@ -539,10 +540,10 @@ IVF-FLAT training options as the IVF shard types. An empty shard uses an eight-byte `PCEMPTY1` marker with manifest row count zero. Queries skip that file. Empty slots remain in the complete generation so centroid identifiers stay stable. -For example, use `index_type => 'diskann'` and -`options => 'diskann.shard=centroid-based,diskann.nlist=64'`, then query with +For example, use `index_type => 'diskann-centroid'` and +`options => 'diskann-centroid.nlist=64'`, then query with `map('centroid.nprobe','8','diskann.l_search','64')`. Persist -`diskann.dimension` and `diskann.metric` on the table for raw fallback. +`diskann-centroid.dimension` and `diskann-centroid.metric` on the table for raw fallback. If source rows change while the build is running, including through compaction, the commit fails and the build must be retried. Appends outside its source ranges @@ -550,8 +551,10 @@ can proceed. Maintenance that reassigns row IDs invalidates the complete index generation and requires a rebuild. This build mode and its reader currently require the Java/Spark implementation -and a matching native library with centroid model APIs. Upgrade every component -that reads these indexes before enabling it. Older readers and Python index +and a matching native library with centroid model APIs. Install the vector plugin on readers +and on jobs that plan, commit or maintain these indexes. The plugin interprets manifest metadata without loading JNI. +For indexes built with the pre-release `shard=centroid-based` option, drop the old index with +that build before upgrading, then rebuild with the new type. Older readers and Python index builders do not support this layout; drop the centroid index before downgrading or rebuilding it through those clients. The model file and each populated shard carry training metadata, so measure model-loading cost and total index size alongside @@ -565,7 +568,7 @@ increasing `centroid.nprobe` alone cannot remove quantization error. For IVF-RQ, calibrate `rq.bits`; for DiskANN, calibrate both outer probes and local graph search width. A shared routing model does not guarantee equal recall. -Calibrate `ivf-pq.nlist` and `ivf-pq.pq.m` together with the query probe width. +Calibrate `ivf-pq-centroid.nlist` and `ivf-pq-centroid.pq.m` together with the query probe width. Increasing `nlist` creates more centroid files, each populated file carrying its own local model. Increasing `pq.m` uses more encoded bytes per vector; for the current 8-bit PQ codes, `pq.m=64` uses twice the code bytes of diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java index 33b7dadc045f..864b3c41aae5 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorGlobalIndexer.java @@ -18,12 +18,27 @@ package org.apache.paimon.globalindex; +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.utils.Range; + +import java.io.IOException; import java.io.Serializable; import java.util.Iterator; +import java.util.List; /** A vector indexer that can train and restore a router for independently built shards. */ public interface CentroidVectorGlobalIndexer extends VectorGlobalIndexer { + /** Persist the trained router and return the model entry with implementation-owned metadata. */ + ResultEntry writeRoutingModel( + GlobalIndexFileWriter io, + String name, + byte[] payload, + String modelId, + int nlist, + List coverage) + throws IOException; + /** Return non-null input ordinals in the order required for training. */ long[] planGlobalTrainingSample(long vectorCount); diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java new file mode 100644 index 000000000000..f6544376fecb --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex; + +import org.apache.paimon.utils.Range; + +import java.util.List; + +/** + * A centroid index groups all files for one field in a partition into a single generation. Its + * reader accepts and returns absolute table row IDs. Committers and scan planners can inspect the + * generation through this factory without opening files or initializing the native runtime. + */ +public interface CentroidVectorIndexFactory extends GlobalIndexerFactory { + /** Exact indexed row ranges for a model entry, or an empty list for a shard entry. */ + List sourceRowRanges(byte[] indexMeta); + + /** Validate that these entries form one complete generation and return its source coverage. */ + List validateGeneration(List indexMetas); +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexerFactoryUtils.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexerFactoryUtils.java index 5d8cfc213442..6581d8007221 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexerFactoryUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexerFactoryUtils.java @@ -24,6 +24,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,6 +75,18 @@ private static GlobalIndexerFactory find(String type) { return null; } + /** Look up the layout without constructing an indexer or interpreting its metadata. */ + @Nullable + public static CentroidVectorIndexFactory centroidFactory(String type) { + GlobalIndexerFactory factory = find(type); + if (factory == null && type.endsWith("-centroid")) { + throw new IllegalStateException("Missing centroid index plugin for type: " + type); + } + return factory instanceof CentroidVectorIndexFactory + ? (CentroidVectorIndexFactory) factory + : null; + } + public static GlobalIndexerFactory load(String type) { GlobalIndexerFactory globalIndexerFactory = find(type); if (globalIndexerFactory == null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java index b5fd84c87281..2dbe24d336f9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java @@ -18,14 +18,12 @@ package org.apache.paimon.globalindex; -import org.apache.paimon.CoreOptions; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.utils.Range; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -35,57 +33,29 @@ private CentroidVectorIndexUtils() {} public static boolean centroidBased(IndexFileMeta file) { return file.globalIndexMeta() != null - && VectorIndexShardMeta.isWrapped(file.globalIndexMeta().indexMeta()) - && CoreOptions.isVectorIndexType(file.indexType()); + && GlobalIndexerFactoryUtils.centroidFactory(file.indexType()) != null; } public static boolean isRoutingModel(IndexFileMeta file) { - return centroidBased(file) && metadata(file).routingModel(); + return centroidBased(file) && !coverage(file).isEmpty(); } - public static VectorIndexShardMeta metadata(IndexFileMeta file) { - return VectorIndexShardMeta.deserialize(file.globalIndexMeta().indexMeta()); - } - - public static IndexFileMeta validateModel(List files) { - IndexFileMeta routing = null; - String modelId = null; - int nlist = 0; - Set centroids = new HashSet<>(); + public static List validateModel(List files) { + String type = files.get(0).indexType(); + List metadata = new ArrayList<>(files.size()); for (IndexFileMeta file : files) { checkArgument( - centroidBased(file), - "Cannot mix centroid-based and range-based vector shards."); - VectorIndexShardMeta meta = metadata(file); - String id = meta.modelId(); - checkArgument( - modelId == null || modelId.equals(id), - "Centroid shards belong to different models."); - modelId = id; - if (meta.routingModel()) { - checkArgument( - routing == null, - "Multiple routing models for the same partition and index."); - routing = file; - nlist = meta.nlist(); - meta.sourceRowRanges(); - } else { - checkArgument(centroids.add(meta.centroid()), "Duplicate centroid shard."); - } + type.equals(file.indexType()), "Cannot mix different centroid index types."); + metadata.add(file.globalIndexMeta().indexMeta()); } - checkArgument(routing != null, "Missing centroid routing model."); - for (int centroid : centroids) { - checkArgument(centroid < nlist, "Invalid centroid shard."); - } - checkArgument(centroids.size() == nlist, "Incomplete centroid model shard set."); - return routing; + return GlobalIndexerFactoryUtils.centroidFactory(type).validateGeneration(metadata); } public static List coverage(IndexFileMeta file) { - if (!centroidBased(file)) { - return Collections.singletonList(file.globalIndexMeta().rowRange()); - } - VectorIndexShardMeta meta = metadata(file); - return meta.routingModel() ? meta.sourceRowRanges() : Collections.emptyList(); + CentroidVectorIndexFactory factory = + GlobalIndexerFactoryUtils.centroidFactory(file.indexType()); + return factory == null + ? Collections.singletonList(file.globalIndexMeta().rowRange()) + : factory.sourceRowRanges(file.globalIndexMeta().indexMeta()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java index dd0b00483af1..98d6bea138d1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java @@ -73,14 +73,39 @@ String write(@Nullable String previousIndexManifest, List ne Set indexes = new HashSet<>(); indexes.addAll(previous.keySet()); indexes.addAll(current.keySet()); + Map centroidTypes = new HashMap<>(); for (String indexName : indexes) { - indexEntries.addAll( + List combined = getIndexManifestFileCombine(indexName) .combine( previous.getOrDefault(indexName, Collections.emptyList()), - current.getOrDefault(indexName, Collections.emptyList()))); + current.getOrDefault(indexName, Collections.emptyList())); + if (!combined.isEmpty() + && CentroidVectorIndexUtils.centroidBased(combined.get(0).indexFile())) { + for (IndexManifestEntry entry : combined) { + centroidTypes.put( + entry.indexFile().globalIndexMeta().indexFieldId(), indexName); + } + } + indexEntries.addAll(combined); } + // Queries resolve a single vector index type for the column across partitions. + if (!centroidTypes.isEmpty()) { + for (IndexManifestEntry entry : indexEntries) { + IndexFileMeta file = entry.indexFile(); + if (file.globalIndexMeta() != null) { + int field = file.globalIndexMeta().indexFieldId(); + String type = centroidTypes.get(field); + checkState( + type == null || type.equals(file.indexType()), + "Cannot mix %s and %s on vector field %s. Drop the previous index before switching types.", + type, + file.indexType(), + field); + } + } + } return indexManifestFile.writeWithoutRolling(indexEntries); } @@ -236,28 +261,20 @@ public List combine( for (IndexManifestEntry entry : added) { indexEntries.put(entry.indexFile().fileName(), entry); } - if (indexEntries.values().stream() - .noneMatch( - entry -> CentroidVectorIndexUtils.centroidBased(entry.indexFile()))) { - return new ArrayList<>(indexEntries.values()); - } - Map>> models = new HashMap<>(); - for (IndexManifestEntry entry : indexEntries.values()) { - IndexFileMeta file = entry.indexFile(); - if (file.globalIndexMeta() == null) { - continue; + if (!indexEntries.isEmpty() + && CentroidVectorIndexUtils.centroidBased( + indexEntries.values().iterator().next().indexFile())) { + Map>> partitions = new HashMap<>(); + for (IndexManifestEntry entry : indexEntries.values()) { + IndexFileMeta file = entry.indexFile(); + partitions + .computeIfAbsent(entry.partition(), key -> new HashMap<>()) + .computeIfAbsent( + file.globalIndexMeta().indexFieldId(), key -> new ArrayList<>()) + .add(file); } - models.computeIfAbsent(entry.partition(), p -> new HashMap<>()) - .computeIfAbsent( - file.indexType() + ":" + file.globalIndexMeta().indexFieldId(), - f -> new ArrayList<>()) - .add(file); - } - for (Map> partition : models.values()) { - for (List files : partition.values()) { - if (files.stream().anyMatch(CentroidVectorIndexUtils::centroidBased)) { - CentroidVectorIndexUtils.validateModel(files); - } + for (Map> partition : partitions.values()) { + partition.values().forEach(CentroidVectorIndexUtils::validateModel); } } return new ArrayList<>(indexEntries.values()); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java index 7ce22a924241..9071e9b822d2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java @@ -646,12 +646,15 @@ private Optional checkCentroidSourceChanges( Snapshot latestSnapshot, List indexEntries) { for (IndexManifestEntry index : indexEntries) { if (index.kind() != FileKind.ADD - || !CentroidVectorIndexUtils.isRoutingModel(index.indexFile())) { + || !CentroidVectorIndexUtils.centroidBased(index.indexFile())) { + continue; + } + List coverage = CentroidVectorIndexUtils.coverage(index.indexFile()); + if (coverage.isEmpty()) { continue; } long baseline = DataEvolutionIndexSourceMeta.fromIndexFile(index.indexFile()).scanSnapshotId(); - List coverage = CentroidVectorIndexUtils.coverage(index.indexFile()); for (long id = baseline + 1; id <= latestSnapshot.id(); id++) { Snapshot snapshot = snapshotManager.snapshot(id); for (ManifestEntry change : diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index 585e7fecaf94..c2f079029794 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -186,8 +186,7 @@ public Plan scan() { if (files.stream().noneMatch(CentroidVectorIndexUtils::centroidBased)) { continue; } - IndexFileMeta routing = CentroidVectorIndexUtils.validateModel(files); - List coverage = CentroidVectorIndexUtils.metadata(routing).sourceRowRanges(); + List coverage = CentroidVectorIndexUtils.validateModel(files); long from = coverage.get(0).from; long to = coverage.get(coverage.size() - 1).to; List scalarFiles = scalarIndexFiles(allIndexFiles, coverage); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java index 4451d7997a62..484863b1795b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java @@ -19,15 +19,11 @@ package org.apache.paimon.index; import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; -import org.apache.paimon.globalindex.CentroidVectorIndexUtils; -import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; -import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Random; @@ -37,14 +33,11 @@ public class IndexFileMetaSerializerTest extends ObjectSerializerTestBase { @Test - void testRoutingModelKindRoundTrip() throws Exception { - byte[] descriptor = - VectorIndexShardMeta.routing( - "model", 1, Collections.singletonList(new Range(10, 1000))) - .serialize(); + void testOpaqueIndexMetadataRoundTrip() throws Exception { + byte[] descriptor = {3, 7, 1}; IndexFileMeta routing = new IndexFileMeta( - "ivf-pq", + "ivf-pq-centroid", "model.index", 100, 0, @@ -52,11 +45,6 @@ void testRoutingModelKindRoundTrip() throws Exception { null); IndexFileMetaSerializer serializer = new IndexFileMetaSerializer(); assertThat(serializer.fromRow(serializer.toRow(routing))).isEqualTo(routing); - assertThat( - CentroidVectorIndexUtils.isRoutingModel( - serializer.fromRow(serializer.toRow(routing)))) - .isTrue(); - org.apache.paimon.manifest.IndexManifestEntry entry = new org.apache.paimon.manifest.IndexManifestEntry( org.apache.paimon.manifest.FileKind.ADD, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java index 6dc79377362c..268d7113330d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java @@ -33,7 +33,6 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CloseableIterator; -import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -42,7 +41,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -199,57 +197,6 @@ public void testGlobalIndexOverlappingRangeRejectedWhenPreviousFileKept() throws .hasMessageContaining("overlapping row range"); } - @ParameterizedTest - @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) - void testCentroidModelsMustBeCompleteAndReplacedTogether(String type) { - List first = centroidModel("first", type); - IndexManifestFileHandler.GlobalIndexCombiner combiner = - new IndexManifestFileHandler.GlobalIndexCombiner(); - assertThat(combiner.combine(Collections.emptyList(), first)) - .containsExactlyInAnyOrderElementsOf(first); - assertThatThrownBy(() -> combiner.combine(Collections.emptyList(), first.subList(0, 2))) - .hasMessageContaining("Incomplete"); - assertThatThrownBy( - () -> - combiner.combine( - first, - Collections.singletonList(first.get(1).toDeleteEntry()))) - .hasMessageContaining("Incomplete"); - List replacement = centroidModel("second", type); - assertThatThrownBy(() -> combiner.combine(first, replacement)) - .hasMessageContaining("previous file"); - List changes = new ArrayList<>(); - first.forEach(entry -> changes.add(entry.toDeleteEntry())); - changes.addAll(replacement); - assertThat(combiner.combine(first, changes)) - .containsExactlyInAnyOrderElementsOf(replacement); - } - - private static List centroidModel(String id, String type) { - List entries = new ArrayList<>(); - for (int centroid = -1; centroid < 2; centroid++) { - byte[] metadata = - centroid == -1 - ? org.apache.paimon.globalindex.VectorIndexShardMeta.routing( - id, 2, Collections.singletonList(new Range(10, 100))) - .serialize() - : org.apache.paimon.globalindex.VectorIndexShardMeta.centroid( - id, centroid) - .serialize(); - IndexFileMeta file = - new IndexFileMeta( - type, - id + "-" + centroid, - 100, - centroid == -1 ? 0 : 10, - null, - null, - new GlobalIndexMeta(10, 100, 1, null, metadata)); - entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, file)); - } - return entries; - } - @Test public void testGlobalIndexOverlappingRangeAllowedAfterDelete() throws Exception { TestAppendFileStore fileStore = diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 459e0e5239b4..c1fa20de29fa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -24,6 +24,8 @@ import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; import org.apache.paimon.fs.Path; +import org.apache.paimon.globalindex.CentroidVectorIndexFactory; +import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileHandler; @@ -51,6 +53,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.MockedStatic; import javax.annotation.Nullable; @@ -76,6 +79,7 @@ import static org.apache.paimon.operation.commit.ConflictDetection.buildDeltaEntriesWithDV; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -1736,89 +1740,91 @@ void testCheckGlobalIndexRowIdExistenceNoConflict() { @Test void testCentroidIndexChecksSourceCoverageRatherThanSparseEnvelope() { - byte[] meta = - org.apache.paimon.globalindex.VectorIndexShardMeta.routing( - "model", - 2, - Arrays.asList( - new org.apache.paimon.utils.Range(10, 19), - new org.apache.paimon.utils.Range(100, 109))) - .serialize(); - IndexFileMeta file = - new IndexFileMeta( - "ivf-pq", - "routing", - 100, - 0, - null, - null, - new GlobalIndexMeta( - 10, - 109, - 1, - null, - meta, - new org.apache.paimon.index.DataEvolutionIndexSourceMeta(1) - .serialize())); - IndexManifestEntry index = new IndexManifestEntry(ADD, BinaryRow.EMPTY_ROW, 0, file); - List data = - Arrays.asList( - createFileEntryWithRowId("a", ADD, 10L, 10L), - createFileEntryWithRowId("b", ADD, 100L, 10L)); - assertThat( - createConflictDetection() - .checkConflicts( - snapshot(1), - data, - Collections.emptyList(), - Collections.singletonList(index), - null, - Snapshot.CommitKind.APPEND)) - .isNotPresent(); - assertThat( - createConflictDetection() - .checkConflicts( - snapshot(1), - data.subList(0, 1), - Collections.emptyList(), - Collections.singletonList(index), - null, - Snapshot.CommitKind.APPEND)) - .isPresent(); - - Snapshot concurrent = snapshot(2); - SnapshotManager snapshots = mock(SnapshotManager.class); - when(snapshots.snapshot(2)).thenReturn(concurrent); - CommitScanner scanner = mock(CommitScanner.class); - ManifestEntry updated = manifestEntry(ADD, "updated", 10L, new Range(10, 19)); - when(scanner.readIncrementalEntries( - concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) - .thenReturn(Collections.singletonList(updated)); - ConflictDetection detection = - createConflictDetection(scanner, true, false, false, snapshots); - assertThat( - detection.checkConflicts( - concurrent, - data, - Collections.emptyList(), - Collections.singletonList(index), - null, - Snapshot.CommitKind.APPEND)) - .hasValueSatisfying( - error -> assertThat(error).hasMessageContaining("source rows changed")); - ManifestEntry appended = manifestEntry(ADD, "appended", 200L, new Range(200, 209)); - when(scanner.readIncrementalEntries( - concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) - .thenReturn(Collections.singletonList(appended)); - assertThat( - detection.checkConflicts( - concurrent, - data, - Collections.emptyList(), - Collections.singletonList(index), - null, - Snapshot.CommitKind.APPEND)) - .isNotPresent(); + byte[] meta = {1}; + CentroidVectorIndexFactory factory = mock(CentroidVectorIndexFactory.class); + when(factory.sourceRowRanges(meta)) + .thenReturn(Arrays.asList(new Range(10, 19), new Range(100, 109))); + try (MockedStatic factories = + mockStatic(GlobalIndexerFactoryUtils.class)) { + factories + .when(() -> GlobalIndexerFactoryUtils.centroidFactory("test-centroid")) + .thenReturn(factory); + IndexFileMeta file = + new IndexFileMeta( + "test-centroid", + "routing", + 100, + 0, + null, + null, + new GlobalIndexMeta( + 10, + 109, + 1, + null, + meta, + new org.apache.paimon.index.DataEvolutionIndexSourceMeta(1) + .serialize())); + IndexManifestEntry index = new IndexManifestEntry(ADD, BinaryRow.EMPTY_ROW, 0, file); + List data = + Arrays.asList( + createFileEntryWithRowId("a", ADD, 10L, 10L), + createFileEntryWithRowId("b", ADD, 100L, 10L)); + assertThat( + createConflictDetection() + .checkConflicts( + snapshot(1), + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isNotPresent(); + assertThat( + createConflictDetection() + .checkConflicts( + snapshot(1), + data.subList(0, 1), + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isPresent(); + + Snapshot concurrent = snapshot(2); + SnapshotManager snapshots = mock(SnapshotManager.class); + when(snapshots.snapshot(2)).thenReturn(concurrent); + CommitScanner scanner = mock(CommitScanner.class); + ManifestEntry updated = manifestEntry(ADD, "updated", 10L, new Range(10, 19)); + when(scanner.readIncrementalEntries( + concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) + .thenReturn(Collections.singletonList(updated)); + ConflictDetection detection = + createConflictDetection(scanner, true, false, false, snapshots); + assertThat( + detection.checkConflicts( + concurrent, + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .hasValueSatisfying( + error -> assertThat(error).hasMessageContaining("source rows changed")); + ManifestEntry appended = manifestEntry(ADD, "appended", 200L, new Range(200, 209)); + when(scanner.readIncrementalEntries( + concurrent, Collections.singletonList(BinaryRow.EMPTY_ROW))) + .thenReturn(Collections.singletonList(appended)); + assertThat( + detection.checkConflicts( + concurrent, + data, + Collections.emptyList(), + Collections.singletonList(index), + null, + Snapshot.CommitKind.APPEND)) + .isNotPresent(); + } } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java index bb955b57822b..418ea005d3c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java @@ -18,12 +18,8 @@ package org.apache.paimon.table.sink; -import org.apache.paimon.globalindex.CentroidVectorIndexUtils; -import org.apache.paimon.index.DataEvolutionIndexSourceMeta; -import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; -import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; @@ -33,7 +29,6 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.Arrays; -import java.util.Base64; import static org.apache.paimon.index.IndexFileMetaSerializerTest.randomIndexFile; import static org.apache.paimon.manifest.ManifestCommittableSerializerTest.randomCompactIncrement; @@ -44,26 +39,6 @@ /** Test for {@link CommitMessageSerializer}. */ public class CommitMessageSerializerTest { - @Test - void readsPreviewRoutingCommit() throws IOException { - // Produced by the eight-field v15 writer. indexMeta contains its original JSON descriptor. - byte[] bytes = - Base64.getDecoder() - .decode( - "AAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAQAAMAAAAAAAAAgAAABIAAAAbW9k" - + "ZWwAAIUMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAAABQAAAADQAAAPAAAABpdmYtZmxh" - + "dAAIAAAAAAAADAAAAAAAAAATAAAAAAAAAAIAAAAAAAAAAAAAAAAAAABSAAAAOAAAABAAAACQAAAAeyJz" - + "aGFyZE1vZGUiOiJjZW50cm9pZC1iYXNlZCIsIm1vZGVsSWQiOiJnIiwibmxpc3QiOiIxIiwic291cmNl" - + "Um93UmFuZ2VzIjoiMTI6MTkifQAAAAAAAERFSVgAAAABAAAAAAAAAAVST1VUSU5HX01PREVMAAAAAAAA" - + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAF"); - CommitMessageImpl message = - (CommitMessageImpl) new CommitMessageSerializer().deserialize(15, bytes); - IndexFileMeta model = message.newFilesIncrement().newIndexFiles().get(0); - assertThat(CentroidVectorIndexUtils.isRoutingModel(model)).isTrue(); - assertThat(CentroidVectorIndexUtils.coverage(model)).containsExactly(new Range(12, 19)); - assertThat(DataEvolutionIndexSourceMeta.fromIndexFile(model).scanSnapshotId()).isEqualTo(5); - } - @Test public void test() throws IOException { CommitMessageSerializer serializer = new CommitMessageSerializer(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java index 563283f3a341..2faf4833041d 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/CentroidGlobalIndexTopoBuilder.java @@ -21,18 +21,14 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexBuilderUtils; import org.apache.paimon.globalindex.GlobalIndexFileReadWrite; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.IndexedSplit; -import org.apache.paimon.globalindex.ResultEntry; -import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.VectorIndexUtils; import org.apache.paimon.globalindex.VectorTrainingModel; -import org.apache.paimon.globalindex.VectorTrainingModelIO; import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.CompactIncrement; @@ -190,9 +186,6 @@ public List buildIndex( GlobalIndexFileReadWrite io = context.io(); String modelFile = "vector-" + indexType + "-global-index-training-model-" + modelId + ".index"; - try (PositionOutputStream out = io.newOutputStream(modelFile)) { - VectorTrainingModelIO.write(out, payload, modelId); - } IndexFileMeta modelMeta = GlobalIndexBuilderUtils.toIndexFileMetas( table.fileIO(), @@ -202,12 +195,9 @@ public List buildIndex( Collections.singletonList(indexField), indexType, Collections.singletonList( - new ResultEntry( - modelFile, - 0, - VectorIndexShardMeta.routing( - modelId, nlist, ranges) - .serialize())), + indexer.writeRoutingModel( + io, modelFile, payload, modelId, nlist, + ranges)), context.sourceMeta()) .get(0); messages.add(context.message(Collections.singletonList(modelMeta))); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java index 6d500d4cf57f..7e5a401df302 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilder.java @@ -118,13 +118,6 @@ public List buildIndex( List rowRangesToBuild = new ArrayList<>( GlobalIndexBuilderUtils.unindexedRowRanges(snapshot, currentIndexes)); - checkArgument( - currentIndexes.stream() - .noneMatch( - entry -> - org.apache.paimon.globalindex.CentroidVectorIndexUtils - .centroidBased(entry.indexFile())), - "Cannot build range shards alongside a centroid model; drop the existing index or perform a full centroid rebuild."); byte[] sourceMeta = new DataEvolutionIndexSourceMeta(snapshot.id()).serialize(); boolean detectDataFileChange = new Options(table.options(), options.toMap()).get(GLOBAL_INDEX_COLUMN_UPDATE_ACTION) diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java index aca61d8e50df..3a893bea1295 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/GlobalIndexTopologyBuilderUtils.java @@ -18,51 +18,19 @@ package org.apache.paimon.spark.globalindex; -import org.apache.paimon.CoreOptions; -import org.apache.paimon.options.Options; +import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.spark.globalindex.sorted.SortedIndexTopoBuilder; -import java.util.Map; - /** Utility class for creating {@link GlobalIndexTopologyBuilder} implementations. */ public class GlobalIndexTopologyBuilderUtils { public static GlobalIndexTopologyBuilder createTopoBuilder(String indexType) { + if (GlobalIndexerFactoryUtils.centroidFactory(indexType) != null) { + return new CentroidGlobalIndexTopoBuilder(); + } if (SortedIndexTopoBuilder.supports(indexType)) { return new SortedIndexTopoBuilder(); } return new DefaultGlobalIndexTopoBuilder(); } - - public static GlobalIndexTopologyBuilder createTopoBuilder(String indexType, Options options) { - return createTopoBuilder(indexType, options, null); - } - - public static GlobalIndexTopologyBuilder createTopoBuilder( - String indexType, Options options, String fieldName) { - String key = indexType + ".shard"; - String fieldKey = "fields." + fieldName + ".shard"; - Map dynamic = options.dynamicOptions(); - Map stored = options.toMap(); - String shard = - dynamic.containsKey(fieldKey) - ? dynamic.get(fieldKey) - : dynamic.containsKey(key) - ? dynamic.get(key) - : stored.getOrDefault( - fieldKey, stored.getOrDefault(key, "range-based")); - if (CoreOptions.isVectorIndexType(indexType)) { - if ("centroid-based".equals(shard)) { - return new CentroidGlobalIndexTopoBuilder(); - } - if (!"range-based".equals(shard)) { - throw new IllegalArgumentException("Unknown vector shard mode: " + shard); - } - } - if ("centroid-based".equals(shard)) { - throw new IllegalArgumentException( - "Centroid sharding requires a supported vector index."); - } - return createTopoBuilder(indexType); - } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java index 6858a312e3db..ed349c65340e 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedure.java @@ -186,8 +186,7 @@ public InternalRow[] call(InternalRow args) { } GlobalIndexTopologyBuilder topoBuilder = - GlobalIndexTopologyBuilderUtils.createTopoBuilder( - indexType, userOptions, indexFields.get(0).name()); + GlobalIndexTopologyBuilderUtils.createTopoBuilder(indexType); List indexResults = topoBuilder.buildIndex( diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java index fa12c11d62dc..17ddeadbc3c8 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/globalindex/DefaultGlobalIndexTopoBuilderTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -62,33 +61,9 @@ void testFMIndexUsesDefaultTopologyBuilder() { } @Test - void testCentroidTopologyRespectsFieldAndExplicitOptionPrecedence() { - Map stored = new HashMap<>(); - stored.put("ivf-pq.shard", "range-based"); - stored.put("fields.v.shard", "centroid-based"); - assertThat( - GlobalIndexTopologyBuilderUtils.createTopoBuilder( - "ivf-pq", new Options(stored), "v")) - .isInstanceOf(CentroidGlobalIndexTopoBuilder.class); - Options overridden = - new Options(stored, Collections.singletonMap("ivf-pq.shard", "range-based")); - assertThat(GlobalIndexTopologyBuilderUtils.createTopoBuilder("ivf-pq", overridden, "v")) - .isInstanceOf(DefaultGlobalIndexTopoBuilder.class); - assertThat( - GlobalIndexTopologyBuilderUtils.createTopoBuilder( - "ivf-pq", new Options(stored), "other")) + void ordinaryVectorIndexUsesDefaultTopology() { + assertThat(GlobalIndexTopologyBuilderUtils.createTopoBuilder("test-vector-ann")) .isInstanceOf(DefaultGlobalIndexTopoBuilder.class); - for (String type : Arrays.asList("ivf-flat", "ivf-sq", "ivf-rq", "diskann")) { - assertThat( - GlobalIndexTopologyBuilderUtils.createTopoBuilder( - type, new Options(stored), "v")) - .isInstanceOf(CentroidGlobalIndexTopoBuilder.class); - } - assertThatThrownBy( - () -> - GlobalIndexTopologyBuilderUtils.createTopoBuilder( - "btree", new Options(stored), "v")) - .hasMessageContaining("supported vector index"); } @Test diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java index 22d37d565dd9..e3837a59b612 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/benchmark/VectorShardingBenchmark.java @@ -29,11 +29,13 @@ import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.OffsetGlobalIndexReader; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.options.Options; import org.apache.paimon.predicate.VectorSearch; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.spark.SparkCatalog; @@ -46,9 +48,9 @@ import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.FloatType; import org.apache.paimon.types.RowType; -import org.apache.paimon.vector.index.NativeVectorGlobalIndexer; import org.apache.spark.sql.SparkSession; @@ -299,13 +301,9 @@ private void run() throws Exception { spark.sql( "CREATE TABLE vectors (id INT, v ARRAY) TBLPROPERTIES (" + "'bucket'='-1','row-tracking.enabled'='true','data-evolution.enabled'='true'," - + "'" - + indexType - + ".dimension'='" + + "'fields.v.dimension'='" + dimension - + "','" - + indexType - + ".metric'='l2'," + + "','fields.v.metric'='l2'," + "'global-index.row-count-per-shard'='" + rangeRows + "'," @@ -324,13 +322,9 @@ private void run() throws Exception { for (String mode : modes) { // Every family can reuse the immutable source table. spark.sql( - "ALTER TABLE vectors SET TBLPROPERTIES ('" - + indexType - + ".dimension'='" + "ALTER TABLE vectors SET TBLPROPERTIES ('fields.v.dimension'='" + dimension - + "','" - + indexType - + ".metric'='l2')") + + "','fields.v.metric'='l2')") .collect(); long elapsed = -1; if (!skipBuild) { @@ -339,7 +333,16 @@ private void run() throws Exception { options.forEach((key, value) -> pairs.add(key + "=" + value)); for (String type : Arrays.asList( - "ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann")) { + "ivf-flat", + "ivf-pq", + "ivf-sq", + "ivf-rq", + "diskann", + "ivf-flat-centroid", + "ivf-pq-centroid", + "ivf-sq-centroid", + "ivf-rq-centroid", + "diskann-centroid")) { spark.sql( "CALL sys.drop_global_index(table=>'bench.vectors'," + "index_column=>'v',index_type=>'" @@ -351,7 +354,7 @@ private void run() throws Exception { spark.sql( "CALL sys.create_global_index(table=>'bench.vectors'," + "index_column=>'v',index_type=>'" - + indexType + + typeForMode(mode) + "',options=>'" + String.join(",", pairs) + "')") @@ -366,7 +369,7 @@ private void run() throws Exception { .snapshot(Long.parseLong(readSnapshot)); List files = new ArrayList<>(); for (IndexManifestEntry entry : - table.store().newIndexFileHandler().scan(snapshot, indexType)) { + table.store().newIndexFileHandler().scan(snapshot, typeForMode(mode))) { files.add(entry.indexFile()); } if (files.isEmpty()) { @@ -388,7 +391,7 @@ private void run() throws Exception { files.size(), bytes, snapshot.id(), - indexType); + typeForMode(mode)); builds.flush(); System.out.printf( "BUILD repeat=%d mode=%s seconds=%.3f files=%d MiB=%.2f%n", @@ -476,7 +479,7 @@ private void run() throws Exception { reuseReaders ? -1 : stats.bytes.sum(), reuseReaders ? -1 : stats.routingBytes.sum(), lSearch, - indexType); + typeForMode(mode)); } } measurements.flush(); @@ -506,30 +509,33 @@ private FileStoreTable table() { private Map buildOptions(String mode) { Map options = new LinkedHashMap<>(); - options.put(indexType + ".shard", mode); + String type = typeForMode(mode); if (!"diskann".equals(indexType) || "centroid-based".equals(mode)) { - options.put(indexType + ".nlist", String.valueOf(nlist)); + options.put(type + ".nlist", String.valueOf(nlist)); } if ("ivf-pq".equals(indexType) || "diskann".equals(indexType)) { - options.put(indexType + ".pq.m", String.valueOf(dimension / 4)); + options.put(type + ".pq.m", String.valueOf(dimension / 4)); } - options.put(indexType + ".train.sample-ratio", "1.0"); + options.put(type + ".train.sample-ratio", "1.0"); String extra = System.getProperty("benchmark." + mode + ".options", ""); if (!extra.isEmpty()) { for (String pair : extra.split(",")) { String[] entry = pair.split("=", 2); if (entry.length != 2 || !entry[0].startsWith(indexType + ".") - || entry[1].isEmpty() - || (indexType + ".shard").equals(entry[0])) { + || entry[1].isEmpty()) { throw new IllegalArgumentException("Invalid benchmark option: " + pair); } - options.put(entry[0], entry[1]); + options.put(type + entry[0].substring(indexType.length()), entry[1]); } } return options; } + private String typeForMode(String mode) { + return "centroid-based".equals(mode) ? indexType + "-centroid" : indexType; + } + private List probesForMode(String mode) { if ("diskann".equals(indexType) && "range-based".equals(mode)) { return new ArrayList<>(Collections.singletonList(0)); @@ -539,7 +545,7 @@ private List probesForMode(String mode) { return new ArrayList<>(probes); } List widths = new ArrayList<>(); - int modeNlist = Integer.parseInt(buildOptions(mode).get(indexType + ".nlist")); + int modeNlist = Integer.parseInt(buildOptions(mode).get(typeForMode(mode) + ".nlist")); for (String value : override.split(",")) { int width = Integer.parseInt(value); if (width < 1 || width > modeNlist || widths.contains(width)) { @@ -681,9 +687,12 @@ private List openReaders( List metas, GlobalIndexFileReader input) throws IOException { - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer( - new ArrayType(new FloatType()), Collections.emptyMap(), indexType); + GlobalIndexer indexer = + GlobalIndexer.create( + typeForMode(mode), + Collections.singletonList( + new DataField(0, "v", new ArrayType(new FloatType()))), + new Options()); List readers = new ArrayList<>(); try { if ("centroid-based".equals(mode)) { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala index dc910b23c9a2..c34e873b6c3a 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala @@ -18,10 +18,10 @@ package org.apache.paimon.spark.sql -import org.apache.paimon.globalindex.{VectorIndexShardMeta, VectorTrainingModelIO} import org.apache.paimon.globalindex.CentroidVectorIndexUtils import org.apache.paimon.index.IndexFileMeta import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.vector.index.{VectorIndexShardMeta, VectorTrainingModelIO} import scala.collection.JavaConverters._ @@ -31,9 +31,9 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { private def build(): Unit = { spark .sql("""CALL sys.create_global_index( - |table => 'test.T', index_column => 'v', index_type => 'ivf-pq', + |table => 'test.T', index_column => 'v', index_type => 'ivf-pq-centroid', |partitions => 'p=0', - |options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=4,ivf-pq.pq.m=1')""".stripMargin) + |options => 'ivf-pq-centroid.nlist=4,ivf-pq-centroid.pq.m=1')""".stripMargin) .collect() } @@ -56,8 +56,8 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { |'data-evolution.enabled' = 'true', |'deletion-vectors.enabled' = 'true', |'vector-index.search-mode' = 'detail', - |'ivf-pq.dimension' = '2', - |'ivf-pq.metric' = 'l2', + |'ivf-pq-centroid.dimension' = '2', + |'ivf-pq-centroid.metric' = 'l2', |'global-index.row-count-per-shard' = '64', |'global-index.build.max-parallelism' = '2')""".stripMargin) def insert(from: Int, to: Int, partition: Int): Unit = { @@ -75,7 +75,7 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { table .store() .newIndexFileHandler() - .scan(table.snapshotManager().latestSnapshot(), "ivf-pq") + .scan(table.snapshotManager().latestSnapshot(), "ivf-pq-centroid") .asScala .map(_.indexFile()) .toSeq @@ -95,10 +95,9 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { intercept[Exception] { spark - .sql( - """CALL sys.create_global_index(table => 'test.T', index_column => 'v', - |index_type => 'ivf-pq', - |options => 'ivf-pq.shard=centroid-based,ivf-pq.nlist=4,ivf-pq.pq.m=1')""".stripMargin) + .sql("""CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => 'ivf-pq-centroid', + |options => 'ivf-pq-centroid.nlist=4,ivf-pq-centroid.pq.m=1')""".stripMargin) .collect() } assert(indexes.size == 5) @@ -126,7 +125,7 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { spark.sql("""CREATE TABLE T (id INT, v ARRAY) TBLPROPERTIES ( |'bucket'='-1', 'row-tracking.enabled'='true', 'data-evolution.enabled'='true', |'deletion-vectors.enabled'='true', 'vector-index.search-mode'='fast', - |'ivf-flat.dimension'='2', 'ivf-flat.metric'='l2', + |'ivf-flat-centroid.dimension'='2', 'ivf-flat-centroid.metric'='l2', |'global-index.row-count-per-shard'='32', |'global-index.build.max-parallelism'='4')""".stripMargin) spark.sql("""INSERT INTO T SELECT cast(id AS INT), @@ -136,13 +135,13 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { def buildModel(): Array[Byte] = { spark.sql( """CALL sys.create_global_index(table => 'test.T', index_column => 'v', - |index_type => 'ivf-flat', - |options => 'ivf-flat.shard=centroid-based,ivf-flat.nlist=2,ivf-flat.train.sample-ratio=0.125')""".stripMargin) + |index_type => 'ivf-flat-centroid', + |options => 'ivf-flat-centroid.nlist=2,ivf-flat-centroid.train.sample-ratio=0.125')""".stripMargin) val table = loadTable("T") val model = table .store() .newIndexFileHandler() - .scan(table.snapshotManager().latestSnapshot(), "ivf-flat") + .scan(table.snapshotManager().latestSnapshot(), "ivf-flat-centroid") .asScala .map(_.indexFile()) .find(f => CentroidVectorIndexUtils.isRoutingModel(f)) @@ -176,20 +175,22 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { spark.sql(s"""CREATE TABLE T (id INT, v ARRAY) |TBLPROPERTIES ('bucket'='-1', 'row-tracking.enabled'='true', |'data-evolution.enabled'='true', 'deletion-vectors.enabled'='true', - |'vector-index.search-mode'='fast', '$kind.dimension'='4', - |'$kind.metric'='l2', 'global-index.row-count-per-shard'='64', + |'vector-index.search-mode'='fast', 'fields.v.dimension'='4', + |'fields.v.metric'='l2', 'global-index.row-count-per-shard'='64', |'global-index.build.max-parallelism'='2')""".stripMargin) spark.sql( """INSERT INTO T SELECT cast(id AS INT), |array(cast(id AS FLOAT), cast(id % 17 AS FLOAT), |cast(id % 23 AS FLOAT), cast(id % 31 AS FLOAT)) FROM range(0, 256)""".stripMargin) - val pq = if (kind == "ivf-pq" || kind == "diskann") s",$kind.pq.m=1" else "" - def create(mode: String): Unit = + val centroidType = s"$kind-centroid" + def create(indexType: String): Unit = { + val pq = if (kind == "ivf-pq" || kind == "diskann") s",$indexType.pq.m=1" else "" spark - .sql(s"""CALL sys.create_global_index(table => 'test.T', index_column => 'v', - |index_type => '$kind', - |options => '$kind.shard=$mode,$kind.nlist=4$pq')""".stripMargin) + .sql( + s"""CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => '$indexType', options => '$indexType.nlist=4$pq')""".stripMargin) .collect() + } def nearest(mode: String): Array[Int] = { val opts = if (kind == "diskann" && mode == "centroid-based") @@ -203,12 +204,12 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { .collect() .map(_.getInt(0)) } - create("centroid-based") + create(centroidType) val table = loadTable("T") val files = table .store() .newIndexFileHandler() - .scan(table.snapshotManager().latestSnapshot(), kind) + .scan(table.snapshotManager().latestSnapshot(), centroidType) .asScala .map(_.indexFile()) assert( @@ -220,9 +221,9 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { assert(centroids.toSet == Set(0, 1, 2, 3)) spark .sql(s"""CALL sys.drop_global_index(table => 'test.T', index_column => 'v', - |index_type => '$kind')""".stripMargin) + |index_type => '$centroidType')""".stripMargin) .collect() - create("range-based") + create(kind) assert(nearest("range-based").toSeq == Seq(200)) } } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java index 48d291844d65..ee284fbf9750 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java @@ -22,8 +22,6 @@ import org.apache.paimon.globalindex.AbstractVectorGlobalIndexReader; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; -import org.apache.paimon.globalindex.VectorIndexShardMeta; -import org.apache.paimon.globalindex.VectorTrainingModelIO; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.index.vector.CentroidRouter; import org.apache.paimon.predicate.BatchVectorSearch; @@ -72,9 +70,6 @@ final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexRea String modelId = null; long count = 0; for (GlobalIndexIOMeta file : entries) { - checkArgument( - VectorIndexShardMeta.isWrapped(file.metadata()), - "Cannot mix centroid-based and range-based shards."); VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(file.metadata()); String id = meta.modelId(); checkArgument(modelId == null || modelId.equals(id), "Mismatched centroid model."); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java new file mode 100644 index 000000000000..1578c651ae08 --- /dev/null +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexer.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.GlobalIndexReader; +import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; +import org.apache.paimon.globalindex.GlobalIndexWriter; +import org.apache.paimon.globalindex.ResultEntry; +import org.apache.paimon.globalindex.VectorGlobalModelTrainer; +import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.globalindex.io.GlobalIndexFileReader; +import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; +import org.apache.paimon.index.vector.CentroidRouter; +import org.apache.paimon.index.vector.VectorIndexTrainer; +import org.apache.paimon.index.vector.VectorIndexTraining; +import org.apache.paimon.types.DataType; +import org.apache.paimon.utils.Range; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +/** Outer centroid routing with an independently trained native index in each populated shard. */ +final class NativeCentroidVectorGlobalIndexer extends NativeVectorGlobalIndexer + implements CentroidVectorGlobalIndexer { + private final String localNlist; + + NativeCentroidVectorGlobalIndexer( + DataType fieldType, + Map options, + String identifier, + double trainSampleRatio) { + super(fieldType, options, identifier, trainSampleRatio); + this.localNlist = this.options.get("centroid.local-nlist"); + if (localNlist != null + && ("diskann".equals(identifier) + || (!"auto".equals(localNlist) && Integer.parseInt(localNlist) <= 0))) { + throw new IllegalArgumentException( + "centroid.local-nlist requires local IVF and a positive count or auto."); + } + } + + @Override + public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) { + throw new UnsupportedOperationException( + "Centroid indexes require a routed, distributed build."); + } + + @Override + public ResultEntry writeRoutingModel( + GlobalIndexFileWriter io, + String name, + byte[] payload, + String modelId, + int nlist, + List coverage) + throws IOException { + try (OutputStream out = io.newOutputStream(name)) { + VectorTrainingModelIO.write(out, payload, modelId); + } + return new ResultEntry( + name, 0, VectorIndexShardMeta.routing(modelId, nlist, coverage).serialize()); + } + + @Override + public GlobalIndexReader createReader( + GlobalIndexFileReader fileReader, + List files, + long totalRowCount, + List rowRanges, + ExecutorService executor) { + return new NativeCentroidRoutedIndexReader( + fileReader, files, fieldType, executor, identifier); + } + + @Override + public long[] planGlobalTrainingSample(long vectorCount) { + long candidates = + NativeVectorGlobalIndexWriter.trainingCandidateCount(vectorCount, trainSampleRatio); + if (candidates == 0) { + throw new IllegalArgumentException( + "Cannot train a centroid model without non-null vectors."); + } + try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { + long[] positions = trainer.planTrainingSample(candidates); + for (int i = 0; i < positions.length; i++) { + positions[i] = + NativeVectorGlobalIndexWriter.sampleIndex( + positions[i], vectorCount, candidates); + } + return positions; + } + } + + VectorIndexTrainer createCoarseTrainer(long vectorCount) { + Map coarse = new LinkedHashMap<>(); + coarse.put("index.type", "ivf_flat"); + coarse.put("dimension", options.get("dimension")); + coarse.put("metric", options.get("metric")); + coarse.put("ivf.coarse-assignment", "exact"); + String nlist = options.getOrDefault("nlist", "auto"); + if ("diskann".equals(identifier) && "auto".equals(nlist.trim())) { + nlist = "64"; + } + coarse.put("nlist", nlist); + coarse.put( + "expected-vector-count", + options.getOrDefault("expected-vector-count", Long.toString(vectorCount))); + if (options.containsKey("ivf.train.max-points-per-centroid")) { + coarse.put( + "ivf.train.max-points-per-centroid", + options.get("ivf.train.max-points-per-centroid")); + } + return VectorIndexTrainer.create(coarse); + } + + GlobalIndexSingleColumnWriter createLocalWriter( + GlobalIndexFileWriter fileWriter, int centroid, String modelId) { + Map local = new LinkedHashMap<>(options); + if ("diskann".equals(identifier)) { + local.remove("nlist"); + } else { + local.put("nlist", localNlist == null ? "auto" : localNlist); + } + local.remove("expected-vector-count"); + return new NativeVectorGlobalIndexWriter( + fileWriter, + fieldType, + local, + identifier, + trainSampleRatio, + VectorIndexShardMeta.centroid(modelId, centroid)); + } + + @Override + public VectorTrainingModel trainGlobalModel(long vectorCount, Iterator batches) { + try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { + while (batches.hasNext()) { + TrainingBatch batch = batches.next(); + trainer.addTrainingVectors(batch.vectors, batch.count); + } + try (VectorIndexTraining coarse = trainer.finishTraining()) { + return wrap(CentroidRouter.fromTraining(coarse)); + } + } + } + + @Override + public VectorGlobalModelTrainer createGlobalModelTrainer() { + return new NativeVectorGlobalModelTrainer( + fieldType, options, trainSampleRatio, identifier, this); + } + + @Override + public VectorTrainingModel restoreTrainingModel(byte[] payload) { + return wrap(CentroidRouter.deserialize(payload)); + } + + NativeVectorTrainingModel wrap(CentroidRouter model) { + return new NativeVectorTrainingModel(model, this); + } +} diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java new file mode 100644 index 000000000000..853d03e75891 --- /dev/null +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.vector.index; + +import org.apache.paimon.globalindex.CentroidVectorIndexFactory; +import org.apache.paimon.types.DataType; +import org.apache.paimon.utils.Range; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Registered centroid index types share the router and select a local native algorithm. */ +public abstract class NativeCentroidVectorGlobalIndexerFactory + extends NativeVectorGlobalIndexerFactory implements CentroidVectorIndexFactory { + private final String algorithm; + + NativeCentroidVectorGlobalIndexerFactory(String algorithm) { + this.algorithm = algorithm; + } + + @Override + public String identifier() { + return algorithm + "-centroid"; + } + + @Override + String nativeAlgorithm() { + return algorithm; + } + + @Override + NativeVectorGlobalIndexer createIndexer( + DataType fieldType, Map options, String algorithm, double ratio) { + return new NativeCentroidVectorGlobalIndexer(fieldType, options, algorithm, ratio); + } + + @Override + public List sourceRowRanges(byte[] indexMeta) { + VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(indexMeta); + return meta.routingModel() ? meta.sourceRowRanges() : Collections.emptyList(); + } + + @Override + public List validateGeneration(List indexMetas) { + List coverage = null; + String modelId = null; + int nlist = 0; + Set centroids = new HashSet<>(); + for (byte[] bytes : indexMetas) { + VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(bytes); + String id = meta.modelId(); + checkArgument( + modelId == null || modelId.equals(id), + "Centroid shards belong to different models."); + modelId = id; + if (meta.routingModel()) { + checkArgument( + coverage == null, + "Multiple routing models for the same partition and index."); + coverage = meta.sourceRowRanges(); + nlist = meta.nlist(); + } else { + checkArgument(centroids.add(meta.centroid()), "Duplicate centroid shard."); + } + } + checkArgument(coverage != null, "Missing centroid routing model."); + for (int centroid : centroids) { + checkArgument(centroid < nlist, "Invalid centroid shard."); + } + checkArgument(centroids.size() == nlist, "Incomplete centroid model shard set."); + return coverage; + } + + /** Centroid routing over IVF-FLAT shards. */ + public static final class IvfFlat extends NativeCentroidVectorGlobalIndexerFactory { + public IvfFlat() { + super(IvfFlatVectorGlobalIndexerFactory.IDENTIFIER); + } + } + + /** Centroid routing over IVF-PQ shards. */ + public static final class IvfPq extends NativeCentroidVectorGlobalIndexerFactory { + public IvfPq() { + super(IvfPqAlgorithmVectorGlobalIndexerFactory.IDENTIFIER); + } + } + + /** Centroid routing over IVF-SQ shards. */ + public static final class IvfSq extends NativeCentroidVectorGlobalIndexerFactory { + public IvfSq() { + super(IvfSqVectorGlobalIndexerFactory.IDENTIFIER); + } + } + + /** Centroid routing over IVF-RQ shards. */ + public static final class IvfRq extends NativeCentroidVectorGlobalIndexerFactory { + public IvfRq() { + super(IvfRqVectorGlobalIndexerFactory.IDENTIFIER); + } + } + + /** Centroid routing over DiskANN shards. */ + public static final class DiskAnn extends NativeCentroidVectorGlobalIndexerFactory { + public DiskAnn() { + super(DiskAnnVectorGlobalIndexerFactory.IDENTIFIER); + } + } +} diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java index b81a3be7d0e5..a83e53cddbf9 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java @@ -24,7 +24,6 @@ import org.apache.paimon.fs.PositionOutputStreamWrapper; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.ResultEntry; -import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; import org.apache.paimon.index.vector.CentroidRouter; import org.apache.paimon.index.vector.VectorIndexTrainer; @@ -425,7 +424,8 @@ private void addTrainingBatches(VectorIndexTrainer trainer, long trainCount) } } - CentroidRouter finishRoutingModel(NativeVectorGlobalIndexer indexer) throws IOException { + CentroidRouter finishRoutingModel(NativeCentroidVectorGlobalIndexer indexer) + throws IOException { if (count == 0) { throw new IllegalArgumentException( "Cannot train a centroid model without non-null vectors."); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java index ac302af4f397..16a54dce76a9 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java @@ -18,39 +18,29 @@ package org.apache.paimon.vector.index; -import org.apache.paimon.globalindex.CentroidVectorGlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexReader; -import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.GlobalIndexWriter; -import org.apache.paimon.globalindex.VectorGlobalModelTrainer; -import org.apache.paimon.globalindex.VectorIndexShardMeta; -import org.apache.paimon.globalindex.VectorTrainingModel; +import org.apache.paimon.globalindex.VectorGlobalIndexer; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; -import org.apache.paimon.index.vector.CentroidRouter; -import org.apache.paimon.index.vector.VectorIndexTrainer; -import org.apache.paimon.index.vector.VectorIndexTraining; import org.apache.paimon.types.DataType; import org.apache.paimon.utils.Range; -import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutorService; /** Native vector global indexer backed by paimon-vector-index-java. */ -public class NativeVectorGlobalIndexer implements CentroidVectorGlobalIndexer { +public class NativeVectorGlobalIndexer implements VectorGlobalIndexer { static final String DEFAULT_METRIC = "inner_product"; - private final DataType fieldType; - private final Map options; - private final String identifier; - private final String localNlist; - private final double trainSampleRatio; + final DataType fieldType; + final Map options; + final String identifier; + final double trainSampleRatio; public NativeVectorGlobalIndexer( DataType fieldType, Map options, String identifier) { @@ -69,13 +59,6 @@ public NativeVectorGlobalIndexer( this.fieldType = fieldType; this.options = Objects.requireNonNull(options, "options must not be null"); this.identifier = Objects.requireNonNull(identifier, "identifier must not be null"); - this.localNlist = this.options.get("centroid.local-nlist"); - if (localNlist != null - && ("diskann".equals(identifier) - || (!"auto".equals(localNlist) && Integer.parseInt(localNlist) <= 0))) { - throw new IllegalArgumentException( - "centroid.local-nlist requires local IVF and a positive count or auto."); - } if (Double.isNaN(trainSampleRatio) || Double.isInfinite(trainSampleRatio) || trainSampleRatio <= 0 @@ -100,101 +83,10 @@ public GlobalIndexReader createReader( long totalRowCount, List rowRanges, ExecutorService executor) { - if (files.stream().anyMatch(file -> VectorIndexShardMeta.isWrapped(file.metadata()))) { - return new NativeCentroidRoutedIndexReader( - fileReader, files, fieldType, executor, identifier); - } return new NativeVectorGlobalIndexReader( fileReader, files, fieldType, executor, identifier); } - @Override - public long[] planGlobalTrainingSample(long vectorCount) { - long candidates = - NativeVectorGlobalIndexWriter.trainingCandidateCount(vectorCount, trainSampleRatio); - if (candidates == 0) { - throw new IllegalArgumentException( - "Cannot train a centroid model without non-null vectors."); - } - try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { - long[] positions = trainer.planTrainingSample(candidates); - for (int i = 0; i < positions.length; i++) { - positions[i] = - NativeVectorGlobalIndexWriter.sampleIndex( - positions[i], vectorCount, candidates); - } - return positions; - } - } - - VectorIndexTrainer createCoarseTrainer(long vectorCount) { - Map coarse = new LinkedHashMap<>(); - coarse.put("index.type", "ivf_flat"); - coarse.put("dimension", options.get("dimension")); - coarse.put("metric", options.get("metric")); - coarse.put("ivf.coarse-assignment", "exact"); - String nlist = options.getOrDefault("nlist", "auto"); - if ("diskann".equals(identifier) && "auto".equals(nlist.trim())) { - nlist = "64"; - } - coarse.put("nlist", nlist); - coarse.put( - "expected-vector-count", - options.getOrDefault("expected-vector-count", Long.toString(vectorCount))); - if (options.containsKey("ivf.train.max-points-per-centroid")) { - coarse.put( - "ivf.train.max-points-per-centroid", - options.get("ivf.train.max-points-per-centroid")); - } - return VectorIndexTrainer.create(coarse); - } - - GlobalIndexSingleColumnWriter createLocalWriter( - GlobalIndexFileWriter fileWriter, int centroid, String modelId) { - Map local = new LinkedHashMap<>(options); - if ("diskann".equals(identifier)) { - local.remove("nlist"); - } else { - local.put("nlist", localNlist == null ? "auto" : localNlist); - } - local.remove("expected-vector-count"); - return new NativeVectorGlobalIndexWriter( - fileWriter, - fieldType, - local, - identifier, - trainSampleRatio, - VectorIndexShardMeta.centroid(modelId, centroid)); - } - - @Override - public VectorTrainingModel trainGlobalModel(long vectorCount, Iterator batches) { - try (VectorIndexTrainer trainer = createCoarseTrainer(vectorCount)) { - while (batches.hasNext()) { - TrainingBatch batch = batches.next(); - trainer.addTrainingVectors(batch.vectors, batch.count); - } - try (VectorIndexTraining coarse = trainer.finishTraining()) { - return wrap(CentroidRouter.fromTraining(coarse)); - } - } - } - - @Override - public VectorGlobalModelTrainer createGlobalModelTrainer() { - return new NativeVectorGlobalModelTrainer( - fieldType, options, trainSampleRatio, identifier, this); - } - - @Override - public VectorTrainingModel restoreTrainingModel(byte[] payload) { - return wrap(CentroidRouter.deserialize(payload)); - } - - NativeVectorTrainingModel wrap(CentroidRouter model) { - return new NativeVectorTrainingModel(model, this); - } - @Override public String metric() { return options.getOrDefault("metric", DEFAULT_METRIC); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java index 82e4af55a420..5eb6de97bbd5 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java @@ -44,21 +44,51 @@ public GlobalIndexer create(List indexFields, Options options) { } DataField field = indexFields.get(0); String identifier = identifier(); - return new NativeVectorGlobalIndexer( - field.type(), - nativeOptions(field.type(), options, identifier, field.name()), - identifier, - trainSampleRatio(options, identifier, field.name())); + if (resolveFieldOverriddenKey(options.dynamicOptions(), identifier, field.name(), "shard") + != null + || resolveFieldOverriddenKey(options.toMap(), identifier, field.name(), "shard") + != null) { + throw new IllegalArgumentException( + "Use the index type to select the layout (for example, ivf-pq or ivf-pq-centroid). Remove the shard option."); + } + String algorithm = nativeAlgorithm(); + Map nativeOptions = + nativeOptions(field.type(), options, identifier, algorithm, field.name()); + double ratio = trainSampleRatio(options, identifier, field.name()); + return createIndexer(field.type(), nativeOptions, algorithm, ratio); + } + + String nativeAlgorithm() { + return identifier(); + } + + NativeVectorGlobalIndexer createIndexer( + DataType fieldType, Map options, String algorithm, double ratio) { + return new NativeVectorGlobalIndexer(fieldType, options, algorithm, ratio); } static Map nativeOptions( DataType fieldType, Options tableOptions, String identifier, String fieldName) { + return nativeOptions(fieldType, tableOptions, identifier, identifier, fieldName); + } + + private static Map nativeOptions( + DataType fieldType, + Options tableOptions, + String identifier, + String algorithm, + String fieldName) { Map nativeOptions = new LinkedHashMap<>(); - collectNativeOptions(nativeOptions, tableOptions.toMap(), identifier, fieldName, false); collectNativeOptions( - nativeOptions, tableOptions.dynamicOptions(), identifier, fieldName, true); - - nativeOptions.put("index.type", identifier.replace('-', '_')); + nativeOptions, tableOptions.toMap(), identifier, algorithm, fieldName, false); + collectNativeOptions( + nativeOptions, + tableOptions.dynamicOptions(), + identifier, + algorithm, + fieldName, + true); + nativeOptions.put("index.type", algorithm.replace('-', '_')); nativeOptions.put( "dimension", String.valueOf(dimension(fieldType, nativeOptions, identifier))); nativeOptions.putIfAbsent("metric", NativeVectorGlobalIndexer.DEFAULT_METRIC); @@ -130,6 +160,7 @@ private static void collectNativeOptions( Map nativeOptions, Map options, String identifier, + String algorithm, String fieldName, boolean validate) { String optionPrefix = identifier + "."; @@ -144,7 +175,7 @@ private static void collectNativeOptions( entry.getKey(), entry.getKey(), entry.getValue(), - identifier, + algorithm, validate); } } @@ -156,7 +187,7 @@ private static void collectNativeOptions( optionKey, optionKey.substring(optionPrefix.length()), entry.getValue(), - identifier, + algorithm, validate); } } @@ -168,7 +199,7 @@ private static void collectNativeOptions( optionKey, optionKey.substring(fieldPrefix.length()), entry.getValue(), - identifier, + algorithm, validate); } } diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java index cbfd09b0efe9..55bb3f9c6edb 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalModelTrainer.java @@ -28,7 +28,7 @@ /** Reuses bounded heap ingestion and deterministic disk-backed sampling from the range writer. */ final class NativeVectorGlobalModelTrainer implements VectorGlobalModelTrainer { private final NativeVectorGlobalIndexWriter samples; - private final NativeVectorGlobalIndexer indexer; + private final NativeCentroidVectorGlobalIndexer indexer; private boolean finished; NativeVectorGlobalModelTrainer( @@ -36,7 +36,7 @@ final class NativeVectorGlobalModelTrainer implements VectorGlobalModelTrainer { Map options, double ratio, String identifier, - NativeVectorGlobalIndexer indexer) { + NativeCentroidVectorGlobalIndexer indexer) { this.indexer = indexer; this.samples = new NativeVectorGlobalIndexWriter(null, fieldType, options, identifier, ratio); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java index 50d4b9126cc3..cb05380821c4 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorTrainingModel.java @@ -27,9 +27,10 @@ final class NativeVectorTrainingModel implements VectorTrainingModel { private final CentroidRouter training; - private final NativeVectorGlobalIndexer localIndexer; + private final NativeCentroidVectorGlobalIndexer localIndexer; - NativeVectorTrainingModel(CentroidRouter training, NativeVectorGlobalIndexer localIndexer) { + NativeVectorTrainingModel( + CentroidRouter training, NativeCentroidVectorGlobalIndexer localIndexer) { this.localIndexer = localIndexer; this.training = training; } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorIndexShardMeta.java similarity index 74% rename from paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java rename to paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorIndexShardMeta.java index 8b8f195898b6..d6fc1adb92d1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorIndexShardMeta.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorIndexShardMeta.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.paimon.globalindex; +package org.apache.paimon.vector.index; import org.apache.paimon.utils.Range; @@ -27,7 +27,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -36,13 +35,10 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Manifest metadata shared by vector implementations and engine-independent planning. */ +/** Private wire format for the indexMeta of native centroid models and shards. */ public final class VectorIndexShardMeta { private static final byte[] MAGIC = {'P', 'C', 'S', 'M'}; private static final byte VERSION = 1; - // The preview writer emitted this key first in a LinkedHashMap. - private static final byte[] PREVIEW_PREFIX = - "{\"shardMode\":\"centroid-based\",".getBytes(StandardCharsets.UTF_8); private static final String ABSOLUTE_ROW_ID = "ABSOLUTE_ROW_ID"; private static final ObjectMapper MAPPER = new ObjectMapper(); private static final TypeReference> TYPE = @@ -55,51 +51,31 @@ private VectorIndexShardMeta(Map values, @Nullable byte[] payloa this.payloadMeta = payloadMeta; } - /** Ordinary index metadata is opaque and can use any encoding. */ - public static boolean isWrapped(@Nullable byte[] bytes) { - return startsWith(bytes, MAGIC) || startsWith(bytes, PREVIEW_PREFIX); - } - - private static boolean startsWith(@Nullable byte[] bytes, byte[] prefix) { - if (bytes == null || bytes.length < prefix.length) { - return false; - } - for (int i = 0; i < prefix.length; i++) { - if (bytes[i] != prefix[i]) { - return false; - } - } - return true; - } - public static VectorIndexShardMeta deserialize(byte[] bytes) { try { Map descriptor; byte[] payload = null; - if (startsWith(bytes, PREVIEW_PREFIX)) { - descriptor = MAPPER.readValue(bytes, TYPE); - } else { - checkArgument( - startsWith(bytes, MAGIC) - && bytes.length >= MAGIC.length + 1 + 2 * Integer.BYTES, - "Invalid centroid metadata header."); - ByteBuffer input = ByteBuffer.wrap(bytes); - input.position(MAGIC.length); - checkArgument(input.get() == VERSION, "Unsupported centroid metadata version."); - int descriptorSize = input.getInt(); - int payloadSize = input.getInt(); - checkArgument( - descriptorSize > 0 - && descriptorSize <= input.remaining() - && payloadSize >= -1 - && input.remaining() - descriptorSize == Math.max(0, payloadSize), - "Invalid centroid metadata lengths."); - descriptor = MAPPER.readValue(bytes, input.position(), descriptorSize, TYPE); - if (payloadSize >= 0) { - input.position(input.position() + descriptorSize); - payload = new byte[payloadSize]; - input.get(payload); - } + checkArgument( + bytes != null && bytes.length >= MAGIC.length + 1 + 2 * Integer.BYTES, + "Invalid centroid metadata header."); + ByteBuffer input = ByteBuffer.wrap(bytes); + for (byte expected : MAGIC) { + checkArgument(input.get() == expected, "Invalid centroid metadata header."); + } + checkArgument(input.get() == VERSION, "Unsupported centroid metadata version."); + int descriptorSize = input.getInt(); + int payloadSize = input.getInt(); + checkArgument( + descriptorSize > 0 + && descriptorSize <= input.remaining() + && payloadSize >= -1 + && input.remaining() - descriptorSize == Math.max(0, payloadSize), + "Invalid centroid metadata lengths."); + descriptor = MAPPER.readValue(bytes, input.position(), descriptorSize, TYPE); + if (payloadSize >= 0) { + input.position(input.position() + descriptorSize); + payload = new byte[payloadSize]; + input.get(payload); } checkArgument( descriptor != null diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorTrainingModelIO.java similarity index 98% rename from paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java rename to paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorTrainingModelIO.java index e326582d7229..5430d26b0394 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/VectorTrainingModelIO.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorTrainingModelIO.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.paimon.globalindex; +package org.apache.paimon.vector.index; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; diff --git a/paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory b/paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory index 28a1439e5703..f47c0a0a7317 100644 --- a/paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory +++ b/paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory @@ -18,3 +18,8 @@ org.apache.paimon.vector.index.IvfPqAlgorithmVectorGlobalIndexerFactory org.apache.paimon.vector.index.IvfSqVectorGlobalIndexerFactory org.apache.paimon.vector.index.IvfRqVectorGlobalIndexerFactory org.apache.paimon.vector.index.DiskAnnVectorGlobalIndexerFactory +org.apache.paimon.vector.index.NativeCentroidVectorGlobalIndexerFactory$IvfFlat +org.apache.paimon.vector.index.NativeCentroidVectorGlobalIndexerFactory$IvfPq +org.apache.paimon.vector.index.NativeCentroidVectorGlobalIndexerFactory$IvfSq +org.apache.paimon.vector.index.NativeCentroidVectorGlobalIndexerFactory$IvfRq +org.apache.paimon.vector.index.NativeCentroidVectorGlobalIndexerFactory$DiskAnn diff --git a/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java b/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java new file mode 100644 index 000000000000..36b0799bcb91 --- /dev/null +++ b/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.utils.Range; +import org.apache.paimon.vector.index.VectorIndexShardMeta; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Atomic manifest replacement for each registered centroid index type. */ +class CentroidIndexManifestTest { + @ParameterizedTest + @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) + void testCentroidModelsMustBeCompleteAndReplacedTogether(String type) { + List first = centroidModel("first", type); + IndexManifestFileHandler.GlobalIndexCombiner combiner = + new IndexManifestFileHandler.GlobalIndexCombiner(); + assertThat(combiner.combine(Collections.emptyList(), first)) + .containsExactlyInAnyOrderElementsOf(first); + assertThatThrownBy(() -> combiner.combine(Collections.emptyList(), first.subList(0, 2))) + .hasMessageContaining("Incomplete"); + assertThatThrownBy( + () -> + combiner.combine( + first, + Collections.singletonList(first.get(1).toDeleteEntry()))) + .hasMessageContaining("Incomplete"); + List replacement = centroidModel("second", type); + assertThatThrownBy(() -> combiner.combine(first, replacement)) + .hasMessageContaining("previous file"); + List changes = new ArrayList<>(); + first.forEach(entry -> changes.add(entry.toDeleteEntry())); + changes.addAll(replacement); + assertThat(combiner.combine(first, changes)) + .containsExactlyInAnyOrderElementsOf(replacement); + } + + @Test + void rejectsMixedIndexTypesAcrossPartitionsAtManifestWrite() { + IndexFileMeta range = + new IndexFileMeta( + "ivf-pq", + "local", + 10, + 1, + new GlobalIndexMeta(400, 401, 1, null, null), + null); + IndexManifestEntry entry = + new IndexManifestEntry(FileKind.ADD, BinaryRow.singleColumn(1), 0, range); + List centroid = centroidModel("model", "ivf-pq"); + IndexManifestFile manifest = mock(IndexManifestFile.class); + IndexManifestFileHandler handler = + new IndexManifestFileHandler(manifest, BucketMode.BUCKET_UNAWARE); + when(manifest.read("prior-range")).thenReturn(Collections.singletonList(entry)); + when(manifest.read("prior-centroid")).thenReturn(centroid); + assertThatThrownBy(() -> handler.write("prior-range", centroid)) + .hasMessageContaining("Drop the previous index"); + assertThatThrownBy(() -> handler.write("prior-centroid", Collections.singletonList(entry))) + .hasMessageContaining("Drop the previous index"); + } + + private static List centroidModel(String id, String type) { + List entries = new ArrayList<>(); + for (int centroid = -1; centroid < 2; centroid++) { + byte[] metadata = + centroid == -1 + ? VectorIndexShardMeta.routing( + id, 2, Collections.singletonList(new Range(10, 100))) + .serialize() + : VectorIndexShardMeta.centroid(id, centroid).serialize(); + IndexFileMeta file = + new IndexFileMeta( + type + "-centroid", + id + "-" + centroid, + 100, + centroid == -1 ? 0 : 10, + null, + null, + new GlobalIndexMeta(10, 100, 1, null, metadata)); + entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, file)); + } + return entries; + } +} diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java index 9cb83c64cf4b..a2c3cfeb4994 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java @@ -30,9 +30,7 @@ import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.VectorGlobalModelTrainer; -import org.apache.paimon.globalindex.VectorIndexShardMeta; import org.apache.paimon.globalindex.VectorTrainingModel; -import org.apache.paimon.globalindex.VectorTrainingModelIO; import org.apache.paimon.globalindex.io.GlobalIndexFileReader; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; import org.apache.paimon.index.vector.CentroidRouter; @@ -80,8 +78,8 @@ void distributedSampleBudgetUsesCorpusSizeAndConfiguredRatio() { options.put("index.type", "ivf_flat"); options.put("metric", "l2"); options.put("dimension", "2"); - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer( + NativeCentroidVectorGlobalIndexer indexer = + new NativeCentroidVectorGlobalIndexer( new ArrayType(new FloatType()), options, "ivf-flat", 0.25); assertThat(indexer.planGlobalTrainingSample(5)).containsExactly(0L, 2L); assertThat(indexer.planGlobalTrainingSample(1000000)).hasSize(65536); @@ -118,8 +116,8 @@ void diskAnnLocalSettingsDoNotChangeTheOuterSampleOrCenters() throws Exception { for (int seed : new int[] {42, 93}) { disk.put("diskann.seed", Integer.toString(seed)); disk.put("diskann.memory-budget-bytes", seed == 42 ? "8388608" : "4194304"); - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer( + NativeCentroidVectorGlobalIndexer indexer = + new NativeCentroidVectorGlobalIndexer( new ArrayType(new FloatType()), disk, "diskann", 1.0); assertThat(indexer.planGlobalTrainingSample(100000)).isEqualTo(expected); try (VectorTrainingModel model = @@ -148,8 +146,9 @@ void plannedRowsReproduceStreamingModel(String type) throws Exception { if (type.equals("diskann")) { options.put("diskann.seed", "93"); } - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer(new ArrayType(new FloatType()), options, type, 0.5); + NativeCentroidVectorGlobalIndexer indexer = + new NativeCentroidVectorGlobalIndexer( + new ArrayType(new FloatType()), options, type, 0.5); long[] positions = indexer.planGlobalTrainingSample(rows); assertThat(positions.length).isLessThan((rows + 1) / 2); float[] selected = new float[2 * positions.length]; @@ -185,8 +184,8 @@ void distributedBatchesTrainOnceUsingTheFullCorpusShape() throws Exception { options.put("index.type", "ivf_flat"); options.put("metric", "l2"); options.put("dimension", "2"); - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer( + NativeCentroidVectorGlobalIndexer indexer = + new NativeCentroidVectorGlobalIndexer( new ArrayType(new FloatType()), options, "ivf-flat", 0.0625); float[] samples = new float[128]; for (int i = 0; i < samples.length; i++) { @@ -223,8 +222,9 @@ void testRoutingPrunesFilesAndPreservesAbsoluteIds(String type) throws Exception if ("ivf-pq".equals(type) || diskann) { options.put("pq.m", "1"); } - NativeVectorGlobalIndexer indexer = - new NativeVectorGlobalIndexer(new ArrayType(new FloatType()), options, type); + NativeCentroidVectorGlobalIndexer indexer = + new NativeCentroidVectorGlobalIndexer( + new ArrayType(new FloatType()), options, type, 1.0); GlobalIndexFileWriter files = new GlobalIndexFileWriter() { @Override diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java similarity index 81% rename from paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java rename to paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java index 5a9bef4dd605..44b0b2db3de2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/CentroidVectorIndexUtilsTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java @@ -16,8 +16,9 @@ * limitations under the License. */ -package org.apache.paimon.globalindex; +package org.apache.paimon.vector.index; +import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.utils.Range; @@ -34,11 +35,17 @@ /** Source coverage belongs to complete model generations, never to sparse shard bounds. */ class CentroidVectorIndexUtilsTest { @Test - void onlyWrappedMetadataAffectsPlanning() { + void indexTypeSelectsTheMetadataCodec() { byte[] opaque = {(byte) 0xff, 0, 7, 42}; - assertThat(CentroidVectorIndexUtils.centroidBased(file(opaque, false))).isFalse(); + assertThat(CentroidVectorIndexUtils.centroidBased(file("ivf-pq", opaque, false))).isFalse(); VectorIndexShardMeta shard = VectorIndexShardMeta.centroid("g", 7); byte[] wrapped = shard.wrap(opaque); + IndexFileMeta ordinary = file("ivf-pq", wrapped, false); + assertThat(CentroidVectorIndexUtils.centroidBased(ordinary)).isFalse(); + assertThat(CentroidVectorIndexUtils.coverage(ordinary)) + .containsExactly(new Range(10, 1009)); + assertThatThrownBy(() -> CentroidVectorIndexUtils.coverage(file(opaque, false))) + .hasMessageContaining("Invalid centroid metadata header"); VectorIndexShardMeta decoded = VectorIndexShardMeta.deserialize(wrapped); assertThat(decoded.payloadMeta()).containsExactly(opaque); assertThat(decoded.centroid()).isEqualTo(7); @@ -59,7 +66,8 @@ void onlyWrappedMetadataAffectsPlanning() { @Test void testSparseShardsDoNotClaimEnvelopeCoverage() { List files = model("a"); - assertThat(CentroidVectorIndexUtils.validateModel(files)).isEqualTo(files.get(0)); + assertThat(CentroidVectorIndexUtils.validateModel(files)) + .containsExactly(new Range(10, 19), new Range(1000, 1009)); assertThat(CentroidVectorIndexUtils.coverage(files.get(0))) .containsExactly(new Range(10, 19), new Range(1000, 1009)); assertThat(CentroidVectorIndexUtils.coverage(files.get(1))).isEmpty(); @@ -75,7 +83,7 @@ void testIncompleteMixedAndMismatchedModelsAreRejected() { List mixed = new ArrayList<>(complete); mixed.add(file("{}", false)); assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mixed)) - .hasMessageContaining("mix"); + .hasMessageContaining("different centroid index types"); List mismatch = new ArrayList<>(complete); mismatch.set(2, model("b").get(2)); assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mismatch)) @@ -100,12 +108,16 @@ private static List model(String id) { } private static IndexFileMeta file(String json, boolean routing) { - return file(json.getBytes(java.nio.charset.StandardCharsets.UTF_8), routing); + return file("ivf-pq", json.getBytes(java.nio.charset.StandardCharsets.UTF_8), routing); } private static IndexFileMeta file(byte[] meta, boolean routing) { + return file("ivf-pq-centroid", meta, routing); + } + + private static IndexFileMeta file(String type, byte[] meta, boolean routing) { return new IndexFileMeta( - "ivf-pq", + type, Arrays.toString(meta), 100, routing ? 0 : 10, diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java index c0e034b96f98..68e9bdce82fb 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java @@ -18,15 +18,21 @@ package org.apache.paimon.vector.index; +import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils; import org.apache.paimon.options.Options; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.FloatType; import org.apache.paimon.types.VectorType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -35,6 +41,33 @@ /** Tests for vector global indexer factory SPI registration. */ public class NativeVectorGlobalIndexerFactoryTest { + @ParameterizedTest + @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) + void centroidTypeSelectsItsOwnOptionsAndLocalAlgorithm(String algorithm) { + String type = algorithm + "-centroid"; + Options options = new Options(); + options.setString(algorithm + ".dimension", "12"); + options.setString(type + ".dimension", "17"); + options.setString(type + ".metric", "cosine"); + List fields = + Collections.singletonList(new DataField(0, "v", new ArrayType(new FloatType()))); + NativeCentroidVectorGlobalIndexer indexer = + (NativeCentroidVectorGlobalIndexer) GlobalIndexer.create(type, fields, options); + assertThat(indexer.options) + .containsEntry("index.type", algorithm.replace('-', '_')) + .containsEntry("dimension", "17"); + assertThat(indexer.metric()).isEqualTo("cosine"); + assertThat(GlobalIndexerFactoryUtils.centroidFactory(type)).isNotNull(); + assertThat(GlobalIndexerFactoryUtils.centroidFactory(algorithm)).isNull(); + assertThat(GlobalIndexer.create(algorithm, fields, options)) + .isExactlyInstanceOf(NativeVectorGlobalIndexer.class); + assertThatThrownBy(() -> indexer.createWriter(null)) + .isInstanceOf(UnsupportedOperationException.class); + options.setString(algorithm + ".shard", "centroid-based"); + assertThatThrownBy(() -> GlobalIndexer.create(algorithm, fields, options)) + .hasMessageContaining("index type"); + } + @Test public void testIdentifier() { assertThat(new IvfFlatVectorGlobalIndexerFactory().identifier()).isEqualTo("ivf-flat"); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/VectorTrainingModelIOTest.java similarity index 99% rename from paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java rename to paimon-vector/src/test/java/org/apache/paimon/vector/index/VectorTrainingModelIOTest.java index 043dd2b7c7b8..e2011c6a0688 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/VectorTrainingModelIOTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/VectorTrainingModelIOTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.paimon.globalindex; +package org.apache.paimon.vector.index; import org.junit.jupiter.api.Test; From 10130a35651b3ef65d3b05990a222d48b99b85fb Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 19 Nov 2026 10:43:37 +0800 Subject: [PATCH 6/7] [test] Verify centroid type conflicts with persisted manifests --- .../manifest/CentroidIndexManifestTest.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java b/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java index 36b0799bcb91..ab94abf91061 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java @@ -18,7 +18,9 @@ package org.apache.paimon.manifest; +import org.apache.paimon.TestAppendFileStore; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.format.FileFormat; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.table.BucketMode; @@ -26,17 +28,18 @@ import org.apache.paimon.vector.index.VectorIndexShardMeta; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** Atomic manifest replacement for each registered centroid index type. */ class CentroidIndexManifestTest { @@ -67,7 +70,8 @@ void testCentroidModelsMustBeCompleteAndReplacedTogether(String type) { } @Test - void rejectsMixedIndexTypesAcrossPartitionsAtManifestWrite() { + void rejectsMixedIndexTypesAcrossPartitionsAtManifestWrite(@TempDir Path directory) + throws Exception { IndexFileMeta range = new IndexFileMeta( "ivf-pq", @@ -79,14 +83,23 @@ void rejectsMixedIndexTypesAcrossPartitionsAtManifestWrite() { IndexManifestEntry entry = new IndexManifestEntry(FileKind.ADD, BinaryRow.singleColumn(1), 0, range); List centroid = centroidModel("model", "ivf-pq"); - IndexManifestFile manifest = mock(IndexManifestFile.class); + TestAppendFileStore store = + TestAppendFileStore.createAppendStore(directory, new HashMap<>(), "file"); + IndexManifestFile manifest = + new IndexManifestFile.Factory( + store.fileIO(), + FileFormat.manifestFormat(store.options()), + "zstd", + store.pathFactory(), + null) + .create(); IndexManifestFileHandler handler = new IndexManifestFileHandler(manifest, BucketMode.BUCKET_UNAWARE); - when(manifest.read("prior-range")).thenReturn(Collections.singletonList(entry)); - when(manifest.read("prior-centroid")).thenReturn(centroid); - assertThatThrownBy(() -> handler.write("prior-range", centroid)) + String priorRange = handler.write(null, Collections.singletonList(entry)); + String priorCentroid = handler.write(null, centroid); + assertThatThrownBy(() -> handler.write(priorRange, centroid)) .hasMessageContaining("Drop the previous index"); - assertThatThrownBy(() -> handler.write("prior-centroid", Collections.singletonList(entry))) + assertThatThrownBy(() -> handler.write(priorCentroid, Collections.singletonList(entry))) .hasMessageContaining("Drop the previous index"); } @@ -108,7 +121,7 @@ private static List centroidModel(String id, String type) { null, null, new GlobalIndexMeta(10, 100, 1, null, metadata)); - entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.EMPTY_ROW, 0, file)); + entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.singleColumn(0), 0, file)); } return entries; } From 643346bdd9b5a6d0695ed645c2e99f41132e67e8 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 19 Nov 2026 10:53:07 +0800 Subject: [PATCH 7/7] [vector] Allow range and centroid index types across partitions --- .../multimodal-table/global-index/vector.mdx | 9 +- .../CentroidVectorIndexFactory.java | 7 +- .../globalindex/CentroidVectorIndexUtils.java | 16 +-- .../manifest/IndexManifestFileHandler.java | 46 +------ .../AbstractDataEvolutionVectorRead.java | 21 +-- .../source/DataEvolutionBatchVectorRead.java | 25 +++- .../table/source/DataEvolutionVectorRead.java | 22 ++- .../table/source/DataEvolutionVectorScan.java | 129 +++++++----------- .../read/SparkDataEvolutionVectorRead.java | 54 ++------ .../SparkDataEvolutionVectorReadTest.java | 42 +++--- .../spark/sql/CentroidVectorIndexTest.scala | 59 ++++++++ .../NativeCentroidRoutedIndexReader.java | 32 ++--- ...iveCentroidVectorGlobalIndexerFactory.java | 35 ----- .../manifest/CentroidIndexManifestTest.java | 128 ----------------- .../index/CentroidVectorGlobalIndexTest.java | 6 - .../index/CentroidVectorIndexUtilsTest.java | 22 --- 16 files changed, 210 insertions(+), 443 deletions(-) delete mode 100644 paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java diff --git a/docs/docs/multimodal-table/global-index/vector.mdx b/docs/docs/multimodal-table/global-index/vector.mdx index b0ffa49d05d9..8a8b4b85d58b 100644 --- a/docs/docs/multimodal-table/global-index/vector.mdx +++ b/docs/docs/multimodal-table/global-index/vector.mdx @@ -488,9 +488,12 @@ CALL sys.create_global_index( Use the registered index type in both `index_type` and the option prefix. The ordinary names (`ivf-pq`, for example) select range-based sharding. -Drop the previous index on the vector column before switching types, including -when different partitions are built in separate calls. A centroid -build pins one snapshot. Spark map tasks count non-null vectors in their input +Different partitions of the same table can use different layouts. For example, keep an +existing `ivf-pq` partition while building a new one with `ivf-pq-centroid`. A search +opens each partition with its registered reader and merges results by score. Set the same +distance metric for the column across layouts, for example with `fields.embedding.metric`. + +A centroid build pins one snapshot. Spark map tasks count non-null vectors in their input splits. Using those counts and the configured training ratio, the driver plans the exact row ordinals that the outer IVF-FLAT trainer would retain. Each map task reads its own splits and shuffles selected vectors by reservoir slot. diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java index f6544376fecb..1ffe76af5511 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexFactory.java @@ -24,13 +24,10 @@ /** * A centroid index groups all files for one field in a partition into a single generation. Its - * reader accepts and returns absolute table row IDs. Committers and scan planners can inspect the - * generation through this factory without opening files or initializing the native runtime. + * reader accepts and returns absolute table row IDs. Scan planners can obtain its source coverage + * through this factory without opening files or initializing the native runtime. */ public interface CentroidVectorIndexFactory extends GlobalIndexerFactory { /** Exact indexed row ranges for a model entry, or an empty list for a shard entry. */ List sourceRowRanges(byte[] indexMeta); - - /** Validate that these entries form one complete generation and return its source coverage. */ - List validateGeneration(List indexMetas); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java index 2dbe24d336f9..cb4735291cd7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/CentroidVectorIndexUtils.java @@ -21,13 +21,10 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.utils.Range; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.apache.paimon.utils.Preconditions.checkArgument; - -/** Validation and coverage of an atomically committed centroid model and its shards. */ +/** Source row coverage supplied by the registered vector index type. */ public final class CentroidVectorIndexUtils { private CentroidVectorIndexUtils() {} @@ -40,17 +37,6 @@ public static boolean isRoutingModel(IndexFileMeta file) { return centroidBased(file) && !coverage(file).isEmpty(); } - public static List validateModel(List files) { - String type = files.get(0).indexType(); - List metadata = new ArrayList<>(files.size()); - for (IndexFileMeta file : files) { - checkArgument( - type.equals(file.indexType()), "Cannot mix different centroid index types."); - metadata.add(file.globalIndexMeta().indexMeta()); - } - return GlobalIndexerFactoryUtils.centroidFactory(type).validateGeneration(metadata); - } - public static List coverage(IndexFileMeta file) { CentroidVectorIndexFactory factory = GlobalIndexerFactoryUtils.centroidFactory(file.indexType()); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java index 98d6bea138d1..d78740ffc8f9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java @@ -19,7 +19,6 @@ package org.apache.paimon.manifest; import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.globalindex.CentroidVectorIndexUtils; import org.apache.paimon.index.DataEvolutionIndexSourceMeta; import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.GlobalIndexMeta; @@ -73,39 +72,14 @@ String write(@Nullable String previousIndexManifest, List ne Set indexes = new HashSet<>(); indexes.addAll(previous.keySet()); indexes.addAll(current.keySet()); - Map centroidTypes = new HashMap<>(); for (String indexName : indexes) { - List combined = + indexEntries.addAll( getIndexManifestFileCombine(indexName) .combine( previous.getOrDefault(indexName, Collections.emptyList()), - current.getOrDefault(indexName, Collections.emptyList())); - if (!combined.isEmpty() - && CentroidVectorIndexUtils.centroidBased(combined.get(0).indexFile())) { - for (IndexManifestEntry entry : combined) { - centroidTypes.put( - entry.indexFile().globalIndexMeta().indexFieldId(), indexName); - } - } - indexEntries.addAll(combined); + current.getOrDefault(indexName, Collections.emptyList()))); } - // Queries resolve a single vector index type for the column across partitions. - if (!centroidTypes.isEmpty()) { - for (IndexManifestEntry entry : indexEntries) { - IndexFileMeta file = entry.indexFile(); - if (file.globalIndexMeta() != null) { - int field = file.globalIndexMeta().indexFieldId(); - String type = centroidTypes.get(field); - checkState( - type == null || type.equals(file.indexType()), - "Cannot mix %s and %s on vector field %s. Drop the previous index before switching types.", - type, - file.indexType(), - field); - } - } - } return indexManifestFile.writeWithoutRolling(indexEntries); } @@ -261,22 +235,6 @@ public List combine( for (IndexManifestEntry entry : added) { indexEntries.put(entry.indexFile().fileName(), entry); } - if (!indexEntries.isEmpty() - && CentroidVectorIndexUtils.centroidBased( - indexEntries.values().iterator().next().indexFile())) { - Map>> partitions = new HashMap<>(); - for (IndexManifestEntry entry : indexEntries.values()) { - IndexFileMeta file = entry.indexFile(); - partitions - .computeIfAbsent(entry.partition(), key -> new HashMap<>()) - .computeIfAbsent( - file.globalIndexMeta().indexFieldId(), key -> new ArrayList<>()) - .add(file); - } - for (Map> partition : partitions.values()) { - partition.values().forEach(CentroidVectorIndexUtils::validateModel); - } - } return new ArrayList<>(indexEntries.values()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index 4ec380110d49..d19b6601c312 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -63,6 +63,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -71,6 +72,7 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -139,6 +141,17 @@ protected AbstractDataEvolutionVectorRead( } } + protected static Collection> groupByIndexType( + List splits) { + Map> groups = new TreeMap<>(); + for (IndexVectorSearchSplit split : splits) { + groups.computeIfAbsent( + split.vectorIndexFiles().get(0).indexType(), k -> new ArrayList<>()) + .add(split); + } + return groups.values(); + } + protected GlobalIndexer createGlobalIndexer(List splits) { IndexFileMeta firstFile = firstVectorIndexFile(splits); GlobalIndexMeta firstMeta = checkNotNull(firstFile.globalIndexMeta()); @@ -503,14 +516,6 @@ protected String vectorIndexType(List splits) { return firstVectorIndexFile(splits).indexType(); } - protected ScoredGlobalIndexResult[] emptyScoredResults(int n) { - ScoredGlobalIndexResult[] results = new ScoredGlobalIndexResult[n]; - for (int i = 0; i < n; i++) { - results[i] = ScoredGlobalIndexResult.createEmpty(); - } - return results; - } - private List buildIOMetaList( IndexPathFactory indexPathFactory, List vectorIndexFiles) { List indexIOMetaList = new ArrayList<>(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionBatchVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionBatchVectorRead.java index a94f0d6dba9b..14b6367a5f72 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionBatchVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionBatchVectorRead.java @@ -79,19 +79,30 @@ private List readBatch(List splits) { return empty; } - GlobalIndexer globalIndexer = - indexSplits.isEmpty() ? null : createGlobalIndexer(indexSplits); - ScoredGlobalIndexResult[] indexedResults = - indexSplits.isEmpty() - ? emptyScoredResults(n) - : readIndexedBatch(indexSplits, globalIndexer); + List byType = new ArrayList<>(); + GlobalIndexer rawIndexer = null; + for (List group : groupByIndexType(indexSplits)) { + GlobalIndexer indexer = createGlobalIndexer(group); + byType.add(readIndexedBatch(group, indexer)); + if (rawIndexer == null) { + rawIndexer = indexer; + } + } List results = new ArrayList<>(n); RoaringNavigableMap64 rawPreFilter = rawPreFilter(rawSplits); for (int i = 0; i < n; i++) { + List partial = new ArrayList<>(byType.size()); + for (ScoredGlobalIndexResult[] group : byType) { + partial.add(group[i]); + } results.add( withRawSearch( - indexedResults[i], rawSplits, globalIndexer, rawPreFilter, vectors[i])); + ScoredGlobalIndexResult.merge(partial), + rawSplits, + rawIndexer, + rawPreFilter, + vectors[i])); } return results; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorRead.java index fbac5ef7b786..ee5b14aa10aa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorRead.java @@ -73,13 +73,21 @@ protected GlobalIndexResult readSplits(List splits) return GlobalIndexResult.createEmpty(); } - GlobalIndexer globalIndexer = - indexSplits.isEmpty() ? null : createGlobalIndexer(indexSplits); - ScoredGlobalIndexResult result = - indexSplits.isEmpty() - ? ScoredGlobalIndexResult.createEmpty() - : readIndexed(indexSplits, globalIndexer); - return withRawSearch(result, rawSplits, globalIndexer, rawPreFilter(rawSplits), vector); + List results = new ArrayList<>(); + GlobalIndexer rawIndexer = null; + for (List group : groupByIndexType(indexSplits)) { + GlobalIndexer indexer = createGlobalIndexer(group); + results.add(readIndexed(group, indexer)); + if (rawIndexer == null) { + rawIndexer = indexer; + } + } + return withRawSearch( + ScoredGlobalIndexResult.merge(results), + rawSplits, + rawIndexer, + rawPreFilter(rawSplits), + vector); } protected ScoredGlobalIndexResult readIndexed( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java index c2f079029794..1a4a6be30e3a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java @@ -38,6 +38,7 @@ import org.apache.paimon.table.source.snapshot.TimeTravelUtil; import org.apache.paimon.types.DataField; import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Range; import javax.annotation.Nullable; @@ -127,75 +128,59 @@ public Plan scan() { indexEntries.stream() .map(IndexManifestEntry::indexFile) .collect(Collectors.toList()); - String vectorIndexType = vectorIndexType(allIndexFiles); - if (vectorIndexType == null) { - vectorIndexType = configuredVectorIndexType(); - } - - // Group vector index files by (rowRangeStart, rowRangeEnd). A file is treated as the vector - // index for this search only when the vector column is its PRIMARY field (indexFieldId); - // the canonical ES hybrid layout is vector-as-primary with text/scalar columns carried as - // extra fields. If the vector column were instead an extra field, no IndexVectorSearchSplit - // is produced here and the search falls back to a brute-force RawVectorSearchSplit (correct - // results, but the ANN index is bypassed). - Map> vectorByRange = new HashMap<>(); + // Only the primary field serves this vector search. Each split goes to its own index type. + Map, List> rangeGroups = new HashMap<>(); + Map, List> centroidGroups = new HashMap<>(); List vectorIndexFiles = new ArrayList<>(); - for (IndexFileMeta indexFile : allIndexFiles) { - GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta()); + for (IndexManifestEntry entry : indexEntries) { + IndexFileMeta file = entry.indexFile(); + GlobalIndexMeta meta = checkNotNull(file.globalIndexMeta()); if (isPrimaryColumn(meta, vectorColumn.id())) { - vectorIndexFiles.add(indexFile); - if (CentroidVectorIndexUtils.centroidBased(indexFile)) { - continue; + vectorIndexFiles.add(file); + if (CentroidVectorIndexUtils.centroidBased(file)) { + centroidGroups + .computeIfAbsent( + Pair.of(entry.partition(), file.indexType()), + k -> new ArrayList<>()) + .add(file); + } else { + rangeGroups + .computeIfAbsent( + Pair.of(meta.rowRange(), file.indexType()), + k -> new ArrayList<>()) + .add(file); } - Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd()); - vectorByRange.computeIfAbsent(range, k -> new ArrayList<>()).add(indexFile); } } - // Build splits: for each vector range, attach matching scalar index files List splits = new ArrayList<>(); - for (Map.Entry> entry : vectorByRange.entrySet()) { - Range range = entry.getKey(); - List vectorFiles = entry.getValue(); - List scalarFiles = - allIndexFiles.stream() - .filter( - f -> { - GlobalIndexMeta globalIndex = - checkNotNull(f.globalIndexMeta()); - if (!canServeScalarFilter(globalIndex)) { - return false; - } - return range.hasIntersection(globalIndex.rowRange()); - }) + List indexedRanges = new ArrayList<>(); + for (Map.Entry, List> entry : rangeGroups.entrySet()) { + Range range = entry.getKey().getLeft(); + indexedRanges.add(range); + splits.add( + new IndexVectorSearchSplit( + range.from, + range.to, + entry.getValue(), + scalarIndexFiles(allIndexFiles, Collections.singletonList(range)))); + } + for (List files : centroidGroups.values()) { + Range envelope = files.get(0).globalIndexMeta().rowRange(); + List coverage = + files.stream() + .flatMap(file -> CentroidVectorIndexUtils.coverage(file).stream()) .collect(Collectors.toList()); - splits.add(new IndexVectorSearchSplit(range.from, range.to, vectorFiles, scalarFiles)); + indexedRanges.addAll(coverage); + splits.add( + new IndexVectorSearchSplit( + envelope.from, + envelope.to, + files, + scalarIndexFiles(allIndexFiles, coverage))); } CoreOptions effectiveOptions = effectiveCoreOptions(); - Map> partitionVectors = new HashMap<>(); - for (IndexManifestEntry entry : indexEntries) { - if (isPrimaryColumn( - checkNotNull(entry.indexFile().globalIndexMeta()), vectorColumn.id())) { - partitionVectors - .computeIfAbsent(entry.partition(), p -> new ArrayList<>()) - .add(entry.indexFile()); - } - } - for (List files : partitionVectors.values()) { - if (files.stream().noneMatch(CentroidVectorIndexUtils::centroidBased)) { - continue; - } - List coverage = CentroidVectorIndexUtils.validateModel(files); - long from = coverage.get(0).from; - long to = coverage.get(coverage.size() - 1).to; - List scalarFiles = scalarIndexFiles(allIndexFiles, coverage); - splits.add(new IndexVectorSearchSplit(from, to, files, scalarFiles)); - // FAST scalar fallback is restricted to actual source coverage, not the envelope. - for (Range range : coverage) { - vectorByRange.putIfAbsent(range, Collections.emptyList()); - } - } GlobalIndexSearchMode vectorSearchMode = effectiveOptions.vectorIndexSearchMode(); List rawRowRanges = new DataEvolutionGlobalIndexCoverage( @@ -216,8 +201,7 @@ public Plan scan() { scalarUnindexedRanges = Range.and( scalarUnindexedRanges, - Range.sortAndMergeOverlap( - new ArrayList<>(vectorByRange.keySet()), true)); + Range.sortAndMergeOverlap(indexedRanges, true)); } rawRowRanges = Range.sortAndMergeOverlap(addAll(rawRowRanges, scalarUnindexedRanges), true); @@ -227,7 +211,10 @@ public Plan scan() { new RawVectorSearchSplit( rawRowRanges, scalarIndexFiles(allIndexFiles, rawRowRanges), - vectorIndexType)); + vectorIndexFiles.stream() + .map(IndexFileMeta::indexType) + .min(String::compareTo) + .orElseGet(this::configuredVectorIndexType))); } @Nullable Snapshot planSnapshot = snapshot; @@ -351,26 +338,6 @@ private static List addAll(List left, List right) { return result; } - @Nullable - private String vectorIndexType(List indexFiles) { - String indexType = null; - for (IndexFileMeta indexFile : indexFiles) { - GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta()); - if (!isPrimaryColumn(meta, vectorColumn.id())) { - continue; - } - if (indexType == null) { - indexType = indexFile.indexType(); - } else if (!indexType.equals(indexFile.indexType())) { - throw new IllegalArgumentException( - String.format( - "Vector column '%s' has multiple index types: %s and %s.", - vectorColumn.name(), indexType, indexFile.indexType())); - } - } - return indexType; - } - @Nullable private String configuredVectorIndexType() { return VectorIndexOptions.resolve( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java index 1233c70468f9..7037f19bcac3 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorRead.java @@ -19,7 +19,6 @@ package org.apache.paimon.spark.read; import org.apache.paimon.globalindex.GlobalIndexReadThreadPool; -import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.GlobalIndexResultSerializer; import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; @@ -29,8 +28,6 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.source.DataEvolutionVectorRead; import org.apache.paimon.table.source.IndexVectorSearchSplit; -import org.apache.paimon.table.source.RawVectorSearchSplit; -import org.apache.paimon.table.source.VectorScan; import org.apache.paimon.types.DataField; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Range; @@ -70,37 +67,15 @@ public SparkDataEvolutionVectorRead( } @Override - public GlobalIndexResult read(VectorScan.Plan plan) { - this.planSnapshot = plan.snapshot(); - List indexSplits = new ArrayList<>(); - List rawSplits = new ArrayList<>(); - prepareSplits(plan.splits(), indexSplits, rawSplits); - if (indexSplits.isEmpty() && rawSplits.isEmpty()) { - return GlobalIndexResult.createEmpty(); - } - - GlobalIndexer globalIndexer = - !indexSplits.isEmpty() && !rawSplits.isEmpty() - ? createGlobalIndexer(indexSplits) - : null; - ScoredGlobalIndexResult result = - indexSplits.isEmpty() - ? ScoredGlobalIndexResult.createEmpty() - : readIndexSplitsInSpark(indexSplits, globalIndexer); - return result.or(readRawSplitsInSpark(rawSplits, globalIndexer, rawPreFilter(rawSplits))) - .topK(limit); - } - - protected ScoredGlobalIndexResult readIndexSplitsInSpark( - List splits, @Nullable GlobalIndexer globalIndexer) { + protected ScoredGlobalIndexResult readIndexed( + List splits, GlobalIndexer globalIndexer) { if (splits.isEmpty()) { return ScoredGlobalIndexResult.createEmpty(); } int parallelism = sparkParallelism(); if (splits.size() < parallelism * 2) { - return readIndexed( - splits, globalIndexer == null ? createGlobalIndexer(splits) : globalIndexer); + return super.readIndexed(splits, globalIndexer); } List preFilters = preFilters(splits); @@ -174,31 +149,26 @@ protected ScoredGlobalIndexResult readIndexSplitsInSpark( List remoteResults = mapInSpark(splitGroups, task, splitGroups.size()); - GlobalIndexer rerankGlobalIndexer = - globalIndexer == null ? createGlobalIndexer(splits) : globalIndexer; return maybeRerankIndexedResult( - mergeRemoteResults(remoteResults, searchLimit), - indexType, - rerankGlobalIndexer, - vector); + mergeRemoteResults(remoteResults, searchLimit), indexType, globalIndexer, vector); } - protected ScoredGlobalIndexResult readRawSplitsInSpark( - List splits, + @Override + protected ScoredGlobalIndexResult readRawSearch( + List rawRowRanges, + @Nullable RoaringNavigableMap64 preFilter, @Nullable GlobalIndexer globalIndexer, - @Nullable RoaringNavigableMap64 preFilter) { - List rawRowRanges = rawRowRanges(splits); + float[] queryVector) { if (rawRowRanges.isEmpty()) { return ScoredGlobalIndexResult.createEmpty(); } int parallelism = sparkParallelism(); if (rawRowCount(rawRowRanges) < parallelism * 2L) { - return readRawSearch( - rawRowRanges, preFilter, rawSearchIndexer(splits, globalIndexer), vector); + return super.readRawSearch(rawRowRanges, preFilter, globalIndexer, queryVector); } - String metric = rawSearchMetric(rawSearchIndexer(splits, globalIndexer)); + String metric = rawSearchMetric(globalIndexer); List> rangeGroups = rangeGroups(rawRowRanges, parallelism); List serializedSplits = new ArrayList<>(rangeGroups.size()); for (List rangeGroup : rangeGroups) { @@ -225,7 +195,7 @@ protected ScoredGlobalIndexResult readRawSplitsInSpark( rowRanges, deserializePreFilter(serializedSplit.preFilter), metric, - vector); + queryVector); results.add(splitResult); } ScoredGlobalIndexResult result = diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java index ce87c4e9a48f..a2d34692da60 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java @@ -147,12 +147,14 @@ private static Snapshot snapshot(long id) { @Test public void testRawSearchSplitsRangesAcrossSparkTasks() { RecordingSparkVectorRead read = new RecordingSparkVectorRead(); - RawVectorSearchSplit rawSplit = - new RawVectorSearchSplit( - Collections.singletonList(new Range(0, 63)), Collections.emptyList(), null); - - ScoredGlobalIndexResult result = - read.readRawSplitsInSpark(Collections.singletonList(rawSplit), null, null); + GlobalIndexResult result = + read.read( + () -> + Collections.singletonList( + new RawVectorSearchSplit( + Collections.singletonList(new Range(0, 63)), + Collections.emptyList(), + null))); assertThat(read.rawSearchRanges).containsExactly(new Range(0, 31), new Range(32, 63)); assertThat(read.sparkParallelism).isEqualTo(2); @@ -164,7 +166,7 @@ public void testDistributedIndexRefinesAfterGlobalMerge() { DistributedRefineSparkVectorRead read = new DistributedRefineSparkVectorRead(); ScoredGlobalIndexResult result = - read.readIndexSplitsInSpark(indexSplits("test-vector-ann", 4), new L2Indexer()); + read.readIndexed(indexSplits("test-vector-ann", 4), new L2Indexer()); assertThat(read.sparkParallelism).isEqualTo(2); assertThat(read.rawSearchCandidateRows).containsExactly(0L, 2L); @@ -178,8 +180,7 @@ public void testDistributedIndexMaterializesManySplitsInSingleTask() throws Exce SingleTaskSparkVectorRead read = new SingleTaskSparkVectorRead(createTable()); ScoredGlobalIndexResult result = - read.readIndexSplitsInSpark( - indexSplits("test-vector-ann", splitCount), new L2Indexer()); + read.readIndexed(indexSplits("test-vector-ann", splitCount), new L2Indexer()); assertThat(read.sparkParallelism).isEqualTo(1); assertThat(read.scoreCalls).hasValue(splitCount); @@ -201,8 +202,11 @@ public void testSharedFieldOptionsReachLocalAndDistributedIndexReads(int splitCo SingleTaskSparkVectorRead read = new SingleTaskSparkVectorRead(createTable(options), "cosine"); - ScoredGlobalIndexResult result = - read.readIndexSplitsInSpark(indexSplits("test-shared-vector", splitCount), null); + GlobalIndexResult result = + read.read( + () -> + new java.util.ArrayList<>( + indexSplits("test-shared-vector", splitCount))); assertThat(read.evaluatedSplits).hasValue(splitCount); assertThat(result.results()).contains(splitCount - 1L); @@ -262,23 +266,19 @@ private Snapshot plannedSnapshot() { } @Override - protected GlobalIndexResult readSplits(List splits) { - throw new AssertionError("Raw search should not fall back to local vector read."); - } - - @Override - protected ScoredGlobalIndexResult readIndexSplitsInSpark( + protected ScoredGlobalIndexResult readIndexed( List splits, @Nullable GlobalIndexer globalIndexer) { throw new AssertionError("Index search is not part of this test."); } @Override - protected ScoredGlobalIndexResult readRawSplitsInSpark( - List splits, + protected ScoredGlobalIndexResult readRawSearch( + List ranges, + @Nullable RoaringNavigableMap64 preFilter, @Nullable GlobalIndexer globalIndexer, - @Nullable RoaringNavigableMap64 preFilter) { + float[] queryVector) { rawSparkPathUsed = true; - assertThat(splits).hasSize(1); + assertThat(ranges).containsExactly(new Range(42, 42)); assertThat(globalIndexer).isNull(); assertThat(preFilter).isNull(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala index c34e873b6c3a..13faa8fdc5c9 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CentroidVectorIndexTest.scala @@ -120,6 +120,65 @@ class CentroidVectorIndexTest extends PaimonSparkTestBase { assert(nearest(300) != 300) } } + Seq(true, false).foreach { + centroidFirst => + test( + s"range and centroid partitions share vector search with centroid built first=$centroidFirst") { + withTable("T") { + spark.sql("""CREATE TABLE T (id INT, v ARRAY, p INT) PARTITIONED BY (p) + |TBLPROPERTIES ('bucket'='-1', 'row-tracking.enabled'='true', + |'data-evolution.enabled'='true', 'vector-index.search-mode'='detail', + |'fields.v.dimension'='2', 'fields.v.metric'='l2', + |'global-index.row-count-per-shard'='32')""".stripMargin) + Seq(0, 1, 2).foreach { + part => + val first = if (part == 2) 5000 else 0 + spark.sql(s"""INSERT INTO T SELECT CAST(id + ${part * 128} AS INT), + |array(CAST(IF(id = 0, $first, 1000 + id) AS FLOAT), 0.0f), $part + |FROM range(0, 128, 1, 1)""".stripMargin) + } + val builds = Seq(("ivf-pq-centroid", 0), ("ivf-pq", 1)) + (if (centroidFirst) builds else builds.reverse).foreach { + case (kind, part) => + spark + .sql(s"""CALL sys.create_global_index(table => 'test.T', index_column => 'v', + |index_type => '$kind', partitions => 'p=$part', + |options => '$kind.nlist=2,$kind.pq.m=1')""".stripMargin) + .collect() + } + + val ids = spark + .sql("""SELECT id FROM vector_search('T','v',array(0.0f,0.0f),2, + |map('ivf.nprobe','2','centroid.nprobe','2'))""".stripMargin) + .collect() + .map(_.getInt(0)) + .toSet + assert(ids == Set(0, 128)) + + val table = loadTable("T") + val local = table + .newVectorSearchBuilder() + .withVectorColumn("v") + .withVector(Array(0.0f, 0.0f)) + .withLimit(2) + .withOption("centroid.nprobe", "2") + .withOption("ivf.nprobe", "2") + .executeLocal() + assert(local.results().toArray(2).toSet == Set(0L, 128L)) + val batch = table + .newBatchVectorSearchBuilder() + .withVectorColumn("v") + .withVectors(Array(Array(0.0f, 0.0f), Array(5000.0f, 0.0f))) + .withLimit(2) + .withOption("centroid.nprobe", "2") + .withOption("ivf.nprobe", "2") + .executeBatchLocal() + assert(batch.get(0).results().toArray(2).toSet == Set(0L, 128L)) + assert(batch.get(1).results().contains(256L)) + } + } + } + test("distributed training preserves the model across sparse splits and different task counts") { withTable("T") { spark.sql("""CREATE TABLE T (id INT, v ARRAY) TBLPROPERTIES ( diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java index ee284fbf9750..e5c65914f1f8 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidRoutedIndexReader.java @@ -40,6 +40,7 @@ import java.util.concurrent.ExecutorService; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** Routes before opening shard files; all filters and results use absolute Paimon row IDs. */ final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexReader { @@ -67,13 +68,9 @@ final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexRea this.indexType = indexType; GlobalIndexIOMeta routing = null; VectorIndexShardMeta modelMeta = null; - String modelId = null; long count = 0; for (GlobalIndexIOMeta file : entries) { VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(file.metadata()); - String id = meta.modelId(); - checkArgument(modelId == null || modelId.equals(id), "Mismatched centroid model."); - modelId = id; if (meta.routingModel()) { modelMeta = meta; checkArgument(routing == null, "Multiple routing models."); @@ -81,26 +78,19 @@ final class NativeCentroidRoutedIndexReader extends AbstractVectorGlobalIndexRea } else { checkArgument( file.rowCount() >= 0, "Centroid shards must record their vector count."); - checkArgument( - shards.put( - meta.centroid(), - new GlobalIndexIOMeta( - file.filePath(), - file.fileSize(), - file.rowCount(), - meta.payloadMeta())) - == null, - "Duplicate centroid shard."); + shards.put( + meta.centroid(), + new GlobalIndexIOMeta( + file.filePath(), + file.fileSize(), + file.rowCount(), + meta.payloadMeta())); count = Math.addExact(count, file.rowCount()); } } checkArgument(routing != null, "Missing routing model."); this.routing = routing; this.metadata = modelMeta; - checkArgument(shards.size() == metadata.nlist(), "Incomplete centroid shard set."); - for (int id : shards.keySet()) { - checkArgument(id < metadata.nlist(), "Invalid centroid ID."); - } this.vectorCount = count; } @@ -134,7 +124,11 @@ private synchronized NativeVectorGlobalIndexReader reader(int centroid) { c -> new NativeVectorGlobalIndexReader( files, - Collections.singletonList(shards.get(c)), + Collections.singletonList( + checkNotNull( + shards.get(c), + "Missing routed centroid shard %s.", + c)), fieldType, executor, indexType)); diff --git a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java index 853d03e75891..17cc7b92f64b 100644 --- a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java +++ b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeCentroidVectorGlobalIndexerFactory.java @@ -23,12 +23,8 @@ import org.apache.paimon.utils.Range; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; - -import static org.apache.paimon.utils.Preconditions.checkArgument; /** Registered centroid index types share the router and select a local native algorithm. */ public abstract class NativeCentroidVectorGlobalIndexerFactory @@ -61,37 +57,6 @@ public List sourceRowRanges(byte[] indexMeta) { return meta.routingModel() ? meta.sourceRowRanges() : Collections.emptyList(); } - @Override - public List validateGeneration(List indexMetas) { - List coverage = null; - String modelId = null; - int nlist = 0; - Set centroids = new HashSet<>(); - for (byte[] bytes : indexMetas) { - VectorIndexShardMeta meta = VectorIndexShardMeta.deserialize(bytes); - String id = meta.modelId(); - checkArgument( - modelId == null || modelId.equals(id), - "Centroid shards belong to different models."); - modelId = id; - if (meta.routingModel()) { - checkArgument( - coverage == null, - "Multiple routing models for the same partition and index."); - coverage = meta.sourceRowRanges(); - nlist = meta.nlist(); - } else { - checkArgument(centroids.add(meta.centroid()), "Duplicate centroid shard."); - } - } - checkArgument(coverage != null, "Missing centroid routing model."); - for (int centroid : centroids) { - checkArgument(centroid < nlist, "Invalid centroid shard."); - } - checkArgument(centroids.size() == nlist, "Incomplete centroid model shard set."); - return coverage; - } - /** Centroid routing over IVF-FLAT shards. */ public static final class IvfFlat extends NativeCentroidVectorGlobalIndexerFactory { public IvfFlat() { diff --git a/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java b/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java deleted file mode 100644 index ab94abf91061..000000000000 --- a/paimon-vector/src/test/java/org/apache/paimon/manifest/CentroidIndexManifestTest.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.manifest; - -import org.apache.paimon.TestAppendFileStore; -import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.format.FileFormat; -import org.apache.paimon.index.GlobalIndexMeta; -import org.apache.paimon.index.IndexFileMeta; -import org.apache.paimon.table.BucketMode; -import org.apache.paimon.utils.Range; -import org.apache.paimon.vector.index.VectorIndexShardMeta; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Atomic manifest replacement for each registered centroid index type. */ -class CentroidIndexManifestTest { - @ParameterizedTest - @ValueSource(strings = {"ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann"}) - void testCentroidModelsMustBeCompleteAndReplacedTogether(String type) { - List first = centroidModel("first", type); - IndexManifestFileHandler.GlobalIndexCombiner combiner = - new IndexManifestFileHandler.GlobalIndexCombiner(); - assertThat(combiner.combine(Collections.emptyList(), first)) - .containsExactlyInAnyOrderElementsOf(first); - assertThatThrownBy(() -> combiner.combine(Collections.emptyList(), first.subList(0, 2))) - .hasMessageContaining("Incomplete"); - assertThatThrownBy( - () -> - combiner.combine( - first, - Collections.singletonList(first.get(1).toDeleteEntry()))) - .hasMessageContaining("Incomplete"); - List replacement = centroidModel("second", type); - assertThatThrownBy(() -> combiner.combine(first, replacement)) - .hasMessageContaining("previous file"); - List changes = new ArrayList<>(); - first.forEach(entry -> changes.add(entry.toDeleteEntry())); - changes.addAll(replacement); - assertThat(combiner.combine(first, changes)) - .containsExactlyInAnyOrderElementsOf(replacement); - } - - @Test - void rejectsMixedIndexTypesAcrossPartitionsAtManifestWrite(@TempDir Path directory) - throws Exception { - IndexFileMeta range = - new IndexFileMeta( - "ivf-pq", - "local", - 10, - 1, - new GlobalIndexMeta(400, 401, 1, null, null), - null); - IndexManifestEntry entry = - new IndexManifestEntry(FileKind.ADD, BinaryRow.singleColumn(1), 0, range); - List centroid = centroidModel("model", "ivf-pq"); - TestAppendFileStore store = - TestAppendFileStore.createAppendStore(directory, new HashMap<>(), "file"); - IndexManifestFile manifest = - new IndexManifestFile.Factory( - store.fileIO(), - FileFormat.manifestFormat(store.options()), - "zstd", - store.pathFactory(), - null) - .create(); - IndexManifestFileHandler handler = - new IndexManifestFileHandler(manifest, BucketMode.BUCKET_UNAWARE); - String priorRange = handler.write(null, Collections.singletonList(entry)); - String priorCentroid = handler.write(null, centroid); - assertThatThrownBy(() -> handler.write(priorRange, centroid)) - .hasMessageContaining("Drop the previous index"); - assertThatThrownBy(() -> handler.write(priorCentroid, Collections.singletonList(entry))) - .hasMessageContaining("Drop the previous index"); - } - - private static List centroidModel(String id, String type) { - List entries = new ArrayList<>(); - for (int centroid = -1; centroid < 2; centroid++) { - byte[] metadata = - centroid == -1 - ? VectorIndexShardMeta.routing( - id, 2, Collections.singletonList(new Range(10, 100))) - .serialize() - : VectorIndexShardMeta.centroid(id, centroid).serialize(); - IndexFileMeta file = - new IndexFileMeta( - type + "-centroid", - id + "-" + centroid, - 100, - centroid == -1 ? 0 : 10, - null, - null, - new GlobalIndexMeta(10, 100, 1, null, metadata)); - entries.add(new IndexManifestEntry(FileKind.ADD, BinaryRow.singleColumn(0), 0, file)); - } - return entries; - } -} diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java index a2c3cfeb4994..53dc59376231 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorGlobalIndexTest.java @@ -414,11 +414,5 @@ public PositionOutputStream newOutputStream(String name) } finally { executor.shutdownNow(); } - - List incomplete = new ArrayList<>(metas); - incomplete.remove(incomplete.size() - 1); - assertThatThrownBy(() -> indexer.createReader(fileReader, incomplete, 256, null, executor)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Incomplete"); } } diff --git a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java index 44b0b2db3de2..875402d476fa 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/vector/index/CentroidVectorIndexUtilsTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -66,8 +65,6 @@ void indexTypeSelectsTheMetadataCodec() { @Test void testSparseShardsDoNotClaimEnvelopeCoverage() { List files = model("a"); - assertThat(CentroidVectorIndexUtils.validateModel(files)) - .containsExactly(new Range(10, 19), new Range(1000, 1009)); assertThat(CentroidVectorIndexUtils.coverage(files.get(0))) .containsExactly(new Range(10, 19), new Range(1000, 1009)); assertThat(CentroidVectorIndexUtils.coverage(files.get(1))).isEmpty(); @@ -75,25 +72,6 @@ void testSparseShardsDoNotClaimEnvelopeCoverage() { assertThat(CentroidVectorIndexUtils.centroidBased(file("{}", false))).isFalse(); } - @Test - void testIncompleteMixedAndMismatchedModelsAreRejected() { - List complete = model("a"); - assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(complete.subList(0, 2))) - .hasMessageContaining("Incomplete"); - List mixed = new ArrayList<>(complete); - mixed.add(file("{}", false)); - assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mixed)) - .hasMessageContaining("different centroid index types"); - List mismatch = new ArrayList<>(complete); - mismatch.set(2, model("b").get(2)); - assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(mismatch)) - .hasMessageContaining("different models"); - List duplicate = new ArrayList<>(complete); - duplicate.add(complete.get(1)); - assertThatThrownBy(() -> CentroidVectorIndexUtils.validateModel(duplicate)) - .hasMessageContaining("Duplicate"); - } - private static List model(String id) { return Arrays.asList( file(