diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactGlobalIndexDropper.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactGlobalIndexDropper.java deleted file mode 100644 index 810398454d87..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactGlobalIndexDropper.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.append.dataevolution; - -import org.apache.paimon.Snapshot; -import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.index.IndexFileMeta; -import org.apache.paimon.io.CompactIncrement; -import org.apache.paimon.io.DataFileMeta; -import org.apache.paimon.io.DataIncrement; -import org.apache.paimon.manifest.IndexManifestEntry; -import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.CommitMessage; -import org.apache.paimon.table.sink.CommitMessageImpl; -import org.apache.paimon.utils.Pair; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; -import static org.apache.paimon.types.VectorType.isVectorStoreFile; - -/** Drops partition-level global indexes invalidated by materialized data-evolution compaction. */ -class DataEvolutionCompactGlobalIndexDropper { - - private final FileStoreTable table; - private final Snapshot snapshot; - - DataEvolutionCompactGlobalIndexDropper(FileStoreTable table, Snapshot snapshot) { - this.table = table; - this.snapshot = snapshot; - } - - List dropGlobalIndexes(List messages) { - Set partitions = materializedPartitions(messages); - if (partitions.isEmpty()) { - return Collections.emptyList(); - } - - // Scan at the latest snapshot, not the compaction's base: a global index - // built concurrently and committed after this compaction planned still - // references pre-materialization row-ids and must be dropped too. - Snapshot scanSnapshot = table.snapshotManager().latestSnapshot(); - if (scanSnapshot == null) { - scanSnapshot = snapshot; - } - - Map, List> deletedIndexes = new LinkedHashMap<>(); - for (IndexManifestEntry entry : - table.store() - .newIndexFileHandler() - .scan( - scanSnapshot, - e -> - partitions.contains(e.partition()) - && e.indexFile().globalIndexMeta() != null)) { - deletedIndexes - .computeIfAbsent( - Pair.of(entry.partition(), entry.bucket()), k -> new ArrayList<>()) - .add(entry.indexFile()); - } - - List result = new ArrayList<>(); - for (Map.Entry, List> entry : - deletedIndexes.entrySet()) { - result.add( - toCommitMessage( - entry.getKey().getLeft(), entry.getKey().getRight(), entry.getValue())); - } - return result; - } - - private Set materializedPartitions(List messages) { - Set result = new LinkedHashSet<>(); - for (CommitMessage message : messages) { - CommitMessageImpl commitMessage = (CommitMessageImpl) message; - CompactIncrement compactIncrement = commitMessage.compactIncrement(); - if (normalFiles(compactIncrement.compactBefore()).isEmpty()) { - continue; - } - // only involve partitions with deletions materialization - if (compactIncrement.compactAfter().stream() - .allMatch(file -> file.firstRowId() == null)) { - result.add(commitMessage.partition()); - } - } - return result; - } - - private List normalFiles(List files) { - return files.stream() - .filter(file -> !isBlobFile(file.fileName()) && !isVectorStoreFile(file.fileName())) - .collect(Collectors.toList()); - } - - private CommitMessage toCommitMessage( - BinaryRow partition, int bucket, List deletedIndexFiles) { - return new CommitMessageImpl( - partition, - bucket, - null, - DataIncrement.emptyIncrement(), - new CompactIncrement( - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList(), - deletedIndexFiles)); - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java index 463b6863bf13..238f98e2b2e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactionCommitPreparation.java @@ -25,7 +25,12 @@ import java.util.ArrayList; import java.util.List; -/** Prepares extra commit messages for data-evolution compaction before committing. */ +/** + * Prepares deletion-vector commit messages for data-evolution compaction. + * + *

Global indexes invalidated by materialization are removed by the core commit using the latest + * snapshot on each attempt. + */ public class DataEvolutionCompactionCommitPreparation { private final FileStoreTable table; @@ -41,9 +46,6 @@ public List prepare(List messages) { result.addAll( new DataEvolutionCompactDeletionVectorRewriter(table, snapshot) .rewriteDeletionVectors(messages)); - result.addAll( - new DataEvolutionCompactGlobalIndexDropper(table, snapshot) - .dropGlobalIndexes(messages)); return result; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 1d976c9e4862..0c0bd9524ab3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -1029,18 +1029,7 @@ CommitChangesProvider dataEvolutionChangesProvider( } return latestSnapshot -> { - List indexFiles = - changes.indexFiles.stream() - // Replace global-index deletions prepared against an older snapshot. - .filter( - entry -> - entry.kind() != FileKind.DELETE - || entry.indexFile().globalIndexMeta() == null - || !materializedBuckets.contains( - Pair.of( - entry.partition(), - entry.bucket()))) - .collect(Collectors.toList()); + List indexFiles = new ArrayList<>(changes.indexFiles); // This provider is invoked again after every optimistic-commit conflict. Scanning the // latest snapshot here guarantees that an index committed concurrently is either diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index 5cc4309339f7..2d5ab91edd8f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -41,6 +41,7 @@ import org.apache.paimon.io.DataIncrement; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.manifest.IndexManifestFile; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; @@ -1007,15 +1008,18 @@ public void testIndexFiles() throws Exception { public void testMaterializedCompactionOnlyRefreshesGlobalIndexInSameBucket() throws Exception { TestFileStore store = createStore(false, 2); BinaryRow partition = gen.getPartition(gen.next()); - IndexManifestEntry materializedBucketDelete = - globalIndexDeleteEntry(partition, 0, "materialized-bucket-index"); - IndexManifestEntry otherBucketDelete = - globalIndexDeleteEntry(partition, 1, "other-bucket-index"); + IndexManifestEntry materializedBucketIndex = + globalIndexEntry(partition, 0, "materialized-bucket-index"); + IndexManifestEntry otherBucketIndex = globalIndexEntry(partition, 1, "other-bucket-index"); + IndexManifestFile manifestFile = store.indexManifestFileFactory().create(); + String manifest = + manifestFile.writeWithoutRolling( + Arrays.asList(materializedBucketIndex, otherBucketIndex)); + Snapshot snapshot = mock(Snapshot.class); + when(snapshot.indexManifest()).thenReturn(manifest); CommitChanges changes = new CommitChanges( - Collections.emptyList(), - Collections.emptyList(), - Arrays.asList(materializedBucketDelete, otherBucketDelete)); + Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); try (FileStoreCommitImpl commit = store.newCommit()) { CommitChanges refreshed = @@ -1023,18 +1027,23 @@ public void testMaterializedCompactionOnlyRefreshesGlobalIndexInSameBucket() thr changes, Snapshot.CommitKind.COMPACT, Collections.singleton(Pair.of(partition, 0))) - .provide(null); + .provide(snapshot); - assertThat(refreshed.indexFiles).containsExactly(otherBucketDelete); + assertThat(refreshed.indexFiles) + .containsExactly(materializedBucketIndex.toDeleteEntry()); + String refreshedManifest = + manifestFile.writeIndexFiles( + manifest, refreshed.indexFiles, store.bucketMode()); + assertThat(manifestFile.read(refreshedManifest)).containsExactly(otherBucketIndex); } } - private static IndexManifestEntry globalIndexDeleteEntry( + private static IndexManifestEntry globalIndexEntry( BinaryRow partition, int bucket, String fileName) { IndexFileMeta file = new IndexFileMeta( "btree", fileName, 1, 1, new GlobalIndexMeta(0, 0, 0, null, null), null); - return new IndexManifestEntry(FileKind.DELETE, partition, bucket, file); + return new IndexManifestEntry(FileKind.ADD, partition, bucket, file); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java index 66d1bac85d6e..ee561d4abe83 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java @@ -40,6 +40,7 @@ import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; import org.apache.paimon.format.blob.BlobFileFormat; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; import org.apache.paimon.fs.local.LocalFileIO; @@ -91,6 +92,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -104,6 +106,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.catchThrowable; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -1039,6 +1044,86 @@ public void testMaterializeDeletionVectors() throws Exception { .doesNotContainAnyElementsOf(oldAnchorFiles); } + @ParameterizedTest + @ValueSource(strings = {"before", "after", "retry", "fully-deleted"}) + public void testMaterializeInvalidatesGlobalIndexesAtCommit(String indexTiming) + throws Exception { + FileStoreTable table = createPartitionedReassignTable("materialize_global_indexes", false); + writePartitionRows(table, "a", 0, 1, 2); + writePartitionRows(table, "b", 3, 4, 5); + BinaryRow partitionA = partition(table, "a"); + boolean fullyDeleted = "fully-deleted".equals(indexTiming); + commitDeletionVectors( + table, + partitionA, + Collections.singletonList( + new DvSpec( + new Range(0, 2), + fullyDeleted ? new long[] {0, 1, 2} : new long[] {1}))); + commitGlobalIndex( + table, partition(table, "b"), globalIndexFile("untouched-index", new Range(3, 5))); + IndexFileMeta oldIndex = globalIndexFile("old-index", new Range(0, 2)); + if (!"after".equals(indexTiming)) { + commitGlobalIndex(table, partitionA, oldIndex); + } + List rowsBefore = readPartitionedRows(table); + Snapshot snapshot = table.latestSnapshot().get(); + List messages = prepareMaterializeDeletionVectors(table, snapshot, null); + assertThat( + messages.stream() + .flatMap( + message -> + ((CommitMessageImpl) message) + .compactIncrement().deletedIndexFiles() + .stream()) + .filter(file -> file.globalIndexMeta() != null)) + .isEmpty(); + if ("after".equals(indexTiming)) { + commitGlobalIndex(table, partitionA, globalIndexFile("late-index", new Range(0, 2))); + } + + AtomicInteger publicationAttempts = new AtomicInteger(); + FileStoreTable commitTable = table; + if ("retry".equals(indexTiming)) { + FileIO fileIO = spy(table.fileIO()); + doAnswer( + invocation -> { + Path path = invocation.getArgument(0); + if (path.getName().startsWith("snapshot-") + && publicationAttempts.getAndIncrement() == 0) { + // Replace old-index after this attempt generated its deletions. + // The retry must regenerate DELETE concurrent-index. + commit( + table, + Collections.singletonList( + indexCompactMessage( + partitionA, + oldIndex, + globalIndexFile( + "concurrent-index", + new Range(0, 2))))); + } + return invocation.callRealMethod(); + }) + .when(fileIO) + .tryToWriteAtomic(any(Path.class), anyString()); + commitTable = + new AppendOnlyFileStoreTable( + fileIO, table.location(), table.schema(), table.catalogEnvironment()); + } + commitMaterializeDeletionVectors( + commitTable, snapshot, messages, "test-materialize-global-index"); + + assertThat(liveGlobalIndexFileNames(table)).containsExactly("untouched-index"); + assertThat(readPartitionedRows(table)).containsExactlyInAnyOrderElementsOf(rowsBefore); + if ("retry".equals(indexTiming)) { + assertThat(publicationAttempts.get()).isEqualTo(2); + } + if (fullyDeleted) { + assertThat(currentDataFiles(table, partitionA)).isEmpty(); + } + } + @Test public void testStaleMaterializeRejectedAfterConcurrentUpdate() throws Exception { createTableDefault();