Repository navigation
[core] Remove redundant global-index preparation during DV materialization #11252
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,6 +40,7 @@ | |
| import org.apache.paimon.deletionvectors.DeletionVector; | ||
| import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer; | ||
| import org.apache.paimon.format.blob.BlobFileFormat; | ||
| import org.apache.paimon.fs.FileIO; | ||
| import org.apache.paimon.fs.Path; | ||
| import org.apache.paimon.fs.SeekableInputStream; | ||
| import org.apache.paimon.fs.local.LocalFileIO; | ||
|
|
@@ -91,6 +92,7 @@ | |
| import java.util.Map; | ||
| import java.util.Set; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.IntStream; | ||
|
|
||
|
|
@@ -104,6 +106,9 @@ | |
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.assertj.core.api.Assertions.assertThatThrownBy; | ||
| import static org.assertj.core.api.Assertions.catchThrowable; | ||
| import static org.mockito.ArgumentMatchers.any; | ||
| import static org.mockito.ArgumentMatchers.anyString; | ||
| import static org.mockito.Mockito.doAnswer; | ||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.never; | ||
| import static org.mockito.Mockito.spy; | ||
|
|
@@ -1039,6 +1044,86 @@ public void testMaterializeDeletionVectors() throws Exception { | |
| .doesNotContainAnyElementsOf(oldAnchorFiles); | ||
| } | ||
|
|
||
| @ParameterizedTest | ||
| @ValueSource(strings = {"before", "after", "retry", "fully-deleted"}) | ||
| public void testMaterializeInvalidatesGlobalIndexesAtCommit(String indexTiming) | ||
| throws Exception { | ||
| FileStoreTable table = createPartitionedReassignTable("materialize_global_indexes", false); | ||
| writePartitionRows(table, "a", 0, 1, 2); | ||
| writePartitionRows(table, "b", 3, 4, 5); | ||
| BinaryRow partitionA = partition(table, "a"); | ||
| boolean fullyDeleted = "fully-deleted".equals(indexTiming); | ||
| commitDeletionVectors( | ||
| table, | ||
| partitionA, | ||
| Collections.singletonList( | ||
| new DvSpec( | ||
| new Range(0, 2), | ||
| fullyDeleted ? new long[] {0, 1, 2} : new long[] {1}))); | ||
| commitGlobalIndex( | ||
| table, partition(table, "b"), globalIndexFile("untouched-index", new Range(3, 5))); | ||
| IndexFileMeta oldIndex = globalIndexFile("old-index", new Range(0, 2)); | ||
| if (!"after".equals(indexTiming)) { | ||
| commitGlobalIndex(table, partitionA, oldIndex); | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the
If we also commit
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Updated the retry case to start with old-index and replace it with concurrent-index during the first commit attempt. It now verifies that the retry regenerates the deletions against the latest snapshot. |
||
| List<String> rowsBefore = readPartitionedRows(table); | ||
| Snapshot snapshot = table.latestSnapshot().get(); | ||
| List<CommitMessage> messages = prepareMaterializeDeletionVectors(table, snapshot, null); | ||
| assertThat( | ||
| messages.stream() | ||
| .flatMap( | ||
| message -> | ||
| ((CommitMessageImpl) message) | ||
| .compactIncrement().deletedIndexFiles() | ||
| .stream()) | ||
| .filter(file -> file.globalIndexMeta() != null)) | ||
| .isEmpty(); | ||
| if ("after".equals(indexTiming)) { | ||
| commitGlobalIndex(table, partitionA, globalIndexFile("late-index", new Range(0, 2))); | ||
| } | ||
|
|
||
| AtomicInteger publicationAttempts = new AtomicInteger(); | ||
| FileStoreTable commitTable = table; | ||
| if ("retry".equals(indexTiming)) { | ||
| FileIO fileIO = spy(table.fileIO()); | ||
| doAnswer( | ||
| invocation -> { | ||
| Path path = invocation.getArgument(0); | ||
| if (path.getName().startsWith("snapshot-") | ||
| && publicationAttempts.getAndIncrement() == 0) { | ||
| // Replace old-index after this attempt generated its deletions. | ||
| // The retry must regenerate DELETE concurrent-index. | ||
| commit( | ||
| table, | ||
| Collections.singletonList( | ||
| indexCompactMessage( | ||
| partitionA, | ||
| oldIndex, | ||
| globalIndexFile( | ||
| "concurrent-index", | ||
| new Range(0, 2))))); | ||
| } | ||
| return invocation.callRealMethod(); | ||
| }) | ||
| .when(fileIO) | ||
| .tryToWriteAtomic(any(Path.class), anyString()); | ||
| commitTable = | ||
| new AppendOnlyFileStoreTable( | ||
| fileIO, table.location(), table.schema(), table.catalogEnvironment()); | ||
| } | ||
| commitMaterializeDeletionVectors( | ||
| commitTable, snapshot, messages, "test-materialize-global-index"); | ||
|
|
||
| assertThat(liveGlobalIndexFileNames(table)).containsExactly("untouched-index"); | ||
| assertThat(readPartitionedRows(table)).containsExactlyInAnyOrderElementsOf(rowsBefore); | ||
| if ("retry".equals(indexTiming)) { | ||
| assertThat(publicationAttempts.get()).isEqualTo(2); | ||
| } | ||
| if (fullyDeleted) { | ||
| assertThat(currentDataFiles(table, partitionA)).isEmpty(); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void testStaleMaterializeRejectedAfterConcurrentUpdate() throws Exception { | ||
| createTableDefault(); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Now that
DataEvolutionCompactGlobalIndexDropperis removed and preparation only emitsDELETION_VECTORS_INDEXchanges, nothing preparesFileKind.DELETEglobal-index entries against an older snapshot anymore (DataEvolutionCompactDeletionVectorRewriter#checkNoCompactIndexChangesalso rejects pre-existing compact index changes).Should we also clean up the leftover filter in
FileStoreCommitImpl#dataEvolutionChangesProvider(lines 1031–1043,// Replace global-index deletions prepared against an older snapshot.) tonew ArrayList<>(changes.indexFiles)and updateFileStoreCommitTest#testMaterializedCompactionOnlyRefreshesGlobalIndexInSameBucketaccordingly?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Removed the obsolete filtering. Updated the test to verify that only global indexes in the materialized bucket are deleted, while indexes in other buckets remain intact.