Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,12 @@
import java.util.ArrayList;
import java.util.List;

/** Prepares extra commit messages for data-evolution compaction before committing. */
/**
* Prepares deletion-vector commit messages for data-evolution compaction.
*
* <p>Global indexes invalidated by materialization are removed by the core commit using the latest
* snapshot on each attempt.
*/
public class DataEvolutionCompactionCommitPreparation {

private final FileStoreTable table;
Expand All @@ -41,9 +46,6 @@ public List<CommitMessage> prepare(List<CommitMessage> messages) {
result.addAll(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that DataEvolutionCompactGlobalIndexDropper is removed and preparation only emits DELETION_VECTORS_INDEX changes, nothing prepares FileKind.DELETE global-index entries against an older snapshot anymore (DataEvolutionCompactDeletionVectorRewriter#checkNoCompactIndexChanges also rejects pre-existing compact index changes).

Should we also clean up the leftover filter in FileStoreCommitImpl#dataEvolutionChangesProvider (lines 1031–1043, // Replace global-index deletions prepared against an older snapshot.) to new ArrayList<>(changes.indexFiles) and update FileStoreCommitTest#testMaterializedCompactionOnlyRefreshesGlobalIndexInSameBucket accordingly?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed the obsolete filtering. Updated the test to verify that only global indexes in the materialized bucket are deleted, while indexes in other buckets remain intact.

new DataEvolutionCompactDeletionVectorRewriter(table, snapshot)
.rewriteDeletionVectors(messages));
result.addAll(
new DataEvolutionCompactGlobalIndexDropper(table, snapshot)
.dropGlobalIndexes(messages));
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1029,18 +1029,7 @@ CommitChangesProvider dataEvolutionChangesProvider(
}

return latestSnapshot -> {
List<IndexManifestEntry> indexFiles =
changes.indexFiles.stream()
// Replace global-index deletions prepared against an older snapshot.
.filter(
entry ->
entry.kind() != FileKind.DELETE
|| entry.indexFile().globalIndexMeta() == null
|| !materializedBuckets.contains(
Pair.of(
entry.partition(),
entry.bucket())))
.collect(Collectors.toList());
List<IndexManifestEntry> indexFiles = new ArrayList<>(changes.indexFiles);

// This provider is invoked again after every optimistic-commit conflict. Scanning the
// latest snapshot here guarantees that an index committed concurrently is either
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.paimon.io.DataIncrement;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.manifest.IndexManifestFile;
import org.apache.paimon.manifest.ManifestCommittable;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
Expand Down Expand Up @@ -1007,34 +1008,42 @@ public void testIndexFiles() throws Exception {
public void testMaterializedCompactionOnlyRefreshesGlobalIndexInSameBucket() throws Exception {
TestFileStore store = createStore(false, 2);
BinaryRow partition = gen.getPartition(gen.next());
IndexManifestEntry materializedBucketDelete =
globalIndexDeleteEntry(partition, 0, "materialized-bucket-index");
IndexManifestEntry otherBucketDelete =
globalIndexDeleteEntry(partition, 1, "other-bucket-index");
IndexManifestEntry materializedBucketIndex =
globalIndexEntry(partition, 0, "materialized-bucket-index");
IndexManifestEntry otherBucketIndex = globalIndexEntry(partition, 1, "other-bucket-index");
IndexManifestFile manifestFile = store.indexManifestFileFactory().create();
String manifest =
manifestFile.writeWithoutRolling(
Arrays.asList(materializedBucketIndex, otherBucketIndex));
Snapshot snapshot = mock(Snapshot.class);
when(snapshot.indexManifest()).thenReturn(manifest);
CommitChanges changes =
new CommitChanges(
Collections.emptyList(),
Collections.emptyList(),
Arrays.asList(materializedBucketDelete, otherBucketDelete));
Collections.emptyList(), Collections.emptyList(), Collections.emptyList());

try (FileStoreCommitImpl commit = store.newCommit()) {
CommitChanges refreshed =
commit.dataEvolutionChangesProvider(
changes,
Snapshot.CommitKind.COMPACT,
Collections.singleton(Pair.of(partition, 0)))
.provide(null);
.provide(snapshot);

assertThat(refreshed.indexFiles).containsExactly(otherBucketDelete);
assertThat(refreshed.indexFiles)
.containsExactly(materializedBucketIndex.toDeleteEntry());
String refreshedManifest =
manifestFile.writeIndexFiles(
manifest, refreshed.indexFiles, store.bucketMode());
assertThat(manifestFile.read(refreshedManifest)).containsExactly(otherBucketIndex);
}
}

private static IndexManifestEntry globalIndexDeleteEntry(
private static IndexManifestEntry globalIndexEntry(
BinaryRow partition, int bucket, String fileName) {
IndexFileMeta file =
new IndexFileMeta(
"btree", fileName, 1, 1, new GlobalIndexMeta(0, 0, 0, null, null), null);
return new IndexManifestEntry(FileKind.DELETE, partition, bucket, file);
return new IndexManifestEntry(FileKind.ADD, partition, bucket, file);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
import org.apache.paimon.format.blob.BlobFileFormat;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.fs.local.LocalFileIO;
Expand Down Expand Up @@ -91,6 +92,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

Expand All @@ -104,6 +106,9 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.catchThrowable;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
Expand Down Expand Up @@ -1039,6 +1044,86 @@ public void testMaterializeDeletionVectors() throws Exception {
.doesNotContainAnyElementsOf(oldAnchorFiles);
}

@ParameterizedTest
@ValueSource(strings = {"before", "after", "retry", "fully-deleted"})
public void testMaterializeInvalidatesGlobalIndexesAtCommit(String indexTiming)
throws Exception {
FileStoreTable table = createPartitionedReassignTable("materialize_global_indexes", false);
writePartitionRows(table, "a", 0, 1, 2);
writePartitionRows(table, "b", 3, 4, 5);
BinaryRow partitionA = partition(table, "a");
boolean fullyDeleted = "fully-deleted".equals(indexTiming);
commitDeletionVectors(
table,
partitionA,
Collections.singletonList(
new DvSpec(
new Range(0, 2),
fullyDeleted ? new long[] {0, 1, 2} : new long[] {1})));
commitGlobalIndex(
table, partition(table, "b"), globalIndexFile("untouched-index", new Range(3, 5)));
IndexFileMeta oldIndex = globalIndexFile("old-index", new Range(0, 2));
if (!"after".equals(indexTiming)) {
commitGlobalIndex(table, partitionA, oldIndex);
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the "retry" case, partitionA currently has no global index when snapshot is taken ("old-index" is only committed for "before" and "fully-deleted"). As a result:

  1. The assertion at lines 1071–1079 is vacuous for "retry" because partitionA has no global index at snapshot.
  2. Commit attempt 1 does not produce any global-index DELETE entry for partitionA, so the retry does not verify that attempt 1's DELETE entries are discarded/rebuilt cleanly on attempt 2 without causing a duplicate or missing-file IllegalStateException in GlobalIndexCombiner#combine.

If we also commit "old-index" before preparation when "retry".equals(indexTiming) and compact it into "concurrent-index" inside tryToWriteAtomic (via indexCompactMessage(partitionA, oldIndex, globalIndexFile("concurrent-index", new Range(0, 2)))), the "retry" case will verify both that attempt 1's stale DELETE old-index is dropped and that "concurrent-index" from the retried snapshot is invalidated.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated the retry case to start with old-index and replace it with concurrent-index during the first commit attempt. It now verifies that the retry regenerates the deletions against the latest snapshot.

List<String> rowsBefore = readPartitionedRows(table);
Snapshot snapshot = table.latestSnapshot().get();
List<CommitMessage> messages = prepareMaterializeDeletionVectors(table, snapshot, null);
assertThat(
messages.stream()
.flatMap(
message ->
((CommitMessageImpl) message)
.compactIncrement().deletedIndexFiles()
.stream())
.filter(file -> file.globalIndexMeta() != null))
.isEmpty();
if ("after".equals(indexTiming)) {
commitGlobalIndex(table, partitionA, globalIndexFile("late-index", new Range(0, 2)));
}

AtomicInteger publicationAttempts = new AtomicInteger();
FileStoreTable commitTable = table;
if ("retry".equals(indexTiming)) {
FileIO fileIO = spy(table.fileIO());
doAnswer(
invocation -> {
Path path = invocation.getArgument(0);
if (path.getName().startsWith("snapshot-")
&& publicationAttempts.getAndIncrement() == 0) {
// Replace old-index after this attempt generated its deletions.
// The retry must regenerate DELETE concurrent-index.
commit(
table,
Collections.singletonList(
indexCompactMessage(
partitionA,
oldIndex,
globalIndexFile(
"concurrent-index",
new Range(0, 2)))));
}
return invocation.callRealMethod();
})
.when(fileIO)
.tryToWriteAtomic(any(Path.class), anyString());
commitTable =
new AppendOnlyFileStoreTable(
fileIO, table.location(), table.schema(), table.catalogEnvironment());
}
commitMaterializeDeletionVectors(
commitTable, snapshot, messages, "test-materialize-global-index");

assertThat(liveGlobalIndexFileNames(table)).containsExactly("untouched-index");
assertThat(readPartitionedRows(table)).containsExactlyInAnyOrderElementsOf(rowsBefore);
if ("retry".equals(indexTiming)) {
assertThat(publicationAttempts.get()).isEqualTo(2);
}
if (fullyDeleted) {
assertThat(currentDataFiles(table, partitionA)).isEmpty();
}
}

@Test
public void testStaleMaterializeRejectedAfterConcurrentUpdate() throws Exception {
createTableDefault();
Expand Down
Loading