Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ abstract class PaimonBaseScanBuilder
if (predicate == null) {
postScan.append(filter)
} else if (
coreOptions.queryAuthEnabled() && filter.isInstanceOf[IsNotNull] &&
predicate.visit(partitionPredicateVisitor)
filter.isInstanceOf[IsNotNull] && predicate.visit(partitionPredicateVisitor) &&
PaimonBaseScan.hasAuthorizedBranch(table)
) {
// Spark infers null checks for joins. Masks can null partition keys, so evaluate
// these checks on visible rows instead of using them for physical pruning.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import org.apache.paimon.spark.metric.SparkMetricRegistry
import org.apache.paimon.spark.read.{BaseScan, BatchReadTagCleanupListener, PaimonStatistics, PaimonSupportsRuntimeFiltering, SparkHybridSearchBuilderImpl, SparkVectorSearchBuilderImpl}
import org.apache.paimon.spark.sources.PaimonMicroBatchStream
import org.apache.paimon.spark.util.OptionUtils
import org.apache.paimon.table.{DataTable, FileStoreTable, InnerTable}
import org.apache.paimon.table.source.{InnerTableScan, Split}
import org.apache.paimon.table.{DataTable, FallbackReadFileStoreTable, FileStoreTable, InnerTable, Table}
import org.apache.paimon.table.source.{InnerTableScan, Split, Splits}

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.SQLConfHelper
Expand Down Expand Up @@ -91,7 +91,28 @@ abstract class PaimonBaseScan(table: InnerTable)
}

override def filterAttributes(): Array[NamedReference] = {
if (postponeMergeOnRead.enabled) Array.empty else super.filterAttributes()
if (postponeMergeOnRead.enabled) {
return Array.empty
}

val candidates = super.filterAttributes()
table match {
// Fallback and chain scans can read branches with different authorization rules.
case t: FallbackReadFileStoreTable if PaimonBaseScan.hasAuthorizedBranch(t) => Array.empty
case _: FileStoreTable if coreOptions.queryAuthEnabled() && candidates.nonEmpty =>
// An ordinary table plan attaches the same authorization result to every split.
inputSplits.headOption match {
case Some(split) =>
val auth = Splits.authResult(split)
if (auth == null) candidates
else {
val masks = auth.extractColumnMasking()
candidates.filterNot(field => masks.containsKey(field.fieldNames().head))
}
case None => Array.empty
}
case _ => candidates
}
}

private def evalVectorSearch(): GlobalIndexResult = {
Expand Down Expand Up @@ -208,6 +229,13 @@ abstract class PaimonBaseScan(table: InnerTable)

object PaimonBaseScan {

private[spark] def hasAuthorizedBranch(table: Table): Boolean = table match {
case t: FallbackReadFileStoreTable =>
hasAuthorizedBranch(t.wrapped()) || hasAuthorizedBranch(t.other())
case t: FileStoreTable => t.coreOptions().queryAuthEnabled()
case _ => CoreOptions.fromMap(table.options()).queryAuthEnabled()
}

private[spark] def evalVectorSearch(
table: InnerTable,
vectorSearch: VectorSearch,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,9 @@ abstract class PaimonBaseScanBuilder
case None =>
converter.convert(predicate) match {
case Some(paimonPredicate)
if coreOptions.queryAuthEnabled() && predicate.name() == "IS_NOT_NULL" &&
paimonPredicate.visit(partitionPredicateVisitor) =>
if predicate.name() == "IS_NOT_NULL" &&
paimonPredicate.visit(partitionPredicateVisitor) &&
PaimonBaseScan.hasAuthorizedBranch(table) =>
// Spark infers null checks for joins. Masks can null partition keys, so evaluate
// these checks on visible rows instead of using them for physical pruning.
postScan.append(predicate)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,15 @@
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.JsonSerdeUtil;

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.Table;
import org.apache.spark.sql.connector.expressions.NamedReference;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
Expand Down Expand Up @@ -229,6 +232,100 @@
}
}

@Test
public void testDynamicPartitionPruningOnMaskedJoinKey(@TempDir Path warehouse) {
try (SparkSession spark = createSession(warehouse)) {

Check failure on line 237 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 237 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 237 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 237 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown
createTable(spark, false);
setPolicy(spark, false, "constant", "id,region,dt");
spark.sql(
"CREATE TABLE spark_test.default.D (region STRING, dt STRING, k INT) "
+ "USING paimon TBLPROPERTIES ('bucket'='-1')");
spark.sql("INSERT INTO spark_test.default.D VALUES ('CN','***',1),('US','other',0)");
spark.conf().set("spark.sql.optimizer.dynamicPartitionPruning.enabled", true);

String join = "FROM spark_test.default.T l JOIN spark_test.default.D r ON l.dt = r.dt ";
assertRows(
spark,
"SELECT l.id " + join + "WHERE r.k = 1",
Arrays.asList(RowFactory.create(1), RowFactory.create(2)));

// The fact scan receives a runtime filter on region even though dt is masked.
Dataset<Row> result =
spark.sql(
"SELECT /*+ BROADCAST(r) */ l.id "
+ join
+ "AND l.region = r.region WHERE r.k = 1");
assertThat(result.collectAsList()).containsExactly(RowFactory.create(1));
assertThat(result.queryExecution().executedPlan().toString())
.contains("PaimonScan: [T], RuntimePartitionFilters: [Equal(region, CN)]");
}
}

@ParameterizedTest(name = "chain={0}")
@ValueSource(booleans = {false, true})
public void testRuntimeFilteringWithAuthorizedBranch(boolean chain, @TempDir Path warehouse)
throws Exception {
try (SparkSession spark = createSession(warehouse)) {
createTable(spark, chain);
if (!chain) {
spark.sql("CALL spark_test.sys.create_branch('default.T', 'backup')");
spark.sql(
"ALTER TABLE spark_test.default.T SET TBLPROPERTIES ('scan.fallback-branch'='backup')");
}
SparkCatalog catalog =
(SparkCatalog) spark.sessionState().catalogManager().catalog("spark_test");
Identifier id = Identifier.of(new String[] {"default"}, "T");
assertThat(runtimeFilterAttributes(catalog, id)).hasSize(2);

// Enable authorization only on the secondary branch, nested one level deeper for delta.
String branch = chain ? "delta" : "backup";
spark.sql(
"ALTER TABLE spark_test.default.`T$branch_"
+ branch
+ "` SET TBLPROPERTIES ('query-auth.enabled'='true', "
+ "'test.auth-mode'='constant', 'test.allowed'='id,dt,region')");
assertThat(runtimeFilterAttributes(catalog, id)).isEmpty();
}
}

@Test
public void testJoinOnNullablePartitionMaskedOnlyOnBackup(@TempDir Path warehouse) {
try (SparkSession spark = createSession(warehouse)) {

Check failure on line 293 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 293 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 293 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown

Check failure on line 293 in paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java

View workflow job for this annotation

GitHub Actions / Java / Spark 4 / Scala 2.13

unreported exception java.io.IOException; must be caught or declared to be thrown
spark.sql(
"CREATE TABLE spark_test.default.T (id INT, dt STRING) USING paimon "
+ "PARTITIONED BY (dt) TBLPROPERTIES ('bucket'='-1')");
spark.sql("CALL spark_test.sys.create_branch('default.T', 'backup')");
spark.sql("INSERT INTO spark_test.default.T VALUES (1, 'A')");
spark.sql("INSERT INTO spark_test.default.`T$branch_backup` VALUES (2, NULL)");
spark.sql(
"ALTER TABLE spark_test.default.`T$branch_backup` SET TBLPROPERTIES "
+ "('query-auth.enabled'='true')");
spark.sql(
"ALTER TABLE spark_test.default.T SET TBLPROPERTIES "
+ "('scan.fallback-branch'='backup', 'test.auth-mode'='constant', "
+ "'test.allowed'='id,dt')");
spark.sql(
"CREATE TABLE spark_test.default.D (dt STRING, k INT) USING paimon "
+ "TBLPROPERTIES ('bucket'='-1')");
spark.sql("INSERT INTO spark_test.default.D VALUES ('***', 1), ('other', 0)");
spark.conf().set("spark.sql.optimizer.dynamicPartitionPruning.enabled", true);

// The backup row's stored NULL becomes '***' before the inferred IS NOT NULL check.
assertRows(
spark,
"SELECT l.id FROM spark_test.default.T l JOIN spark_test.default.D r "
+ "ON l.dt = r.dt WHERE r.k = 1",
Collections.singletonList(RowFactory.create(2)));
}
}

private static NamedReference[] runtimeFilterAttributes(SparkCatalog catalog, Identifier id)
throws NoSuchTableException {
SparkTable table = (SparkTable) catalog.loadTable(id);
return ((PaimonBaseScan) table.newScanBuilder(CaseInsensitiveStringMap.empty()).build())
.filterAttributes();
}

@ParameterizedTest(name = "chain={0}, nestedPruning={1}")
@CsvSource({"true,true", "true,false", "false,true", "false,false"})
public void testPartitionMetadataAuthorizationWithNestedSchemaPruning(
Expand Down Expand Up @@ -600,10 +697,11 @@
throws NoSuchTableException {
SparkTable sparkTable = (SparkTable) super.loadSparkTable(identifier, extraOptions);
FileStoreTable table = (FileStoreTable) sparkTable.table();
if (!table.coreOptions().queryAuthEnabled()) {
// The catalog can supply rules even when only a secondary branch enables them.
String mode = table.options().get("test.auth-mode");
if (mode == null || "disabled".equals(mode)) {
return sparkTable;
}
String mode = table.options().get("test.auth-mode");
List<String> allowed = Arrays.asList(table.options().get("test.allowed").split(","));
List<String> filters =
"no-rules".equals(mode)
Expand Down
Loading