diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala index 87e0e521a1d2..e9e5385ec628 100644 --- a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala +++ b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala @@ -104,8 +104,8 @@ abstract class PaimonBaseScanBuilder if (predicate == null) { postScan.append(filter) } else if ( - coreOptions.queryAuthEnabled() && filter.isInstanceOf[IsNotNull] && - predicate.visit(partitionPredicateVisitor) + filter.isInstanceOf[IsNotNull] && predicate.visit(partitionPredicateVisitor) && + PaimonBaseScan.hasAuthorizedBranch(table) ) { // Spark infers null checks for joins. Masks can null partition keys, so evaluate // these checks on visible rows instead of using them for physical pruning. diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala index ad277d5b09fb..02ec1ed67768 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala @@ -27,8 +27,8 @@ import org.apache.paimon.spark.metric.SparkMetricRegistry import org.apache.paimon.spark.read.{BaseScan, BatchReadTagCleanupListener, PaimonStatistics, PaimonSupportsRuntimeFiltering, SparkHybridSearchBuilderImpl, SparkVectorSearchBuilderImpl} import org.apache.paimon.spark.sources.PaimonMicroBatchStream import org.apache.paimon.spark.util.OptionUtils -import org.apache.paimon.table.{DataTable, FileStoreTable, InnerTable} -import org.apache.paimon.table.source.{InnerTableScan, Split} +import org.apache.paimon.table.{DataTable, FallbackReadFileStoreTable, FileStoreTable, InnerTable, Table} +import org.apache.paimon.table.source.{InnerTableScan, Split, Splits} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.SQLConfHelper @@ -91,7 +91,28 @@ abstract class PaimonBaseScan(table: InnerTable) } override def filterAttributes(): Array[NamedReference] = { - if (postponeMergeOnRead.enabled) Array.empty else super.filterAttributes() + if (postponeMergeOnRead.enabled) { + return Array.empty + } + + val candidates = super.filterAttributes() + table match { + // Fallback and chain scans can read branches with different authorization rules. + case t: FallbackReadFileStoreTable if PaimonBaseScan.hasAuthorizedBranch(t) => Array.empty + case _: FileStoreTable if coreOptions.queryAuthEnabled() && candidates.nonEmpty => + // An ordinary table plan attaches the same authorization result to every split. + inputSplits.headOption match { + case Some(split) => + val auth = Splits.authResult(split) + if (auth == null) candidates + else { + val masks = auth.extractColumnMasking() + candidates.filterNot(field => masks.containsKey(field.fieldNames().head)) + } + case None => Array.empty + } + case _ => candidates + } } private def evalVectorSearch(): GlobalIndexResult = { @@ -208,6 +229,13 @@ abstract class PaimonBaseScan(table: InnerTable) object PaimonBaseScan { + private[spark] def hasAuthorizedBranch(table: Table): Boolean = table match { + case t: FallbackReadFileStoreTable => + hasAuthorizedBranch(t.wrapped()) || hasAuthorizedBranch(t.other()) + case t: FileStoreTable => t.coreOptions().queryAuthEnabled() + case _ => CoreOptions.fromMap(table.options()).queryAuthEnabled() + } + private[spark] def evalVectorSearch( table: InnerTable, vectorSearch: VectorSearch, diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala index ec5dd1355027..bb06f60df26b 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala @@ -103,8 +103,9 @@ abstract class PaimonBaseScanBuilder case None => converter.convert(predicate) match { case Some(paimonPredicate) - if coreOptions.queryAuthEnabled() && predicate.name() == "IS_NOT_NULL" && - paimonPredicate.visit(partitionPredicateVisitor) => + if predicate.name() == "IS_NOT_NULL" && + paimonPredicate.visit(partitionPredicateVisitor) && + PaimonBaseScan.hasAuthorizedBranch(table) => // Spark infers null checks for joins. Masks can null partition keys, so evaluate // these checks on visible rows instead of using them for physical pruning. postScan.append(predicate) diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java index de85fb0bbd3a..58b4fad19936 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkPartitionMetadataAuthTest.java @@ -34,12 +34,15 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.expressions.NamedReference; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -229,6 +232,100 @@ public void testJoinOnMaskedPartitionKey(String mode, @TempDir Path warehouse) } } + @Test + public void testDynamicPartitionPruningOnMaskedJoinKey(@TempDir Path warehouse) { + try (SparkSession spark = createSession(warehouse)) { + createTable(spark, false); + setPolicy(spark, false, "constant", "id,region,dt"); + spark.sql( + "CREATE TABLE spark_test.default.D (region STRING, dt STRING, k INT) " + + "USING paimon TBLPROPERTIES ('bucket'='-1')"); + spark.sql("INSERT INTO spark_test.default.D VALUES ('CN','***',1),('US','other',0)"); + spark.conf().set("spark.sql.optimizer.dynamicPartitionPruning.enabled", true); + + String join = "FROM spark_test.default.T l JOIN spark_test.default.D r ON l.dt = r.dt "; + assertRows( + spark, + "SELECT l.id " + join + "WHERE r.k = 1", + Arrays.asList(RowFactory.create(1), RowFactory.create(2))); + + // The fact scan receives a runtime filter on region even though dt is masked. + Dataset result = + spark.sql( + "SELECT /*+ BROADCAST(r) */ l.id " + + join + + "AND l.region = r.region WHERE r.k = 1"); + assertThat(result.collectAsList()).containsExactly(RowFactory.create(1)); + assertThat(result.queryExecution().executedPlan().toString()) + .contains("PaimonScan: [T], RuntimePartitionFilters: [Equal(region, CN)]"); + } + } + + @ParameterizedTest(name = "chain={0}") + @ValueSource(booleans = {false, true}) + public void testRuntimeFilteringWithAuthorizedBranch(boolean chain, @TempDir Path warehouse) + throws Exception { + try (SparkSession spark = createSession(warehouse)) { + createTable(spark, chain); + if (!chain) { + spark.sql("CALL spark_test.sys.create_branch('default.T', 'backup')"); + spark.sql( + "ALTER TABLE spark_test.default.T SET TBLPROPERTIES ('scan.fallback-branch'='backup')"); + } + SparkCatalog catalog = + (SparkCatalog) spark.sessionState().catalogManager().catalog("spark_test"); + Identifier id = Identifier.of(new String[] {"default"}, "T"); + assertThat(runtimeFilterAttributes(catalog, id)).hasSize(2); + + // Enable authorization only on the secondary branch, nested one level deeper for delta. + String branch = chain ? "delta" : "backup"; + spark.sql( + "ALTER TABLE spark_test.default.`T$branch_" + + branch + + "` SET TBLPROPERTIES ('query-auth.enabled'='true', " + + "'test.auth-mode'='constant', 'test.allowed'='id,dt,region')"); + assertThat(runtimeFilterAttributes(catalog, id)).isEmpty(); + } + } + + @Test + public void testJoinOnNullablePartitionMaskedOnlyOnBackup(@TempDir Path warehouse) { + try (SparkSession spark = createSession(warehouse)) { + spark.sql( + "CREATE TABLE spark_test.default.T (id INT, dt STRING) USING paimon " + + "PARTITIONED BY (dt) TBLPROPERTIES ('bucket'='-1')"); + spark.sql("CALL spark_test.sys.create_branch('default.T', 'backup')"); + spark.sql("INSERT INTO spark_test.default.T VALUES (1, 'A')"); + spark.sql("INSERT INTO spark_test.default.`T$branch_backup` VALUES (2, NULL)"); + spark.sql( + "ALTER TABLE spark_test.default.`T$branch_backup` SET TBLPROPERTIES " + + "('query-auth.enabled'='true')"); + spark.sql( + "ALTER TABLE spark_test.default.T SET TBLPROPERTIES " + + "('scan.fallback-branch'='backup', 'test.auth-mode'='constant', " + + "'test.allowed'='id,dt')"); + spark.sql( + "CREATE TABLE spark_test.default.D (dt STRING, k INT) USING paimon " + + "TBLPROPERTIES ('bucket'='-1')"); + spark.sql("INSERT INTO spark_test.default.D VALUES ('***', 1), ('other', 0)"); + spark.conf().set("spark.sql.optimizer.dynamicPartitionPruning.enabled", true); + + // The backup row's stored NULL becomes '***' before the inferred IS NOT NULL check. + assertRows( + spark, + "SELECT l.id FROM spark_test.default.T l JOIN spark_test.default.D r " + + "ON l.dt = r.dt WHERE r.k = 1", + Collections.singletonList(RowFactory.create(2))); + } + } + + private static NamedReference[] runtimeFilterAttributes(SparkCatalog catalog, Identifier id) + throws NoSuchTableException { + SparkTable table = (SparkTable) catalog.loadTable(id); + return ((PaimonBaseScan) table.newScanBuilder(CaseInsensitiveStringMap.empty()).build()) + .filterAttributes(); + } + @ParameterizedTest(name = "chain={0}, nestedPruning={1}") @CsvSource({"true,true", "true,false", "false,true", "false,false"}) public void testPartitionMetadataAuthorizationWithNestedSchemaPruning( @@ -600,10 +697,11 @@ protected Table loadSparkTable(Identifier identifier, Map extraO throws NoSuchTableException { SparkTable sparkTable = (SparkTable) super.loadSparkTable(identifier, extraOptions); FileStoreTable table = (FileStoreTable) sparkTable.table(); - if (!table.coreOptions().queryAuthEnabled()) { + // The catalog can supply rules even when only a secondary branch enables them. + String mode = table.options().get("test.auth-mode"); + if (mode == null || "disabled".equals(mode)) { return sparkTable; } - String mode = table.options().get("test.auth-mode"); List allowed = Arrays.asList(table.options().get("test.allowed").split(",")); List filters = "no-rules".equals(mode)