From 6143d2a6ed98909dea33916f75299a1e6f977246 Mon Sep 17 00:00:00 2001 From: vishwasvaidya-cloudsufi Date: Sat, 21 Nov 2026 14:38:22 +0000 Subject: [PATCH 1/2] Fix SQL injection in connector sample path and escape init queries --- .../amazon/redshift/RedshiftConnector.java | 2 +- .../redshift/RedshiftConnectorUnitTest.java | 9 + .../mysql/CloudSQLMySQLConnector.java | 2 +- .../mysql/CloudSQLMySQLConnectorUnitTest.java | 48 +++ .../postgres/CloudSQLPostgreSQLConnector.java | 2 +- .../CloudSQLPostgreSQLConnectorUnitTest.java | 54 +++ database-commons/pom.xml | 5 + .../plugin/db/ConnectionConfigAccessor.java | 9 + .../plugin/db/RollbackOnCloseConnection.java | 55 +++ .../AbstractDBSpecificConnector.java | 77 +++- .../db/source/DataDrivenETLDBInputFormat.java | 12 + .../db/RollbackOnCloseConnectionTest.java | 71 ++++ ...stractDBSpecificConnectorReadOnlyTest.java | 360 ++++++++++++++++++ .../db/connector/QuoteIdentifierTest.java | 52 +++ .../DataDrivenETLDBInputFormatTest.java | 2 + .../databricks/DatabricksConnector.java | 22 +- .../DatabricksConnectorUnitTest.java | 19 +- .../io/cdap/plugin/mssql/SqlServerAction.java | 2 +- .../cdap/plugin/mssql/SqlServerConnector.java | 8 +- .../cdap/plugin/mssql/SqlServerConstants.java | 9 + .../plugin/mssql/SqlServerPostAction.java | 2 +- .../io/cdap/plugin/mssql/SqlServerSink.java | 2 +- .../io/cdap/plugin/mssql/SqlServerSource.java | 2 +- .../mssql/SqlServerConnectorUnitTest.java | 21 + .../mssql/SqlServerInitQueriesUnitTest.java | 97 +++++ .../io/cdap/plugin/mysql/MysqlAction.java | 2 +- .../io/cdap/plugin/mysql/MysqlConnector.java | 7 +- .../io/cdap/plugin/mysql/MysqlConstants.java | 9 + .../io/cdap/plugin/mysql/MysqlPostAction.java | 2 +- .../java/io/cdap/plugin/mysql/MysqlSink.java | 2 +- .../io/cdap/plugin/mysql/MysqlSource.java | 2 +- .../plugin/mysql/MysqlConnectorUnitTest.java | 19 + .../mysql/MysqlInitQueriesUnitTest.java | 104 +++++ .../cdap/plugin/oracle/OracleConnector.java | 7 +- .../oracle/OracleConnectorUnitTest.java | 19 + .../plugin/postgres/PostgresConnector.java | 7 +- .../postgres/PostgresConnectorUnitTest.java | 22 ++ 37 files changed, 1095 insertions(+), 51 deletions(-) create mode 100644 cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorUnitTest.java create mode 100644 cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorUnitTest.java create mode 100644 database-commons/src/main/java/io/cdap/plugin/db/RollbackOnCloseConnection.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/RollbackOnCloseConnectionTest.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorReadOnlyTest.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/connector/QuoteIdentifierTest.java create mode 100644 mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerInitQueriesUnitTest.java create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlInitQueriesUnitTest.java diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java index fb8cac4a7..57a253ebb 100644 --- a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java @@ -82,7 +82,7 @@ protected SchemaReader getSchemaReader(String sessionID) { @Override protected String getTableName(String database, String schema, String table) { - return String.format("\"%s\".\"%s\"", schema, table); + return quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); } @Override diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java index 94ea6004b..16c94a3f0 100644 --- a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -41,6 +41,15 @@ public void getTableNameTest() { CONNECTOR.getTableName("db", "schema", "table")); } + /** + * Unit test for getTableName() with identifiers containing double quotes + */ + @Test + public void getTableNameEscapesQuotesTest() { + Assert.assertEquals("\"my\"\"schema\".\"t\"\"; DELETE FROM x; --\"", + CONNECTOR.getTableName("db", "my\"schema", "t\"; DELETE FROM x; --")); + } + @Test public void getRandomQuery() { Assert.assertEquals("SELECT * FROM TestData\n" + diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index 0751bd160..dccb8b047 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -82,7 +82,7 @@ protected SchemaReader getSchemaReader(String sessionID) { @Override protected String getTableName(String database, String schema, String table) { - return String.format("`%s`.`%s`", database, table); + return quoteIdentifier(database, '`') + "." + quoteIdentifier(table, '`'); } @Override diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorUnitTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorUnitTest.java new file mode 100644 index 000000000..84284ceb4 --- /dev/null +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorUnitTest.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for {@link CloudSQLMySQLConnector} + */ +public class CloudSQLMySQLConnectorUnitTest { + + private static final CloudSQLMySQLConnector CONNECTOR = new CloudSQLMySQLConnector( + new CloudSQLMySQLConnectorConfig("user", "password", "cloudsql-mysql", "", "public", + "project:region:instance", "db", null)); + + /** + * Unit test for getTableName() + */ + @Test + public void getTableNameTest() { + Assert.assertEquals("`db`.`table`", + CONNECTOR.getTableName("db", null, "table")); + } + + /** + * Unit test for getTableName() with identifiers containing backticks + */ + @Test + public void getTableNameEscapesBackticksTest() { + Assert.assertEquals("`my``db`.`t``; DELETE FROM x; --`", + CONNECTOR.getTableName("my`db", null, "t`; DELETE FROM x; --")); + } +} diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index 348a1b94c..8f1921478 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -86,7 +86,7 @@ protected SchemaReader getSchemaReader(String sessionID) { @Override protected String getTableName(String database, String schema, String table) { - return String.format("\"%s\".\"%s\"", schema, table); + return quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); } @Override diff --git a/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorUnitTest.java b/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorUnitTest.java new file mode 100644 index 000000000..fa8cb8b6a --- /dev/null +++ b/cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorUnitTest.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.postgres; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for {@link CloudSQLPostgreSQLConnector} + */ +public class CloudSQLPostgreSQLConnectorUnitTest { + + private static final CloudSQLPostgreSQLConnector CONNECTOR = new CloudSQLPostgreSQLConnector( + new CloudSQLPostgreSQLConnectorConfig("user", "password", "cloudsql-postgresql", "", "public", + "project:region:instance", "db", null)); + + /** + * Unit test for getTableName() + */ + @Test + public void getTableNameTest() { + Assert.assertEquals("\"schema\".\"table\"", + CONNECTOR.getTableName("db", "schema", "table")); + } + + /** + * Unit test for getTableName() with identifiers containing double quotes, using the payload from the + * reported sample path SQL injection + */ + @Test + public void getTableNameEscapesQuotesTest() { + String table = "pg_class\" LIMIT 0; UPDATE audit_vault.write_marker SET value='INJECTED_BY_SAMPLE_PATH' " + + "WHERE id=1; SELECT * FROM \"pg_catalog\".\"pg_class"; + Assert.assertEquals("\"pg_catalog\".\"pg_class\"\" LIMIT 0; UPDATE audit_vault.write_marker SET " + + "value='INJECTED_BY_SAMPLE_PATH' WHERE id=1; SELECT * FROM \"\"pg_catalog\"\".\"\"pg_class\"", + CONNECTOR.getTableName("db", "pg_catalog", table)); + Assert.assertEquals("\"my\"\"schema\".\"table\"", + CONNECTOR.getTableName("db", "my\"schema", "table")); + } +} diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 98bcb3803..00f7a714d 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -109,6 +109,11 @@ ${mockito.version} test + + org.hsqldb + hsqldb + test + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java index 253422e79..f02daea5e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java @@ -35,6 +35,7 @@ public class ConnectionConfigAccessor { private static final String CONNECTION_ARGUMENTS = "io.cdap.plugin.db.connection.arguments"; private static final String INIT_QUERIES = "io.cdap.plugin.db.init.queries"; public static final String AUTO_COMMIT_ENABLED = "io.cdap.plugin.db.output.autocommit.enabled"; + public static final String READ_ONLY = "io.cdap.plugin.db.readonly"; public static final String FETCH_SIZE = "io.cdap.plugin.db.fetch.size"; public static final String OPERATION_NAME = "io.cdap.plugin.db.operation.name"; public static final String RELATION_TABLE_KEY = "io.cdap.plugin.db.relation.table.key"; @@ -101,6 +102,14 @@ public boolean isAutoCommitEnabled() { return configuration.getBoolean(AUTO_COMMIT_ENABLED, false); } + public void setReadOnly(boolean readOnly) { + configuration.setBoolean(READ_ONLY, readOnly); + } + + public boolean isReadOnly() { + return configuration.getBoolean(READ_ONLY, false); + } + public void setFetchSize(Integer fetchSize) { configuration.setInt(FETCH_SIZE, fetchSize); } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/RollbackOnCloseConnection.java b/database-commons/src/main/java/io/cdap/plugin/db/RollbackOnCloseConnection.java new file mode 100644 index 000000000..28f0a3c2b --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/RollbackOnCloseConnection.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * A {@link ForwardingConnection} for read-only operations that turns {@link #commit()} into {@link #rollback()} and + * rolls back any active transaction before closing, so Hadoop's {@code DBRecordReader#close()} or an implicit + * driver commit-on-close can never commit changes. + */ +public class RollbackOnCloseConnection extends ForwardingConnection { + + private static final Logger LOG = LoggerFactory.getLogger(RollbackOnCloseConnection.class); + + public RollbackOnCloseConnection(Connection delegate) { + super(delegate); + } + + @Override + public void commit() throws SQLException { + getDelegate().rollback(); + } + + @Override + public void close() throws SQLException { + try { + if (!getDelegate().isClosed() && !getDelegate().getAutoCommit()) { + getDelegate().rollback(); + } + } catch (SQLException e) { + LOG.debug("Failed to roll back read-only connection before close.", e); + } finally { + super.close(); + } + } +} diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index e0407203b..911dd6c7b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -41,6 +41,8 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.sql.Connection; @@ -48,6 +50,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Map; +import java.util.Objects; import java.util.UUID; import javax.annotation.Nullable; @@ -59,6 +62,8 @@ public abstract class AbstractDBSpecificConnector extends AbstractDBConnector implements BatchConnector { + private static final Logger LOG = LoggerFactory.getLogger(AbstractDBSpecificConnector.class); + private final AbstractDBConnectorConfig config; private final RetryPolicy retryPolicy; protected DBErrorDetailsProvider dbErrorDetailsProvider; @@ -105,6 +110,7 @@ public InputFormatProvider getInputFormatProvider(ConnectorContext context, Samp request.getProperties().get("sampleType"), request.getProperties().get("strata"), sessionID); DataDrivenETLDBInputFormat.setInput(connectionConfigAccessor.getConfiguration(), getDBRecordType(), tableQuery, null, isAutoCommitEnabled()); + connectionConfigAccessor.setReadOnly(true); connectionConfigAccessor.setConnectionArguments(Maps.fromProperties(config.getConnectionArgumentsProperties())); String isolationLevel = getTransactionIsolationLevel(); if (isolationLevel != null) { @@ -115,11 +121,11 @@ public InputFormatProvider getInputFormatProvider(ConnectorContext context, Samp for (Map.Entry argument : additionalArguments.entrySet()) { connectionConfigAccessor.getConfiguration().set(argument.getKey(), argument.getValue()); } - try { + try (Connection connection = getConnection(path)) { Long timeoutMs = request.getTimeoutMs(); Integer timeoutSec = timeoutMs != null ? (int) (timeoutMs / 1000) : null; connectionConfigAccessor - .setSchema(loadTableSchema(getConnection(path), tableQuery, timeoutSec, sessionID).toString()); + .setSchema(loadTableSchema(connection, tableQuery, timeoutSec, sessionID).toString()); } catch (SQLException e) { throw new IOException(String.format("Failed to get table schema due to: %s.", ExceptionUtils.getRootCauseMessage(e)), e); @@ -150,8 +156,18 @@ protected String getConnectionString(String database) { } protected String getTableName(String database, String schema, String table) { - return schema == null ? String.format("\"%s\".\"%s\"", database, table) - : String.format("\"%s\".\"%s\".\"%s\"", database, schema, table); + return schema == null ? quoteIdentifier(database, '"') + "." + quoteIdentifier(table, '"') + : quoteIdentifier(database, '"') + "." + quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); + } + + /** + * Wraps the given identifier in the given quote character, doubling any embedded quote characters so that a + * user supplied database, schema, table or column name can never break out of the quoted identifier. + */ + protected static String quoteIdentifier(String identifier, char quote) { + Objects.requireNonNull(identifier, "identifier to quote must not be null"); + String quoteString = String.valueOf(quote); + return quoteString + identifier.replace(quoteString, quoteString + quoteString) + quoteString; } protected String getTableQuery(String database, String schema, String table) { @@ -197,15 +213,46 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getErrorDetailsProvider())) { - statement.setMaxRows(1); - if (timeoutSec != null) { - statement.setQueryTimeout(timeoutSec); - } - ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, query, - getErrorDetailsProvider()); + boolean rollbackOnExit = configureReadOnlyTransaction(connection); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getErrorDetailsProvider())) { + statement.setMaxRows(1); + if (timeoutSec != null) { + statement.setQueryTimeout(timeoutSec); + } + try (ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + query, getErrorDetailsProvider())) { return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + } + } finally { + if (rollbackOnExit) { + try { + connection.rollback(); + } catch (SQLException e) { + LOG.debug("Failed to roll back read-only schema query transaction.", e); + } + } + } + } + + private boolean configureReadOnlyTransaction(@Nullable Connection connection) { + if (connection == null) { + return false; + } + try { + connection.setReadOnly(true); + } catch (SQLException e) { + LOG.debug("Driver does not support setting connection read-only; continuing.", e); + } + if (isAutoCommitEnabled()) { + return false; + } + try { + connection.setAutoCommit(false); + return true; + } catch (SQLException e) { + LOG.debug("Driver does not support disabling auto-commit; continuing.", e); + return false; } } @@ -218,8 +265,10 @@ protected void setConnectionProperties(Map properties, Connector protected Schema getTableSchema(Connection connection, String database, String schema, String table) throws SQLException { String sessionID = generateSessionID(); - return loadTableSchema(getConnection(), getTableQuery(database, schema, table), - null, sessionID); + try (Connection schemaConnection = getConnection()) { + return loadTableSchema(schemaConnection, getTableQuery(database, schema, table), + null, sessionID); + } } protected String generateSessionID() { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java index 1a67f09ea..d6f37d49a 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormat.java @@ -23,6 +23,7 @@ import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.JDBCDriverShim; import io.cdap.plugin.db.NoOpCommitConnection; +import io.cdap.plugin.db.RollbackOnCloseConnection; import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.conf.Configuration; @@ -154,11 +155,22 @@ public Connection getConnection() { properties.putAll(connectionConfigAccessor.getConnectionArguments()); connection = DriverManager.getConnection(url, properties); + if (connectionConfigAccessor.isReadOnly()) { + try { + connection.setReadOnly(true); + } catch (SQLException e) { + LOG.debug("Driver does not support setting connection read-only; continuing.", e); + } + } + if (connectionConfigAccessor.isAutoCommitEnabled()) { // hack to work around jdbc drivers like the hive driver that throw exceptions on commit this.connection = new NoOpCommitConnection(this.connection); } else { this.connection.setAutoCommit(false); + if (connectionConfigAccessor.isReadOnly()) { + this.connection = new RollbackOnCloseConnection(this.connection); + } } if (connectionConfigAccessor.getFetchSize() > 0) { diff --git a/database-commons/src/test/java/io/cdap/plugin/db/RollbackOnCloseConnectionTest.java b/database-commons/src/test/java/io/cdap/plugin/db/RollbackOnCloseConnectionTest.java new file mode 100644 index 000000000..45c888330 --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/RollbackOnCloseConnectionTest.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Unit tests for {@link RollbackOnCloseConnection}. + */ +public class RollbackOnCloseConnectionTest { + + @Test + public void commit_delegatesToRollbackInsteadOfCommit() throws SQLException { + Connection delegate = Mockito.mock(Connection.class); + RollbackOnCloseConnection wrapper = new RollbackOnCloseConnection(delegate); + + wrapper.commit(); + + Mockito.verify(delegate).rollback(); + Mockito.verify(delegate, Mockito.never()).commit(); + } + + @Test + public void close_activeTransaction_rollsBackBeforeClose() throws SQLException { + Connection delegate = Mockito.mock(Connection.class); + Mockito.when(delegate.isClosed()).thenReturn(false); + Mockito.when(delegate.getAutoCommit()).thenReturn(false); + RollbackOnCloseConnection wrapper = new RollbackOnCloseConnection(delegate); + + wrapper.close(); + + InOrder inOrder = Mockito.inOrder(delegate); + inOrder.verify(delegate).rollback(); + inOrder.verify(delegate).close(); + Mockito.verify(delegate, Mockito.never()).commit(); + } + + @Test + public void close_rollbackThrows_stillClosesDelegate() throws SQLException { + Connection delegate = Mockito.mock(Connection.class); + Mockito.when(delegate.isClosed()).thenReturn(false); + Mockito.when(delegate.getAutoCommit()).thenReturn(false); + Mockito.doThrow(new SQLException("rollback failed")).when(delegate).rollback(); + RollbackOnCloseConnection wrapper = new RollbackOnCloseConnection(delegate); + + wrapper.close(); + + Mockito.verify(delegate).close(); + Assert.assertNotNull(wrapper.getDelegate()); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorReadOnlyTest.java b/database-commons/src/test/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorReadOnlyTest.java new file mode 100644 index 000000000..7e4890d6e --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorReadOnlyTest.java @@ -0,0 +1,360 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db.connector; + +import io.cdap.cdap.api.data.batch.InputFormatProvider; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.SampleRequest; +import io.cdap.cdap.etl.mock.common.MockConnectorConfigurer; +import io.cdap.cdap.etl.mock.common.MockConnectorContext; +import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; +import io.cdap.plugin.db.DBRecord; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskID; +import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.hsqldb.jdbc.JDBCDriver; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Unit and embedded-JDBC tests verifying identifier escaping, read-only transactions, and rollback behavior on + * {@link AbstractDBSpecificConnector}. + */ +public class AbstractDBSpecificConnectorReadOnlyTest { + + @Test + public void loadTableSchema_defaultConnector_setsReadOnlyDisablesAutoCommitAndRollsBack() throws Exception { + Connection connection = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(connection.createStatement()).thenReturn(statement); + Mockito.when(statement.executeQuery("SELECT * FROM \"db\".\"tbl\"")).thenReturn(resultSet); + Mockito.when(resultSet.getMetaData()).thenReturn(metaData); + Mockito.when(metaData.getColumnCount()).thenReturn(1); + Mockito.when(metaData.getColumnName(1)).thenReturn("id"); + Mockito.when(metaData.getColumnType(1)).thenReturn(Types.INTEGER); + Mockito.when(metaData.getColumnTypeName(1)).thenReturn("INTEGER"); + Mockito.when(metaData.isSigned(1)).thenReturn(true); + Mockito.when(metaData.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls); + + StubConnector connector = new StubConnector(connection, false); + Schema schema = connector.loadTableSchema(connection, "SELECT * FROM \"db\".\"tbl\"", 5, "s1"); + Assert.assertNotNull(schema.getField("id")); + + InOrder inOrder = Mockito.inOrder(connection, statement, resultSet); + inOrder.verify(connection).setReadOnly(true); + inOrder.verify(connection).setAutoCommit(false); + inOrder.verify(connection).createStatement(); + inOrder.verify(statement).setMaxRows(1); + inOrder.verify(statement).setQueryTimeout(5); + inOrder.verify(statement).executeQuery("SELECT * FROM \"db\".\"tbl\""); + inOrder.verify(resultSet).close(); + inOrder.verify(statement).close(); + inOrder.verify(connection).rollback(); + Mockito.verify(connection, Mockito.never()).commit(); + } + + @Test + public void loadTableSchema_queryFails_stillRollsBackAndNeverCommits() throws Exception { + Connection connection = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + Mockito.when(connection.createStatement()).thenReturn(statement); + Mockito.when(statement.executeQuery(Mockito.anyString())) + .thenThrow(new SQLException("Multiple ResultSets were returned by the query.")); + + StubConnector connector = new StubConnector(connection, false); + Assert.assertThrows(RuntimeException.class, + () -> connector.loadTableSchema(connection, "SELECT * FROM \"db\".\"tbl\"", null, "s1")); + + InOrder inOrder = Mockito.inOrder(connection, statement); + inOrder.verify(connection).setReadOnly(true); + inOrder.verify(connection).setAutoCommit(false); + inOrder.verify(statement).executeQuery("SELECT * FROM \"db\".\"tbl\""); + inOrder.verify(statement).close(); + inOrder.verify(connection).rollback(); + Mockito.verify(connection, Mockito.never()).commit(); + } + + @Test + public void loadTableSchema_autoCommitConnectorAndUnsupportedReadOnly_toleratesAndSkipsRollback() throws Exception { + Connection connection = mockConnectionWithSingleIntColumn(); + Mockito.doThrow(new SQLFeatureNotSupportedException("readOnly not supported")).when(connection).setReadOnly(true); + + StubConnector connector = new StubConnector(connection, true); + Schema schema = connector.loadTableSchema(connection, "SELECT * FROM `db`.`tbl`", null, "s1"); + Assert.assertNotNull(schema.getField("id")); + + Mockito.verify(connection).setReadOnly(true); + Mockito.verify(connection, Mockito.never()).setAutoCommit(Mockito.anyBoolean()); + Mockito.verify(connection, Mockito.never()).rollback(); + Mockito.verify(connection, Mockito.never()).commit(); + } + + @Test + public void getInputFormatProvider_andGetTableSchema_closeOpenedConnections() throws Exception { + Connection sampleConnection = mockConnectionWithSingleIntColumn(); + StubConnector connector = new StubConnector(sampleConnection, false); + + SampleRequest request = SampleRequest.builder(10).setPath("db/tbl").build(); + InputFormatProvider provider = + connector.getInputFormatProvider(new MockConnectorContext(new MockConnectorConfigurer()), request); + Assert.assertNotNull(provider); + Mockito.verify(sampleConnection).rollback(); + Mockito.verify(sampleConnection).close(); + + Connection specConnection = mockConnectionWithSingleIntColumn(); + StubConnector specConnector = new StubConnector(specConnection, false); + Schema schema = specConnector.getTableSchema(null, "db", null, "tbl"); + Assert.assertNotNull(schema.getField("id")); + Mockito.verify(specConnection).rollback(); + Mockito.verify(specConnection).close(); + } + + @Test + public void sampleAndGenerateSpec_withQuoteAndStackedUpdateInPath_neverExecutesSecondStatement() throws Exception { + String url = "jdbc:hsqldb:mem:vrp_" + UUID.randomUUID().toString().replace('-', '_'); + try (Connection adminConn = DriverManager.getConnection(url, "SA", ""); + Statement stmt = adminConn.createStatement()) { + stmt.execute("CREATE TABLE \"PUBLIC\".\"target_table\" (\"id\" INT, \"name\" VARCHAR(32))"); + stmt.execute("INSERT INTO \"PUBLIC\".\"target_table\" VALUES (1, 'alice')"); + stmt.execute("CREATE TABLE \"PUBLIC\".\"write_marker\" (\"id\" INT PRIMARY KEY, \"val\" VARCHAR(64))"); + stmt.execute("INSERT INTO \"PUBLIC\".\"write_marker\" VALUES (1, 'ORIGINAL')"); + + HsqldbSchemaConnector connector = new HsqldbSchemaConnector(url); + MockConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + + // Benign path succeeds through both generateSpec and full InputFormat/RecordReader sampling. + ConnectorSpec benignSpec = connector.generateSpec( + context, ConnectorSpecRequest.builder().setPath("/PUBLIC/target_table").setConnection("conn").build()); + Assert.assertNotNull(benignSpec.getSchema().getField("id")); + InputFormatProvider provider = connector.getInputFormatProvider( + context, SampleRequest.builder(10).setPath("/PUBLIC/target_table").build()); + List records = readSampleRecords(connector, provider); + Assert.assertEquals(1, records.size()); + Assert.assertEquals("alice", records.get(0).get("name")); + Assert.assertEquals("ORIGINAL", readMarker(adminConn)); + + // Injection payload with embedded '"' and stacked UPDATE stays inside the quoted identifier and cannot modify DB. + String injectedPath = "/PUBLIC/target_table\" LIMIT 0; " + + "UPDATE \"PUBLIC\".\"write_marker\" SET \"val\"='INJECTED_BY_SAMPLE_PATH' WHERE \"id\"=1; " + + "SELECT * FROM \"PUBLIC\".\"target_table"; + Assert.assertThrows(RuntimeException.class, () -> connector.generateSpec( + context, ConnectorSpecRequest.builder().setPath(injectedPath).setConnection("conn").build())); + Assert.assertThrows(RuntimeException.class, () -> connector.getInputFormatProvider( + context, SampleRequest.builder(10).setPath(injectedPath).build())); + Assert.assertEquals("ORIGINAL", readMarker(adminConn)); + + // Defense-in-depth: even if a write statement were passed directly to loadTableSchema, the read-only + // transaction / rollback backstop prevents any modification from persisting. + try (Connection connectorConn = DriverManager.getConnection(url, "SA", "")) { + String updateSql = "UPDATE \"PUBLIC\".\"write_marker\" SET \"val\"='INJECTED_DIRECT' WHERE \"id\"=1"; + Assert.assertThrows(RuntimeException.class, + () -> connector.loadTableSchema(connectorConn, updateSql, null, "s1")); + } + Assert.assertEquals("ORIGINAL", readMarker(adminConn)); + } finally { + try (Connection cleanup = DriverManager.getConnection(url, "SA", ""); + Statement stmt = cleanup.createStatement()) { + stmt.execute("SHUTDOWN"); + } + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List readSampleRecords(HsqldbSchemaConnector connector, + InputFormatProvider provider) throws Exception { + Configuration hConf = new Configuration(); + hConf.setClassLoader(AbstractDBSpecificConnectorReadOnlyTest.class.getClassLoader()); + provider.getInputFormatConfiguration().forEach(hConf::set); + Job job = Job.getInstance(hConf); + job.setJobID(new JobID("sample", 0)); + InputFormat inputFormat = (InputFormat) AbstractDBSpecificConnectorReadOnlyTest.class.getClassLoader() + .loadClass(provider.getInputFormatClassName()).newInstance(); + if (inputFormat instanceof Configurable) { + ((Configurable) inputFormat).setConf(hConf); + } + List splits = inputFormat.getSplits(job); + InputSplit split = splits.get(0); + TaskID taskId = new TaskID(job.getJobID(), TaskType.MAP, 0); + TaskAttemptContext taskContext = new TaskAttemptContextImpl(hConf, new TaskAttemptID(taskId, 0)); + List sample = new ArrayList<>(); + try (RecordReader reader = inputFormat.createRecordReader(split, taskContext)) { + reader.initialize(split, taskContext); + while (reader.nextKeyValue()) { + sample.add(connector.transform(reader.getCurrentKey(), reader.getCurrentValue())); + } + } + return sample; + } + + private static String readMarker(Connection connection) throws SQLException { + try (Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT \"val\" FROM \"PUBLIC\".\"write_marker\" WHERE \"id\" = 1")) { + Assert.assertTrue(rs.next()); + return rs.getString(1); + } + } + + private static Connection mockConnectionWithSingleIntColumn() throws SQLException { + Connection connection = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(connection.createStatement()).thenReturn(statement); + Mockito.when(statement.executeQuery(Mockito.anyString())).thenReturn(resultSet); + Mockito.when(resultSet.getMetaData()).thenReturn(metaData); + Mockito.when(metaData.getColumnCount()).thenReturn(1); + Mockito.when(metaData.getColumnName(1)).thenReturn("id"); + Mockito.when(metaData.getColumnType(1)).thenReturn(Types.INTEGER); + Mockito.when(metaData.getColumnTypeName(1)).thenReturn("INTEGER"); + Mockito.when(metaData.isSigned(1)).thenReturn(true); + Mockito.when(metaData.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls); + return connection; + } + + private static class StubConnectorConfig extends AbstractDBConnectorConfig { + private final String connectionString; + + StubConnectorConfig() { + this("jdbc:stub://localhost:5432/db"); + } + + StubConnectorConfig(String connectionString) { + this.connectionString = connectionString; + this.user = "SA"; + this.password = ""; + } + + @Override + public String getConnectionString() { + return connectionString; + } + + @Override + public Integer getMaxRetryCount() { + return 0; + } + } + + private static class StubConnector extends AbstractDBSpecificConnector { + private final Connection stubConnection; + private final boolean autoCommitEnabled; + + StubConnector(Connection stubConnection, boolean autoCommitEnabled) { + super(new StubConnectorConfig()); + this.driverClass = Driver.class; + this.stubConnection = stubConnection; + this.autoCommitEnabled = autoCommitEnabled; + } + + @Override + public boolean supportSchema() { + return false; + } + + @Override + protected Class getDBRecordType() { + return DBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable key, DBRecord val) { + return null; + } + + @Override + protected Connection getConnection() { + return stubConnection; + } + + @Override + protected Connection getConnection(DBConnectorPath path) { + return stubConnection; + } + + @Override + protected boolean isAutoCommitEnabled() { + return autoCommitEnabled; + } + } + + private static class HsqldbSchemaConnector extends AbstractDBSpecificConnector { + HsqldbSchemaConnector(String url) { + super(new StubConnectorConfig(url)); + this.driverClass = JDBCDriver.class; + } + + @Override + public boolean supportSchema() { + return true; + } + + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + + @Override + protected String getTableName(String database, String schema, String table) { + return quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); + } + + @Override + protected Class getDBRecordType() { + return DBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable key, DBRecord val) { + return val.getRecord(); + } + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/connector/QuoteIdentifierTest.java b/database-commons/src/test/java/io/cdap/plugin/db/connector/QuoteIdentifierTest.java new file mode 100644 index 000000000..f92c59c2e --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/connector/QuoteIdentifierTest.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db.connector; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for {@link AbstractDBSpecificConnector#quoteIdentifier(String, char)} + */ +public class QuoteIdentifierTest { + + @Test + public void testPlainIdentifier() { + Assert.assertEquals("\"table\"", AbstractDBSpecificConnector.quoteIdentifier("table", '"')); + Assert.assertEquals("`table`", AbstractDBSpecificConnector.quoteIdentifier("table", '`')); + } + + @Test + public void testEmbeddedQuotesAreDoubled() { + Assert.assertEquals("\"my\"\"table\"", AbstractDBSpecificConnector.quoteIdentifier("my\"table", '"')); + Assert.assertEquals("`my``table`", AbstractDBSpecificConnector.quoteIdentifier("my`table", '`')); + Assert.assertEquals("\"\"\"\"\"\"", AbstractDBSpecificConnector.quoteIdentifier("\"\"", '"')); + } + + @Test + public void testOtherQuoteCharacterIsLeftAsIs() { + Assert.assertEquals("\"my`table\"", AbstractDBSpecificConnector.quoteIdentifier("my`table", '"')); + Assert.assertEquals("`my\"table`", AbstractDBSpecificConnector.quoteIdentifier("my\"table", '`')); + } + + @Test + public void testInjectionPayloadStaysInsideIdentifier() { + String payload = "pg_class\" LIMIT 0; UPDATE t SET v='x' WHERE id=1; SELECT * FROM \"pg_class"; + Assert.assertEquals("\"pg_class\"\" LIMIT 0; UPDATE t SET v='x' WHERE id=1; SELECT * FROM \"\"pg_class\"", + AbstractDBSpecificConnector.quoteIdentifier(payload, '"')); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java index a07d0dee5..a24b08d8d 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/DataDrivenETLDBInputFormatTest.java @@ -150,6 +150,7 @@ public void testDefaultConnectorInputFormatConfiguration() throws IOException { Map conf = provider.getInputFormatConfiguration(); Assert.assertEquals("false", conf.get(ConnectionConfigAccessor.AUTO_COMMIT_ENABLED)); + Assert.assertEquals("true", conf.get(ConnectionConfigAccessor.READ_ONLY)); Assert.assertNull(conf.get(TransactionIsolationLevel.CONF_KEY)); } @@ -173,6 +174,7 @@ protected String getTransactionIsolationLevel() { Map conf = provider.getInputFormatConfiguration(); Assert.assertEquals("true", conf.get(ConnectionConfigAccessor.AUTO_COMMIT_ENABLED)); + Assert.assertEquals("true", conf.get(ConnectionConfigAccessor.READ_ONLY)); Assert.assertEquals(TransactionIsolationLevel.Level.TRANSACTION_READ_UNCOMMITTED.name(), conf.get(TransactionIsolationLevel.CONF_KEY)); } diff --git a/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksConnector.java b/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksConnector.java index c78ceca6f..e13369b85 100644 --- a/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksConnector.java +++ b/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksConnector.java @@ -45,6 +45,9 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Databricks Database Connector that connects to Databricks database via JDBC. @@ -115,16 +118,10 @@ protected SchemaReader getSchemaReader(String sessionID) { @Override protected String getTableName(String database, String schema, String table) { - if (database == null && schema == null) { - return String.format("`%s`", table); - } - if (database == null) { - return String.format("`%s`.`%s`", schema, table); - } - if (schema == null) { - return String.format("`%s`.`%s`", database, table); - } - return String.format("`%s`.`%s`.`%s`", database, schema, table); + return Stream.of(database, schema, table) + .filter(Objects::nonNull) + .map(part -> quoteIdentifier(part, '`')) + .collect(Collectors.joining(".")); } @Override @@ -136,6 +133,7 @@ protected String getRandomQuery(String tableName, int limit) { @Override protected String getStratifiedQuery(String tableName, int limit, String strata, String sessionID) { + String strataColumn = quoteIdentifier(strata, '`'); return String.format("WITH t_%s AS (\n" + " SELECT *,\n" + " ROW_NUMBER() OVER (ORDER BY %s, RAND()) AS sqn_%s,\n" + @@ -146,8 +144,8 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, "WHERE MOD(sqn_%s, GREATEST(1, CAST(c_%s / %d AS BIGINT))) = 1\n" + "ORDER BY %s\n" + "LIMIT %d", - sessionID, strata, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, - limit, strata, limit); + sessionID, strataColumn, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, + limit, strataColumn, limit); } @Override diff --git a/databricks-plugin/src/test/java/io/cdap/plugin/databricks/DatabricksConnectorUnitTest.java b/databricks-plugin/src/test/java/io/cdap/plugin/databricks/DatabricksConnectorUnitTest.java index a04dfb398..d97ec4e38 100644 --- a/databricks-plugin/src/test/java/io/cdap/plugin/databricks/DatabricksConnectorUnitTest.java +++ b/databricks-plugin/src/test/java/io/cdap/plugin/databricks/DatabricksConnectorUnitTest.java @@ -38,6 +38,21 @@ public void testGetTableName() { CONNECTOR.getTableName(null, null, "my_table")); } + @Test + public void testGetTableNameEscapesBackticks() { + Assert.assertEquals("`main`.`default`.`t``; DELETE FROM x; --`", + CONNECTOR.getTableName("main", "default", "t`; DELETE FROM x; --")); + Assert.assertEquals("`my``schema`.`my``table`", + CONNECTOR.getTableName(null, "my`schema", "my`table")); + } + + @Test + public void testGetStratifiedQueryEscapesStrata() { + String query = CONNECTOR.getStratifiedQuery("`main`.`default`.`my_table`", 10, "id`; DROP TABLE t; --", "s1"); + Assert.assertTrue(query, query.contains("ORDER BY `id``; DROP TABLE t; --`, RAND()")); + Assert.assertTrue(query, query.contains("ORDER BY `id``; DROP TABLE t; --`\n")); + } + @Test public void testGetRandomQuery() { Assert.assertEquals("SELECT * FROM `main`.`default`.`my_table`\n" + @@ -49,13 +64,13 @@ public void testGetRandomQuery() { public void testGetStratifiedQuery() { Assert.assertEquals("WITH t_s1 AS (\n" + " SELECT *,\n" + - " ROW_NUMBER() OVER (ORDER BY id, RAND()) AS sqn_s1,\n" + + " ROW_NUMBER() OVER (ORDER BY `id`, RAND()) AS sqn_s1,\n" + " COUNT(*) OVER () AS c_s1\n" + " FROM `main`.`default`.`my_table`\n" + " )\n" + "SELECT * FROM t_s1\n" + "WHERE MOD(sqn_s1, GREATEST(1, CAST(c_s1 / 10 AS BIGINT))) = 1\n" + - "ORDER BY id\n" + + "ORDER BY `id`\n" + "LIMIT 10", CONNECTOR.getStratifiedQuery("`main`.`default`.`my_table`", 10, "id", "s1")); } diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerAction.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerAction.java index 62c7b9252..6d7da91a2 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerAction.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerAction.java @@ -125,7 +125,7 @@ public Map getDBSpecificArguments() { @Override public List getInitQueries() { if (!Strings.isNullOrEmpty(currentLanguage)) { - return Collections.singletonList(String.format(SqlServerConstants.SET_LANGUAGE_QUERY_FORMAT, currentLanguage)); + return Collections.singletonList(SqlServerConstants.buildSetLanguageQuery(currentLanguage)); } return Collections.emptyList(); diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index 7e86d97ce..4f3ba8963 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -117,7 +117,8 @@ public StructuredRecord transform(LongWritable longWritable, SqlServerSourceDBRe @Override protected String getTableName(String database, String schema, String table) { - return String.format("\"%s\".\"%s\".\"%s\"", database, schema, table); + return quoteIdentifier(database, '"') + "." + quoteIdentifier(schema, '"') + "." + + quoteIdentifier(table, '"'); } @Override @@ -137,6 +138,7 @@ protected String getRandomQuery(String tableName, int limit) { @Override protected String getStratifiedQuery(String tableName, int limit, String strata, String sessionID) { + String strataColumn = quoteIdentifier(strata, '"'); return String.format("WITH t_%s AS (\n" + " SELECT *,\n" + " ROW_NUMBER() OVER (ORDER BY %s, RAND()) AS sqn_%s,\n" + @@ -146,8 +148,8 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, "SELECT TOP %d * FROM t_%s\n" + "WHERE sqn_%s %% CAST(0.5 * ((c_%s / %d + 1) + ABS(c_%s / %d - 1)) AS bigint) = 1\n" + "ORDER BY %s", - sessionID, strata, sessionID, sessionID, tableName, limit, sessionID, sessionID, sessionID, - limit, sessionID, limit, strata); + sessionID, strataColumn, sessionID, sessionID, tableName, limit, sessionID, sessionID, + sessionID, limit, sessionID, limit, strataColumn); } @Override diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConstants.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConstants.java index faa6a7fb0..62402663b 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConstants.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConstants.java @@ -208,4 +208,13 @@ private SqlServerConstants() { */ public static final String SET_LANGUAGE_QUERY_FORMAT = "SET LANGUAGE '%s';"; + /** + * Builds the {@code SET LANGUAGE} initialization query, doubling any embedded single quotes so that a + * user-supplied language value cannot break out of the string literal. Unlike MySQL, SQL Server string literals + * treat backslashes as ordinary characters, so only single quotes need to be doubled. + */ + public static String buildSetLanguageQuery(String currentLanguage) { + return String.format(SET_LANGUAGE_QUERY_FORMAT, currentLanguage.replace("'", "''")); + } + } diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerPostAction.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerPostAction.java index 6540223ed..bf806d219 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerPostAction.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerPostAction.java @@ -125,7 +125,7 @@ public Map getDBSpecificArguments() { @Override public List getInitQueries() { if (!Strings.isNullOrEmpty(currentLanguage)) { - return Collections.singletonList(String.format(SqlServerConstants.SET_LANGUAGE_QUERY_FORMAT, currentLanguage)); + return Collections.singletonList(SqlServerConstants.buildSetLanguageQuery(currentLanguage)); } return Collections.emptyList(); diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 00b627139..0f5d31a26 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -206,7 +206,7 @@ protected SqlServerConnectorConfig getConnection() { @Override public List getInitQueries() { if (!Strings.isNullOrEmpty(currentLanguage)) { - return Collections.singletonList(String.format(SqlServerConstants.SET_LANGUAGE_QUERY_FORMAT, currentLanguage)); + return Collections.singletonList(SqlServerConstants.buildSetLanguageQuery(currentLanguage)); } return Collections.emptyList(); diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index d8d4cf39e..db1970549 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -196,7 +196,7 @@ protected AbstractDBSpecificConnectorConfig getConnection() { @Override public List getInitQueries() { if (!Strings.isNullOrEmpty(currentLanguage)) { - return Collections.singletonList(String.format(SqlServerConstants.SET_LANGUAGE_QUERY_FORMAT, currentLanguage)); + return Collections.singletonList(SqlServerConstants.buildSetLanguageQuery(currentLanguage)); } return Collections.emptyList(); diff --git a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java index 0163cd55e..493e2f0e5 100644 --- a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java +++ b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java @@ -34,6 +34,27 @@ public class SqlServerConnectorUnitTest { // private static final SqlServerConnector CONNECTOR = new SqlServerConnector(new SqlServerConnectorConfig()); + /** + * Unit test for getTableName() with identifiers containing double quotes + */ + @Test + public void getTableNameEscapesQuotesTest() { + Assert.assertEquals("\"db\".\"dbo\".\"t\"\"; DELETE FROM x; --\"", + CONNECTOR.getTableName("db", "dbo", "t\"; DELETE FROM x; --")); + Assert.assertEquals("\"my\"\"db\".\"my\"\"schema\".\"my\"\"table\"", + CONNECTOR.getTableName("my\"db", "my\"schema", "my\"table")); + } + + /** + * Unit test for getStratifiedQuery() with a strata column containing double quotes + */ + @Test + public void getStratifiedQueryEscapesStrataTest() { + String query = CONNECTOR.getStratifiedQuery("\"db\".\"dbo\".\"table\"", 10, "id\"; DROP TABLE t; --", "s1"); + Assert.assertTrue(query, query.contains("ORDER BY \"id\"\"; DROP TABLE t; --\", RAND()")); + Assert.assertTrue(query, query.endsWith("ORDER BY \"id\"\"; DROP TABLE t; --\"")); + } + /** * Unit tests for getTableQuery() */ diff --git a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerInitQueriesUnitTest.java b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerInitQueriesUnitTest.java new file mode 100644 index 000000000..d5ff200ec --- /dev/null +++ b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerInitQueriesUnitTest.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mssql; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +/** + * Unit tests for {@code getInitQueries()} escaping across SQL Server plugin configs. + */ +public class SqlServerInitQueriesUnitTest { + + @Test + public void getInitQueries_emptyOrNullLanguage_returnsEmptyList() { + SqlServerSource.SqlServerSourceConfig sourceConfig = new SqlServerSource.SqlServerSourceConfig(); + SqlServerSink.SqlServerSinkConfig sinkConfig = new SqlServerSink.SqlServerSinkConfig(); + SqlServerAction.SqlServerActionConfig actionConfig = new SqlServerAction.SqlServerActionConfig(); + SqlServerPostAction.SqlServerQueryActionConfig postActionConfig = + new SqlServerPostAction.SqlServerQueryActionConfig(); + + Assert.assertEquals(Collections.emptyList(), sourceConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), sinkConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), actionConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), postActionConfig.getInitQueries()); + + sourceConfig.currentLanguage = ""; + sinkConfig.currentLanguage = ""; + actionConfig.currentLanguage = ""; + postActionConfig.currentLanguage = ""; + + Assert.assertEquals(Collections.emptyList(), sourceConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), sinkConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), actionConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), postActionConfig.getInitQueries()); + } + + @Test + public void getInitQueries_plainLanguage_formatsSetLanguageStatement() { + String expected = "SET LANGUAGE 'us_english';"; + + SqlServerSource.SqlServerSourceConfig sourceConfig = new SqlServerSource.SqlServerSourceConfig(); + sourceConfig.currentLanguage = "us_english"; + Assert.assertEquals(Collections.singletonList(expected), sourceConfig.getInitQueries()); + + SqlServerSink.SqlServerSinkConfig sinkConfig = new SqlServerSink.SqlServerSinkConfig(); + sinkConfig.currentLanguage = "us_english"; + Assert.assertEquals(Collections.singletonList(expected), sinkConfig.getInitQueries()); + + SqlServerAction.SqlServerActionConfig actionConfig = new SqlServerAction.SqlServerActionConfig(); + actionConfig.currentLanguage = "us_english"; + Assert.assertEquals(Collections.singletonList(expected), actionConfig.getInitQueries()); + + SqlServerPostAction.SqlServerQueryActionConfig postActionConfig = + new SqlServerPostAction.SqlServerQueryActionConfig(); + postActionConfig.currentLanguage = "us_english"; + Assert.assertEquals(Collections.singletonList(expected), postActionConfig.getInitQueries()); + } + + @Test + public void getInitQueries_quotesAndBackslashes_doublesSingleQuotesOnly() { + String payload = "us_english\\'; UPDATE t SET v = 'x'; --"; + String expected = "SET LANGUAGE 'us_english\\''; UPDATE t SET v = ''x''; --';"; + + SqlServerSource.SqlServerSourceConfig sourceConfig = new SqlServerSource.SqlServerSourceConfig(); + sourceConfig.currentLanguage = payload; + Assert.assertEquals(Collections.singletonList(expected), sourceConfig.getInitQueries()); + + SqlServerSink.SqlServerSinkConfig sinkConfig = new SqlServerSink.SqlServerSinkConfig(); + sinkConfig.currentLanguage = payload; + Assert.assertEquals(Collections.singletonList(expected), sinkConfig.getInitQueries()); + + SqlServerAction.SqlServerActionConfig actionConfig = new SqlServerAction.SqlServerActionConfig(); + actionConfig.currentLanguage = payload; + Assert.assertEquals(Collections.singletonList(expected), actionConfig.getInitQueries()); + + SqlServerPostAction.SqlServerQueryActionConfig postActionConfig = + new SqlServerPostAction.SqlServerQueryActionConfig(); + postActionConfig.currentLanguage = payload; + Assert.assertEquals(Collections.singletonList(expected), postActionConfig.getInitQueries()); + } +} diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlAction.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlAction.java index c8433b92e..f7955a995 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlAction.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlAction.java @@ -115,7 +115,7 @@ public List getInitQueries() { initQueries.add(MysqlConstants.ANSI_QUOTES_QUERY); } if (!Strings.isNullOrEmpty(sqlMode)) { - initQueries.add(String.format(MysqlConstants.SET_SQL_MODE_QUERY_FORMAT, sqlMode)); + initQueries.add(MysqlConstants.buildSetSqlModeQuery(sqlMode)); } return initQueries; } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 214d8650e..0f854bc88 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -98,7 +98,7 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa @Override protected String getTableName(String database, String schema, String table) { - return String.format("`%s`.`%s`", database, table); + return quoteIdentifier(database, '`') + "." + quoteIdentifier(table, '`'); } @Override @@ -112,6 +112,7 @@ protected String getRandomQuery(String tableName, int limit) { @Override protected String getStratifiedQuery(String tableName, int limit, String strata, String sessionID) { + String strataColumn = quoteIdentifier(strata, '`'); return String.format("WITH t_%s AS (\n" + " SELECT *,\n" + " ROW_NUMBER() OVER (ORDER BY %s, RAND()) AS sqn_%s,\n" + @@ -122,8 +123,8 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, "WHERE MOD(sqn_%s, GREATEST(1, CAST(c_%s / %d AS UNSIGNED))) = 1\n" + "ORDER BY %s\n" + "LIMIT %d", - sessionID, strata, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, - limit, strata, limit); + sessionID, strataColumn, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, + limit, strataColumn, limit); } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java index a73e14e9f..3fea43949 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java @@ -56,4 +56,13 @@ private MysqlConstants() { * Query to append 'ANSI_QUOTES' sql mode to the current value of SQL_MODE system variable. */ public static final String ANSI_QUOTES_QUERY = "SET SESSION sql_mode = (CONCAT(@@sql_mode , ',', 'ANSI_QUOTES'));"; + + /** + * Builds the {@code SET SESSION sql_mode} initialization query, escaping backslashes and single quotes so that a + * user-supplied mode value cannot break out of the string literal. Unlike SQL Server, MySQL treats a backslash as + * an escape character inside string literals by default, so backslashes are doubled before single quotes are doubled. + */ + public static String buildSetSqlModeQuery(String sqlMode) { + return String.format(SET_SQL_MODE_QUERY_FORMAT, sqlMode.replace("\\", "\\\\").replace("'", "''")); + } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlPostAction.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlPostAction.java index 6ae62a3cf..9b3cd3990 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlPostAction.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlPostAction.java @@ -115,7 +115,7 @@ public List getInitQueries() { initQueries.add(MysqlConstants.ANSI_QUOTES_QUERY); } if (!Strings.isNullOrEmpty(sqlMode)) { - initQueries.add(String.format(MysqlConstants.SET_SQL_MODE_QUERY_FORMAT, sqlMode)); + initQueries.add(MysqlConstants.buildSetSqlModeQuery(sqlMode)); } return initQueries; } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 6487ffae6..32b6e6c8e 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -217,7 +217,7 @@ public void validate(FailureCollector collector) { @Override public List getInitQueries() { if (!Strings.isNullOrEmpty(sqlMode)) { - return Collections.singletonList(String.format(MysqlConstants.SET_SQL_MODE_QUERY_FORMAT, sqlMode)); + return Collections.singletonList(MysqlConstants.buildSetSqlModeQuery(sqlMode)); } return Collections.emptyList(); } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index d7d7c400b..c3e7f479b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -191,7 +191,7 @@ public List getInitQueries() { initQueries.add(MysqlConstants.ANSI_QUOTES_QUERY); } if (!Strings.isNullOrEmpty(sqlMode)) { - initQueries.add(String.format(MysqlConstants.SET_SQL_MODE_QUERY_FORMAT, sqlMode)); + initQueries.add(MysqlConstants.buildSetSqlModeQuery(sqlMode)); } return initQueries; } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java index df731d194..d9d0d1717 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java @@ -42,6 +42,25 @@ public void getTableNameTest() { CONNECTOR.getTableName("db", "schema", "table")); } + /** + * Unit test for getTableName() with identifiers containing backticks + */ + @Test + public void getTableNameEscapesBackticksTest() { + Assert.assertEquals("`my``db`.`t``; DELETE FROM x; --`", + CONNECTOR.getTableName("my`db", "schema", "t`; DELETE FROM x; --")); + } + + /** + * Unit test for getStratifiedQuery() with a strata column containing backticks + */ + @Test + public void getStratifiedQueryEscapesStrataTest() { + String query = CONNECTOR.getStratifiedQuery("`db`.`table`", 10, "id`; DROP TABLE t; --", "s1"); + Assert.assertTrue(query, query.contains("ORDER BY `id``; DROP TABLE t; --`, RAND()")); + Assert.assertTrue(query, query.contains("ORDER BY `id``; DROP TABLE t; --`\n")); + } + /** * Unit tests for getTableQuery() */ diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlInitQueriesUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlInitQueriesUnitTest.java new file mode 100644 index 000000000..e9151a420 --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlInitQueriesUnitTest.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +/** + * Unit tests for {@code getInitQueries()} escaping across MySQL plugin configs. + */ +public class MysqlInitQueriesUnitTest { + + @Test + public void getInitQueries_emptyOrNullSqlMode_returnsEmptyList() { + MysqlSource.MysqlSourceConfig sourceConfig = new MysqlSource.MysqlSourceConfig(); + MysqlSink.MysqlSinkConfig sinkConfig = new MysqlSink.MysqlSinkConfig(); + MysqlAction.MysqlActionConfig actionConfig = new MysqlAction.MysqlActionConfig(); + MysqlPostAction.MysqlQueryActionConfig postActionConfig = + new MysqlPostAction.MysqlQueryActionConfig(); + + Assert.assertEquals(Collections.emptyList(), sourceConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), sinkConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), actionConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), postActionConfig.getInitQueries()); + + sourceConfig.sqlMode = ""; + sinkConfig.sqlMode = ""; + actionConfig.sqlMode = ""; + postActionConfig.sqlMode = ""; + + Assert.assertEquals(Collections.emptyList(), sourceConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), sinkConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), actionConfig.getInitQueries()); + Assert.assertEquals(Collections.emptyList(), postActionConfig.getInitQueries()); + } + + @Test + public void getInitQueries_plainSqlModeAndAnsiQuotes_formatsInitQueries() { + String expectedSqlMode = "SET SESSION sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_DATE';"; + + MysqlSource.MysqlSourceConfig sourceConfig = new MysqlSource.MysqlSourceConfig(); + sourceConfig.useAnsiQuotes = true; + sourceConfig.sqlMode = "STRICT_TRANS_TABLES,NO_ZERO_DATE"; + Assert.assertEquals(Arrays.asList(MysqlConstants.ANSI_QUOTES_QUERY, expectedSqlMode), + sourceConfig.getInitQueries()); + + MysqlSink.MysqlSinkConfig sinkConfig = new MysqlSink.MysqlSinkConfig(); + sinkConfig.sqlMode = "STRICT_TRANS_TABLES,NO_ZERO_DATE"; + Assert.assertEquals(Collections.singletonList(expectedSqlMode), sinkConfig.getInitQueries()); + + MysqlAction.MysqlActionConfig actionConfig = new MysqlAction.MysqlActionConfig(); + actionConfig.useAnsiQuotes = true; + actionConfig.sqlMode = "STRICT_TRANS_TABLES,NO_ZERO_DATE"; + Assert.assertEquals(Arrays.asList(MysqlConstants.ANSI_QUOTES_QUERY, expectedSqlMode), + actionConfig.getInitQueries()); + + MysqlPostAction.MysqlQueryActionConfig postActionConfig = + new MysqlPostAction.MysqlQueryActionConfig(); + postActionConfig.useAnsiQuotes = true; + postActionConfig.sqlMode = "STRICT_TRANS_TABLES,NO_ZERO_DATE"; + Assert.assertEquals(Arrays.asList(MysqlConstants.ANSI_QUOTES_QUERY, expectedSqlMode), + postActionConfig.getInitQueries()); + } + + @Test + public void getInitQueries_quotesAndBackslashes_escapesBackslashesBeforeQuotes() { + String payload = "TRADITIONAL\\'; DELETE FROM t; --"; + String expected = "SET SESSION sql_mode = 'TRADITIONAL\\\\''; DELETE FROM t; --';"; + + MysqlSource.MysqlSourceConfig sourceConfig = new MysqlSource.MysqlSourceConfig(); + sourceConfig.sqlMode = payload; + Assert.assertEquals(Collections.singletonList(expected), sourceConfig.getInitQueries()); + + MysqlSink.MysqlSinkConfig sinkConfig = new MysqlSink.MysqlSinkConfig(); + sinkConfig.sqlMode = payload; + Assert.assertEquals(Collections.singletonList(expected), sinkConfig.getInitQueries()); + + MysqlAction.MysqlActionConfig actionConfig = new MysqlAction.MysqlActionConfig(); + actionConfig.sqlMode = payload; + Assert.assertEquals(Collections.singletonList(expected), actionConfig.getInitQueries()); + + MysqlPostAction.MysqlQueryActionConfig postActionConfig = + new MysqlPostAction.MysqlQueryActionConfig(); + postActionConfig.sqlMode = payload; + Assert.assertEquals(Collections.singletonList(expected), postActionConfig.getInitQueries()); + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 469ca45f2..a59c0688e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -135,7 +135,7 @@ protected String getConnectionString(@Nullable String database) { @Override protected String getTableName(String database, String schema, String table) { - return String.format("\"%s\".\"%s\"", schema, table); + return quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); } @Override @@ -157,6 +157,7 @@ protected String getRandomQuery(String tableName, int limit) { @Override protected String getStratifiedQuery(String tableName, int limit, String strata, String sessionID) { + String strataColumn = quoteIdentifier(strata, '"'); return String.format("WITH t_%s AS (\n" + " SELECT %s.*,\n" + " ROW_NUMBER() OVER (ORDER BY %s, dbms_random.random) AS s_%s,\n" + @@ -166,8 +167,8 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, " )\n" + "SELECT * FROM t_%s\n" + "WHERE MOD(s_%s, GREATEST(1, CAST(c_%s / %d AS INTEGER))) = 1 AND ROWNUM <= %d", - sessionID, tableName, strata, sessionID, tableName, sessionID, tableName, - strata, sessionID, sessionID, sessionID, limit, limit); + sessionID, tableName, strataColumn, sessionID, tableName, sessionID, tableName, + strataColumn, sessionID, sessionID, sessionID, limit, limit); } @Override diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java index f89a6c1d7..c73eea17f 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java @@ -37,6 +37,25 @@ public void getTableNameTest() { CONNECTOR.getTableName("db", "schema", "table")); } + /** + * Unit test for getTableName() with identifiers containing double quotes + */ + @Test + public void getTableNameEscapesQuotesTest() { + Assert.assertEquals("\"MY\"\"SCHEMA\".\"T\"\" WHERE 1=1 --\"", + CONNECTOR.getTableName("db", "MY\"SCHEMA", "T\" WHERE 1=1 --")); + } + + /** + * Unit test for getStratifiedQuery() with a strata column containing double quotes + */ + @Test + public void getStratifiedQueryEscapesStrataTest() { + String query = CONNECTOR.getStratifiedQuery("\"SCHEMA\".\"TABLE\"", 10, "ID\", 1) --", "s1"); + Assert.assertTrue(query, query.contains("ORDER BY \"ID\"\", 1) --\", dbms_random.random")); + Assert.assertTrue(query, query.contains("ORDER BY \"ID\"\", 1) --\" )")); + } + /** * Unit tests for getTableQuery() */ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index 5b23ce6cd..4ad703b61 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -117,7 +117,7 @@ public StructuredRecord transform(LongWritable longWritable, PostgresDBRecord re @Override protected String getTableName(String database, String schema, String table) { - return String.format("\"%s\".\"%s\"", schema, table); + return quoteIdentifier(schema, '"') + "." + quoteIdentifier(table, '"'); } @Override @@ -129,6 +129,7 @@ protected String getRandomQuery(String tableName, int limit) { @Override protected String getStratifiedQuery(String tableName, int limit, String strata, String sessionID) { + String strataColumn = quoteIdentifier(strata, '"'); // this query times out on larger datasets; would benefit from optimization return String.format("WITH t_%s AS (\n" + " SELECT *,\n" + @@ -140,8 +141,8 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, "WHERE mod(sqn_%s, GREATEST(1, CAST(c_%s / %d AS bigint))) = 1\n" + "ORDER BY %s\n" + "LIMIT %d", - sessionID, strata, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, - limit, strata, limit); + sessionID, strataColumn, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, + limit, strataColumn, limit); } @Override diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java index f286cf574..47c5f5e94 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java @@ -41,6 +41,28 @@ public void getTableNameTest() { CONNECTOR.getTableName("db", "schema", "table")); } + /** + * Unit test for getTableName() with identifiers containing double quotes + */ + @Test + public void getTableNameEscapesQuotesTest() { + Assert.assertEquals("\"pg_catalog\".\"pg_class\"\" LIMIT 0; UPDATE t SET v='x'; SELECT * FROM \"\"pg_class\"", + CONNECTOR.getTableName("db", "pg_catalog", + "pg_class\" LIMIT 0; UPDATE t SET v='x'; SELECT * FROM \"pg_class")); + Assert.assertEquals("\"my\"\"schema\".\"my\"\"table\"", + CONNECTOR.getTableName("db", "my\"schema", "my\"table")); + } + + /** + * Unit test for getStratifiedQuery() with a strata column containing double quotes + */ + @Test + public void getStratifiedQueryEscapesStrataTest() { + String query = CONNECTOR.getStratifiedQuery("\"schema\".\"table\"", 10, "id\"; DROP TABLE t; --", "s1"); + Assert.assertTrue(query, query.contains("ORDER BY \"id\"\"; DROP TABLE t; --\", random()")); + Assert.assertTrue(query, query.contains("ORDER BY \"id\"\"; DROP TABLE t; --\"\n")); + } + /** * Unit tests for getTableQuery() */ From 89b2ef28f2a9ca96a24596e5a293a46bbb0d3d5e Mon Sep 17 00:00:00 2001 From: vishwasvaidya-cloudsufi Date: Sat, 21 Nov 2026 17:30:16 +0000 Subject: [PATCH 2/2] Validate source queries and split-by field as read-only SELECT --- .../docs/Redshift-batchsource.md | 10 +- .../docs/Redshift-connector.md | 5 + .../widgets/Redshift-batchsource.json | 8 + .../widgets/Redshift-connector.json | 19 + .../docs/AuroraMysql-batchsource.md | 10 +- .../auroradb/mysql/AuroraMysqlSource.java | 11 + .../widgets/AuroraMysql-batchsource.json | 8 + .../docs/AuroraPostgres-batchsource.md | 10 +- .../postgres/AuroraPostgresSource.java | 11 + .../widgets/AuroraPostgres-batchsource.json | 8 + .../docs/CloudSQLMySQL-batchsource.md | 10 +- .../docs/CloudSQLMySQL-connector.md | 4 + .../cloudsql/mysql/CloudSQLMySQLSource.java | 13 + .../widgets/CloudSQLMySQL-batchsink.json | 8 + .../widgets/CloudSQLMySQL-batchsource.json | 8 + .../widgets/CloudSQLMySQL-connector.json | 19 + .../docs/CloudSQLPostgreSQL-batchsource.md | 10 +- .../docs/CloudSQLPostgreSQL-connector.md | 4 + .../postgres/CloudSQLPostgreSQLSource.java | 12 + .../widgets/CloudSQLPostgreSQL-batchsink.json | 8 + .../CloudSQLPostgreSQL-batchsource.json | 8 + .../widgets/CloudSQLPostgreSQL-connector.json | 19 + database-commons/pom.xml | 4 + .../io/cdap/plugin/db/ConnectionConfig.java | 1 + .../AbstractDBSpecificSourceConfig.java | 26 +- .../db/config/DatabaseSourceConfig.java | 7 + .../connector/AbstractDBConnectorConfig.java | 10 + .../plugin/db/source/AbstractDBSource.java | 170 +++- .../plugin/util/SelectQueryValidator.java | 887 ++++++++++++++++++ .../db/source/AbstractDBSourceTest.java | 223 ++++- .../plugin/util/SelectQueryValidatorTest.java | 477 ++++++++++ .../docs/Databricks-batchsource.md | 4 +- .../docs/Databricks-connector.md | 2 + .../plugin/databricks/DatabricksSource.java | 6 + .../widgets/Databricks-batchsource.json | 8 + .../widgets/Databricks-connector.json | 19 + db2-plugin/docs/Db2-batchsource.md | 10 +- .../java/io/cdap/plugin/db2/Db2Source.java | 6 + db2-plugin/widgets/Db2-batchsource.json | 8 + .../docs/Database-batchsource.md | 10 +- .../io/cdap/plugin/jdbc/DatabaseSource.java | 11 + .../cdap/plugin/db/sink/DBSourceTestRun.java | 314 +++++++ .../cdap/plugin/jdbc/DatabaseSourceTest.java | 84 ++ .../widgets/Database-batchsource.json | 8 + mariadb-plugin/docs/Mariadb-batchsource.md | 10 +- .../io/cdap/plugin/mariadb/MariadbSource.java | 11 + .../widgets/Mariadb-batchsource.json | 8 + memsql-plugin/docs/Memsql-batchsource.md | 10 +- .../plugin/memsql/source/MemsqlSource.java | 11 + memsql-plugin/widgets/Memsql-batchsource.json | 8 + mssql-plugin/docs/SQL Server-connector.md | 4 + mssql-plugin/docs/SqlServer-batchsource.md | 10 +- .../resources/errorMessage.properties | 2 +- .../resources/pluginParameters.properties | 1 + .../io/cdap/plugin/mssql/SqlServerSource.java | 6 + .../widgets/SQL Server-connector.json | 19 + mssql-plugin/widgets/SqlServer-batchsink.json | 8 + .../widgets/SqlServer-batchsource.json | 8 + mysql-plugin/docs/MySQL-connector.md | 4 + mysql-plugin/docs/Mysql-batchsource.md | 10 +- .../io/cdap/plugin/mysql/MysqlSource.java | 17 + .../plugin/mysql/MysqlSourceUnitTest.java | 52 + mysql-plugin/widgets/MySQL-connector.json | 19 + mysql-plugin/widgets/Mysql-batchsink.json | 8 + mysql-plugin/widgets/Mysql-batchsource.json | 8 + netezza-plugin/docs/Netezza-batchsource.md | 10 +- .../widgets/Netezza-batchsource.json | 8 + oracle-plugin/docs/Oracle-batchsource.md | 10 +- oracle-plugin/docs/Oracle-connector.md | 4 + .../io/cdap/plugin/oracle/OracleSource.java | 6 + oracle-plugin/widgets/Oracle-batchsink.json | 8 + oracle-plugin/widgets/Oracle-batchsource.json | 8 + oracle-plugin/widgets/Oracle-connector.json | 19 + pom.xml | 6 + .../docs/PostgreSQL-connector.md | 4 + .../docs/Postgres-batchsource.md | 10 +- .../cdap/plugin/postgres/PostgresSource.java | 12 + .../widgets/PostgreSQL-connector.json | 19 + .../widgets/Postgres-batchsink.json | 8 + .../widgets/Postgres-batchsource.json | 8 + saphana-plugin/docs/SapHana-batchsource.md | 10 +- .../io/cdap/plugin/saphana/SapHanaSource.java | 6 + .../widgets/SapHana-batchsource.json | 8 + teradata-plugin/docs/Teradata-batchsource.md | 10 +- .../teradata/source/TeradataSource.java | 6 + .../widgets/Teradata-batchsource.json | 8 + 86 files changed, 2864 insertions(+), 76 deletions(-) create mode 100644 database-commons/src/main/java/io/cdap/plugin/util/SelectQueryValidator.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/util/SelectQueryValidatorTest.java create mode 100644 generic-database-plugin/src/test/java/io/cdap/plugin/jdbc/DatabaseSourceTest.java create mode 100644 mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSourceUnitTest.java diff --git a/amazon-redshift-plugin/docs/Redshift-batchsource.md b/amazon-redshift-plugin/docs/Redshift-batchsource.md index 38873b15a..2f33d1010 100644 --- a/amazon-redshift-plugin/docs/Redshift-batchsource.md +++ b/amazon-redshift-plugin/docs/Redshift-batchsource.md @@ -25,16 +25,20 @@ Properties **Database:** Redshift database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/amazon-redshift-plugin/docs/Redshift-connector.md b/amazon-redshift-plugin/docs/Redshift-connector.md index 368d9e09f..538736617 100644 --- a/amazon-redshift-plugin/docs/Redshift-connector.md +++ b/amazon-redshift-plugin/docs/Redshift-connector.md @@ -24,3 +24,8 @@ Properties **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. + +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json index 586a8993b..9ad641348 100644 --- a/amazon-redshift-plugin/widgets/Redshift-batchsource.json +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -185,6 +185,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json index f392e3a78..a07012786 100644 --- a/amazon-redshift-plugin/widgets/Redshift-connector.json +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -67,6 +67,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/aurora-mysql-plugin/docs/AuroraMysql-batchsource.md b/aurora-mysql-plugin/docs/AuroraMysql-batchsource.md index 5d7ffd00c..b76993046 100644 --- a/aurora-mysql-plugin/docs/AuroraMysql-batchsource.md +++ b/aurora-mysql-plugin/docs/AuroraMysql-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** Aurora DB database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/aurora-mysql-plugin/src/main/java/io/cdap/plugin/auroradb/mysql/AuroraMysqlSource.java b/aurora-mysql-plugin/src/main/java/io/cdap/plugin/auroradb/mysql/AuroraMysqlSource.java index 955d0be9e..3439c0c25 100644 --- a/aurora-mysql-plugin/src/main/java/io/cdap/plugin/auroradb/mysql/AuroraMysqlSource.java +++ b/aurora-mysql-plugin/src/main/java/io/cdap/plugin/auroradb/mysql/AuroraMysqlSource.java @@ -21,8 +21,10 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.SelectQueryValidator; import java.util.Map; import javax.annotation.Nullable; @@ -49,6 +51,15 @@ protected String createConnectionString() { auroraMysqlSourceConfig.host, auroraMysqlSourceConfig.port, auroraMysqlSourceConfig.database); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (auroraMysqlSourceConfig.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + || SelectQueryValidator.Dialect.mayOverrideMysqlSqlMode(auroraMysqlSourceConfig.connectionArguments)) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + return SelectQueryValidator.Dialect.MYSQL; + } + /** * Aurora DB MySQL source config. */ diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json index bd2bb88a9..87a910da2 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json @@ -164,6 +164,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/aurora-postgresql-plugin/docs/AuroraPostgres-batchsource.md b/aurora-postgresql-plugin/docs/AuroraPostgres-batchsource.md index b7474bd16..9c767a35f 100644 --- a/aurora-postgresql-plugin/docs/AuroraPostgres-batchsource.md +++ b/aurora-postgresql-plugin/docs/AuroraPostgres-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** Aurora DB database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/aurora-postgresql-plugin/src/main/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresSource.java b/aurora-postgresql-plugin/src/main/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresSource.java index 5b39b5cf5..c68b2b1c4 100644 --- a/aurora-postgresql-plugin/src/main/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresSource.java +++ b/aurora-postgresql-plugin/src/main/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresSource.java @@ -21,9 +21,11 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Map; @@ -52,6 +54,15 @@ protected String createConnectionString() { auroraPostgresSourceConfig.database); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (auroraPostgresSourceConfig.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS)) { + return SelectQueryValidator.Dialect.UNKNOWN; + } + return SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:postgresql:", + auroraPostgresSourceConfig.connectionArguments); + } + @Override protected SchemaReader getSchemaReader() { return new AuroraPostgresSchemaReader(); diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json index fc2503c67..8847eff69 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json @@ -153,6 +153,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md index 11a863f4f..95f29b960 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-batchsource.md @@ -35,16 +35,20 @@ Can be found in the instance overview page. **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md index 9cd5d582a..07658b4f0 100644 --- a/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md +++ b/cloudsql-mysql-plugin/docs/CloudSQLMySQL-connector.md @@ -41,6 +41,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 43c63aa9e..7fb61964d 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -32,6 +32,7 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; @@ -39,6 +40,7 @@ import io.cdap.plugin.mysql.MysqlSchemaReader; import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Collections; @@ -83,6 +85,17 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + CloudSQLMySQLConnectorConfig connection = cloudsqlMysqlSourceConfig.getConnection(); + if (connection != null + && (connection.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + || SelectQueryValidator.Dialect.mayOverrideMysqlSqlMode(connection.getConnectionArguments()))) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + return SelectQueryValidator.Dialect.MYSQL; + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json index 3a3277ed8..771657ca2 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json @@ -205,6 +205,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json index 20c6d872f..4444bfbff 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json @@ -218,6 +218,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index e641add41..7de84190f 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -106,6 +106,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md index c477f4a23..11b234345 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-batchsource.md @@ -35,16 +35,20 @@ Can be found in the instance overview page. **CloudSQL Instance Type:** Whether the CloudSQL instance to connect to is private or public. Defaults to 'Public'. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md index 467469c30..bce27f6b0 100644 --- a/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md +++ b/cloudsql-postgresql-plugin/docs/CloudSQLPostgreSQL-connector.md @@ -41,6 +41,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index e07251a56..56b368287 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -32,6 +32,7 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; @@ -39,6 +40,7 @@ import io.cdap.plugin.postgres.PostgresSchemaReader; import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Collections; @@ -83,6 +85,16 @@ protected SchemaReader getSchemaReader() { return new PostgresSchemaReader(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + CloudSQLPostgreSQLConnectorConfig connection = cloudsqlPostgresqlSourceConfig.getConnection(); + if (connection != null && connection.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS)) { + return SelectQueryValidator.Dialect.UNKNOWN; + } + String args = connection == null ? null : connection.getConnectionArguments(); + return SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:postgresql:", args); + } + @Override protected Class getDBRecordType() { return PostgresDBRecord.class; diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 8d6578413..0c103d403 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -221,6 +221,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 1308b83e0..06e61167c 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -217,6 +217,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index 6b89ae657..fe6d36cdd 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -105,6 +105,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 00f7a714d..53644cdd0 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -77,6 +77,10 @@ dev.failsafe failsafe + + com.github.jsqlparser + jsqlparser + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 2a5aaa988..d817046e5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -47,6 +47,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + public static final String DISABLE_QUERY_VALIDATION = "disableQueryValidation"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 73d57d356..efc3e7921 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -28,6 +28,7 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.RetryUtils; import java.io.IOException; import java.util.Collections; @@ -56,17 +57,17 @@ public abstract class AbstractDBSpecificSourceConfig extends PluginConfig implem public String referenceName; @Name(IMPORT_QUERY) - @Description("The SELECT query to use to import data from the specified table. " + + @Description("The single read-only SELECT query to use to import data from the specified table. " + "You can specify an arbitrary number of columns to import, or import all columns using *. " + "The Query should contain the '$CONDITIONS' string unless numSplits is set to one. " + - "For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string" + + "For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string " + "will be replaced by 'splitBy' field limits specified by the bounding query.") @Macro protected String importQuery; @Nullable @Name(BOUNDING_QUERY) - @Description("Bounding Query should return the min and max of the " + + @Description("Single read-only SELECT query that returns the min and max of the " + "values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. " + "This is required unless numSplits is set to one.") @Macro @@ -74,7 +75,7 @@ public abstract class AbstractDBSpecificSourceConfig extends PluginConfig implem @Nullable @Name(SPLIT_BY) - @Description("Field Name which will be used to generate splits. This is required unless numSplits is set to one.") + @Description("Single column name used to generate splits. This is required unless numSplits is set to one.") @Macro protected String splitBy; @@ -261,7 +262,8 @@ protected String cleanQuery(@Nullable String query) { @Override public boolean canConnect() { - return !containsMacro(AbstractDBSource.DBSourceConfig.IMPORT_QUERY) && getConnection().canConnect(); + return !containsMacro(AbstractDBSource.DBSourceConfig.IMPORT_QUERY) + && getConnection() != null && getConnection().canConnect(); } @Override @@ -269,16 +271,24 @@ public Integer getFetchSize() { return fetchSize; } + @Override + public boolean isQueryValidationDisabled() { + return getConnection() != null && getConnection().isQueryValidationDisabled(); + } + public Integer getInitialRetryDuration() { - return getConnection().getInitialRetryDuration(); + return getConnection() == null + ? RetryUtils.DEFAULT_INITIAL_RETRY_DURATION_SECONDS : getConnection().getInitialRetryDuration(); } public Integer getMaxRetryDuration() { - return getConnection().getMaxRetryDuration(); + return getConnection() == null + ? RetryUtils.DEFAULT_MAX_RETRY_DURATION_SECONDS : getConnection().getMaxRetryDuration(); } public Integer getMaxRetryCount() { - return getConnection().getMaxRetryCount(); + return getConnection() == null + ? RetryUtils.DEFAULT_MAX_RETRY_COUNT : getConnection().getMaxRetryCount(); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseSourceConfig.java index 8987377b9..677358615 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseSourceConfig.java @@ -90,4 +90,11 @@ public interface DatabaseSourceConfig extends DatabaseConnectionConfig { * @return the number of rows to fetch at a time per split */ Integer getFetchSize(); + + /** + * @return true if read-only SELECT validation on source queries is disabled for backward compatibility + */ + default boolean isQueryValidationDisabled() { + return false; + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 646c5e388..0892f862d 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -83,6 +83,12 @@ public abstract class AbstractDBConnectorConfig extends PluginConfig implements @Macro private Integer maxRetryCount; + @Name(ConnectionConfig.DISABLE_QUERY_VALIDATION) + @Description("Disables read-only SELECT validation and read-only session enforcement on batch source queries " + + "using this connection. Enable only for backward compatibility with trusted existing pipelines. Defaults to false.") + @Nullable + protected Boolean disableQueryValidation; + @Nullable @Override @@ -108,6 +114,10 @@ public Integer getMaxRetryCount() { return maxRetryCount == null ? RetryUtils.DEFAULT_MAX_RETRY_COUNT : maxRetryCount; } + public boolean isQueryValidationDisabled() { + return Boolean.TRUE.equals(disableQueryValidation); + } + @Override public Properties getConnectionArgumentsProperties() { return getConnectionArgumentsProperties(connectionArguments, user, password); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 908d27eb9..c3e586c56 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -35,6 +35,7 @@ import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.cdap.internal.io.SchemaTypeAdapter; +import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.ReferenceBatchSource; import io.cdap.plugin.common.ReferencePluginConfig; @@ -52,6 +53,7 @@ import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; import io.cdap.plugin.util.RetryUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -106,6 +108,7 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); FailureCollector collector = stageConfigurer.getFailureCollector(); sourceConfig.validate(collector); + validateQueries(collector); if (sourceConfig.getSchema() != null) { stageConfigurer.setOutputSchema(sourceConfig.getSchema()); return; @@ -119,9 +122,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { Class driverClass = DBUtils.getDriverClass( pipelineConfigurer, sourceConfig, ConnectionConfig.JDBC_PLUGIN_TYPE); - if (sourceConfig.canConnect()) { + // Do not open a connection or run init queries if source or query validation already failed. + if (sourceConfig.canConnect() && collector.getValidationFailures().isEmpty()) { try { - stageConfigurer.setOutputSchema(getSchema(driverClass)); + stageConfigurer.setOutputSchema(fetchSchema(driverClass)); } catch (IllegalAccessException | InstantiationException e) { collector.addFailure("Unable to instantiate JDBC driver: " + e.getMessage(), null) .withStacktrace(e.getStackTrace()); @@ -136,12 +140,18 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { } public Schema getSchema(Class driverClass) throws IllegalAccessException, + SQLException, InstantiationException { + validateImportQueryIfEnabled(sourceConfig.getImportQuery()); + return fetchSchema(driverClass); + } + + private Schema fetchSchema(Class driverClass) throws IllegalAccessException, SQLException, InstantiationException { DriverCleanup driverCleanup; try { driverCleanup = loadPluginClassAndGetDriver(driverClass); try { - return getSchema(); + return fetchSchemaFromConnection(); } finally { driverCleanup.destroy(); } @@ -161,21 +171,76 @@ public Schema getSchema(Class driverClass) throws IllegalAcces * 3. Error while running the import query. */ public Schema getSchema() throws SQLException { + // Validate the import query before opening a connection or running init queries when called directly. + validateImportQueryIfEnabled(sourceConfig.getImportQuery()); + return fetchSchemaFromConnection(); + } + + private Schema fetchSchemaFromConnection() throws SQLException { try (Connection connection = getConnection()) { + return executeSchemaQuery(connection); + } + } + + private void validateImportQueryIfEnabled(@Nullable String query) { + if (sourceConfig.isQueryValidationDisabled()) { + return; + } + String finalQuery = query != null && query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + try { + SelectQueryValidator.validate(finalQuery, getSqlDialect()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid Import Query: " + e.getMessage(), e); + } + } + + @VisibleForTesting + Schema executeSchemaQuery(Connection connection) throws SQLException { + boolean rollbackOnExit = configureReadOnlyTransaction(connection); + try { executeInitQueries(connection, sourceConfig.getInitQueries()); - String query = sourceConfig.getImportQuery(); - return loadSchemaFromDB(connection, query); + return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + } finally { + if (rollbackOnExit) { + try { + connection.rollback(); + } catch (SQLException e) { + LOG.debug("Failed to roll back read-only schema query transaction.", e); + } + } + } + } + + private boolean configureReadOnlyTransaction(@Nullable Connection connection) { + if (connection == null || sourceConfig.isQueryValidationDisabled()) { + return false; + } + try { + connection.setReadOnly(true); + } catch (SQLException e) { + LOG.debug("Driver does not support setting connection read-only; continuing.", e); + } + if ("TRANSACTION_NONE".equals(sourceConfig.getTransactionIsolationLevel())) { + return false; + } + try { + connection.setAutoCommit(false); + return true; + } catch (SQLException e) { + LOG.debug("Driver does not support disabling auto-commit; continuing.", e); + return false; } } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, getErrorDetailsProvider())) { statement.setMaxRows(1); - String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; - ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, - finalQuery, getErrorDetailsProvider()); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + try (ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + finalQuery, getErrorDetailsProvider())) { + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + } } } @@ -198,8 +263,7 @@ private Schema loadSchemaFromDB(Class driverClass) connectionProperties.putAll(sourceConfig.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, connectionString, connectionProperties, getErrorDetailsProvider())) { - executeInitQueries(connection, sourceConfig.getInitQueries()); - return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + return executeSchemaQuery(connection); } finally { driverCleanup.destroy(); } @@ -228,6 +292,67 @@ protected String getErrorDetailsProviderClassName() { return null; } + /** + * Returns the SQL dialect used to validate source queries and split-by column references. Subclasses should override + * this with their database's {@link SelectQueryValidator.Dialect}. Defaults to + * {@link SelectQueryValidator.Dialect#UNKNOWN}, which conservatively requires queries to be valid across dialects. + */ + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.UNKNOWN; + } + + /** + * Validates that the import query, bounding query, and split-by field are single read-only SELECT statements and + * column references before any query is executed with the connection's credentials. + */ + private void validateQueries(FailureCollector collector) { + // If the connection itself is an unresolved macro at configure time, defer until prepareRun when the connection + // (including its disableQueryValidation flag) is resolved; no connection is opened at configure time in that case. + if (sourceConfig.isQueryValidationDisabled() || sourceConfig.containsMacro(ConfigUtil.NAME_CONNECTION)) { + return; + } + validateQuery(DBSourceConfig.IMPORT_QUERY, "Import Query", sourceConfig.getImportQuery(), collector); + validateQuery(DBSourceConfig.BOUNDING_QUERY, "Bounding Query", sourceConfig.getBoundingQuery(), collector); + validateSplitBy(collector); + } + + private void validateQuery(String property, String label, @Nullable String query, FailureCollector collector) { + // A macro is not resolved at configure time; it is validated at runtime once it holds a concrete value. + if (sourceConfig.containsMacro(property) || Strings.isNullOrEmpty(query)) { + return; + } + try { + SelectQueryValidator.validate(query, getSqlDialect()); + } catch (IllegalArgumentException e) { + collector.addFailure(String.format("Invalid %s: %s", label, e.getMessage()), + "Provide a single SELECT statement that only reads data.") + .withConfigProperty(property); + } + } + + /** + * Validates that the split-by field is a single column name. When numSplits is not 1, the split-by value is + * concatenated into the generated split bound clauses at runtime (via {@code $CONDITIONS} substitution) and + * executed with the connection credentials, so it must not carry extra SQL. + */ + private void validateSplitBy(FailureCollector collector) { + if (sourceConfig.containsMacro(DBSourceConfig.SPLIT_BY) || sourceConfig.containsMacro(DBSourceConfig.NUM_SPLITS)) { + return; + } + Integer numSplits = sourceConfig.getNumSplits(); + String splitBy = sourceConfig.getSplitBy(); + if ((numSplits != null && numSplits == 1) || Strings.isNullOrEmpty(splitBy)) { + return; + } + try { + SelectQueryValidator.validateColumnReference(splitBy, getSqlDialect()); + } catch (IllegalArgumentException e) { + collector.addFailure(String.format("Invalid Split-By Field Name: %s", e.getMessage()), + "Provide a single column name.") + .withConfigProperty(DBSourceConfig.SPLIT_BY); + } + } + /** * Returns the DBErrorDetailsProvider instance. * Override this method to provide a custom DBErrorDetailsProvider instance. @@ -272,6 +397,7 @@ private Connection getConnection() { public void prepareRun(BatchSourceContext context) throws Exception { FailureCollector collector = context.getFailureCollector(); sourceConfig.validate(collector); + validateQueries(collector); collector.getOrThrowException(); LOG.debug("pluginType = {}; pluginName = {}; connectionString = {}; importQuery = {}; " + @@ -337,6 +463,9 @@ public ConnectionConfigAccessor getConnectionConfigAccessor (String driverClassN DataDrivenETLDBInputFormat.setInput(connectionConfigAccessor.getConfiguration(), getDBRecordType(), sourceConfig.getImportQuery(), sourceConfig.getBoundingQuery(), false); + if (!sourceConfig.isQueryValidationDisabled()) { + connectionConfigAccessor.setReadOnly(true); + } if (sourceConfig.getTransactionIsolationLevel() != null) { connectionConfigAccessor.setTransactionIsolationLevel(sourceConfig.getTransactionIsolationLevel()); @@ -410,17 +539,17 @@ public abstract static class DBSourceConfig extends DBConfig implements Database public static final String FETCH_SIZE = "fetchSize"; @Name(IMPORT_QUERY) - @Description("The SELECT query to use to import data from the specified table. " + + @Description("The single read-only SELECT query to use to import data from the specified table. " + "You can specify an arbitrary number of columns to import, or import all columns using *. " + "The Query should contain the '$CONDITIONS' string unless numSplits is set to one. " + - "For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string" + + "For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string " + "will be replaced by 'splitBy' field limits specified by the bounding query.") @Macro public String importQuery; @Nullable @Name(BOUNDING_QUERY) - @Description("Bounding Query should return the min and max of the " + + @Description("Single read-only SELECT query that returns the min and max of the " + "values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. " + "This is required unless numSplits is set to one.") @Macro @@ -428,7 +557,7 @@ public abstract static class DBSourceConfig extends DBConfig implements Database @Nullable @Name(SPLIT_BY) - @Description("Field Name which will be used to generate splits. This is required unless numSplits is set to one.") + @Description("Single column name used to generate splits. This is required unless numSplits is set to one.") @Macro public String splitBy; @@ -454,6 +583,12 @@ public abstract static class DBSourceConfig extends DBConfig implements Database "with the tradeoff of higher memory usage.") private Integer fetchSize; + @Nullable + @Name(ConnectionConfig.DISABLE_QUERY_VALIDATION) + @Description("Disables read-only SELECT validation and read-only session enforcement on source queries. " + + "Enable only for backward compatibility with trusted existing pipelines. Defaults to false.") + public Boolean disableQueryValidation; + public String getImportQuery() { return cleanQuery(importQuery); } @@ -472,6 +607,11 @@ public String getSplitBy() { return splitBy; } + @Override + public boolean isQueryValidationDisabled() { + return Boolean.TRUE.equals(disableQueryValidation); + } + public void validate(FailureCollector collector) { boolean hasOneSplit = false; if (!containsMacro(NUM_SPLITS) && numSplits != null) { diff --git a/database-commons/src/main/java/io/cdap/plugin/util/SelectQueryValidator.java b/database-commons/src/main/java/io/cdap/plugin/util/SelectQueryValidator.java new file mode 100644 index 000000000..b309147bf --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/SelectQueryValidator.java @@ -0,0 +1,887 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.parser.TokenMgrException; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.select.Select; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * Validates that a user-provided query is a single, read-only {@code SELECT} statement before it is executed with the + * credentials of a database connection. + * + *

The check is primarily lexical: the query is scanned with dialect-aware rules for string literals, comments, and + * quoted identifiers, and is required to be a single statement starting with {@code SELECT} or {@code WITH} (or a + * Teradata {@code LOCKING ... FOR ACCESS} modifier) that carries no write, DDL, administrative, {@code INTO}, or + * row/table locking clause outside quoted text. Constructs that parsers and database drivers lex differently when + * encountered outside literals and comments (PostgreSQL dollar-quoted strings, Oracle {@code q'...'} literals, and + * MySQL executable comments) are rejected outright. JSqlParser is run as a best-effort secondary check to confirm + * statement count and statement type when the query is within its grammar. + */ +public final class SelectQueryValidator { + + private static final long PARSE_TIMEOUT_MS = 10_000L; + // Placeholder replaced with the split condition at runtime; it is not valid SQL on its own. + private static final String CONDITIONS_PLACEHOLDER = "$CONDITIONS"; + private static final String STRING_PLACEHOLDER = "_S_"; + private static final String QUOTED_IDENT_PLACEHOLDER = "_Q_"; + + private static final ThreadFactory PARSER_THREAD_FACTORY = new ThreadFactoryBuilder() + .setDaemon(true).setNameFormat("select-query-validator-%d").build(); + // Bounded pool of daemon threads for JSqlParser timeout enforcement; idle threads terminate after keep-alive. + private static final ExecutorService PARSER_EXECUTOR = createParserExecutor(); + + private static ExecutorService createParserExecutor() { + int maxThreads = Math.max(2, Runtime.getRuntime().availableProcessors() * 2); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + maxThreads, maxThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), PARSER_THREAD_FACTORY); + executor.allowCoreThreadTimeOut(true); + return executor; + } + + // Matches a PostgreSQL dollar-quote delimiter ($$ or $tag$) anchored at the current scanner position. + private static final Pattern DOLLAR_QUOTE_AT_START = + Pattern.compile("^\\$([A-Za-z_\\u0080-\\uFFFF][A-Za-z0-9_\\u0080-\\uFFFF]*)?\\$"); + + // Keywords that perform writes, DDL, privilege changes, procedural execution, or server administration. + // A keyword in this set is rejected unless: + // 1. It is dot-qualified by an identifier (e.g. c.grant, t.update), so unquoted column names remain usable; + // 2. It is in FUNCTION_COLLIDING_WRITE_KEYWORDS (TRUNCATE, INSERT), not the first word, and followed by '('; + // 3. It is MERGE immediately followed by JOIN or UNION (SQL Server MERGE JOIN / OPTION (MERGE JOIN)); or + // 4. It is SET immediately preceded by CHARACTER (e.g. CAST(x AS CHAR CHARACTER SET utf8mb4)), not 'AS CHARACTER'. + private static final Set WRITE_KEYWORDS = new HashSet<>(Arrays.asList( + "INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT", "CREATE", "DROP", "ALTER", "TRUNCATE", "RENAME", + "GRANT", "REVOKE", "DENY", "EXEC", "EXECUTE", "INTO", "SET", "DBCC", "SHUTDOWN", "BACKUP", "RESTORE", + "KILL", "RECONFIGURE", "BULK", "LOCK", "WAITFOR")); + + // Of WRITE_KEYWORDS, only TRUNCATE and INSERT also name scalar functions (e.g. TRUNCATE(x, d), INSERT(s, p, l, n)). + private static final Set FUNCTION_COLLIDING_WRITE_KEYWORDS = + new HashSet<>(Arrays.asList("TRUNCATE", "INSERT")); + + // Tokens following FOR that acquire row/table locks (FOR UPDATE, FOR SHARE, FOR NO KEY UPDATE, FOR KEY SHARE). + // Read-only clauses such as FOR READ ONLY and FOR FETCH ONLY are allowed. + private static final Set LOCKING_AFTER_FOR = + new HashSet<>(Arrays.asList("UPDATE", "SHARE", "NO", "KEY")); + + // Allowed leading keywords for a read-only query. + private static final Set ALLOWED_START_KEYWORDS = + new HashSet<>(Arrays.asList("SELECT", "WITH", "LOCKING")); + + /** + * SQL dialect lexical profile used when scanning source queries and column references. + */ + public enum Dialect { + /** + * Standard ANSI SQL (e.g. Oracle, DB2, SAP HANA, Teradata, HSQLDB): single-quoted strings escape quotes only by + * doubling ({@code ''}), {@code "--"} starts a line comment unconditionally, and double quotes quote identifiers. + */ + ANSI(false), + + /** + * Microsoft SQL Server (T-SQL): single-quoted strings escape quotes only via {@code ''}, square brackets + * ({@code [...]}) and double quotes quote identifiers, and {@code /* ... *}{@code /} comments nest. + */ + TSQL(true), + + /** + * PostgreSQL: standard {@code '...'} literals do not treat {@code '\'} as an escape, whereas {@code E'...'} + * literals honor backslash escapes per literal, and {@code /* ... *}{@code /} comments nest. + */ + POSTGRESQL(false), + + /** + * MySQL, MariaDB, and SingleStore with default backslash escaping: {@code '\'} escapes quotes in literals, + * {@code '#'} starts a line comment, {@code "--"} starts a line comment only before whitespace/control/EOF, and + * queries containing {@code '"'} are checked under both string and {@code ANSI_QUOTES} identifier rules. + */ + MYSQL(false), + + /** + * MySQL family when {@code sql_mode} may be overridden (e.g. {@code NO_BACKSLASH_ESCAPES}): applies MySQL comment + * rules and requires the query to pass under both backslash-escape interpretations. + */ + MYSQL_AMBIGUOUS_BACKSLASH(false), + + /** + * Databricks / Spark SQL: {@code '\'} escapes quotes in string literals, {@code "--"} starts a line comment + * unconditionally, and queries containing {@code '"'} are checked under both backslash interpretations. + */ + DATABRICKS(false), + + /** + * Unknown or ambiguous dialect: requires the query to pass conservatively under ANSI, PostgreSQL {@code E'...'}, + * and MySQL (with and without backslash escapes) lexical rules. + */ + UNKNOWN(false); + + private final boolean squareBracketQuotes; + + Dialect(boolean squareBracketQuotes) { + this.squareBracketQuotes = squareBracketQuotes; + } + + /** + * Returns whether square brackets ({@code [...]}) quote identifiers in this dialect. + */ + public boolean usesSquareBracketQuotes() { + return squareBracketQuotes; + } + + /** + * Infers the {@link Dialect} from a JDBC URL and optional connection arguments, returning {@link #UNKNOWN} when + * the URL is null or does not match a known driver prefix. + */ + public static Dialect fromJdbcUrl(@Nullable String connectionString, @Nullable String connectionArguments) { + if (connectionString == null) { + return UNKNOWN; + } + String lower = connectionString.trim().toLowerCase(Locale.ROOT); + if (lower.startsWith("jdbc:sqlserver:") || lower.startsWith("jdbc:jtds:sqlserver:")) { + return TSQL; + } + if (lower.startsWith("jdbc:postgresql:") || lower.startsWith("jdbc:edb:")) { + if (mayOverrideStandardConformingStrings(lower) + || mayOverrideStandardConformingStrings(connectionArguments)) { + return UNKNOWN; + } + return POSTGRESQL; + } + if (lower.startsWith("jdbc:mysql:") || lower.startsWith("jdbc:mariadb:")) { + if (mayOverrideMysqlSqlMode(lower) || mayOverrideMysqlSqlMode(connectionArguments)) { + return MYSQL_AMBIGUOUS_BACKSLASH; + } + return MYSQL; + } + if (lower.startsWith("jdbc:databricks:")) { + return DATABRICKS; + } + if (lower.startsWith("jdbc:oracle:") || lower.startsWith("jdbc:db2:") + || lower.startsWith("jdbc:hsqldb:") || lower.startsWith("jdbc:h2:") + || lower.startsWith("jdbc:derby:") || lower.startsWith("jdbc:sap:") + || lower.startsWith("jdbc:teradata:")) { + return ANSI; + } + return UNKNOWN; + } + + /** + * Returns true if connection arguments may customize MySQL {@code sql_mode} (such as {@code NO_BACKSLASH_ESCAPES}). + */ + public static boolean mayOverrideMysqlSqlMode(@Nullable String text) { + if (text == null || text.isEmpty()) { + return false; + } + String lower = text.toLowerCase(Locale.ROOT); + return lower.contains("sql_mode") || lower.contains("no_backslash_escapes") + || lower.contains("sessionvariables"); + } + + private static boolean mayOverrideStandardConformingStrings(@Nullable String text) { + return text != null && text.toLowerCase(Locale.ROOT).contains("standard_conforming_strings"); + } + } + + private static final class ScanConfig { + final boolean squareBracketQuotes; + final boolean backslashEscapesInSingleQuotes; + final boolean backslashEscapesInDoubleQuotes; + final boolean postgresExtendedStrings; + final boolean hashComments; + final boolean mysqlDashDashComments; + final boolean nestedBlockComments; + + ScanConfig(boolean squareBracketQuotes, + boolean backslashEscapesInSingleQuotes, + boolean backslashEscapesInDoubleQuotes, + boolean postgresExtendedStrings, + boolean hashComments, + boolean mysqlDashDashComments, + boolean nestedBlockComments) { + this.squareBracketQuotes = squareBracketQuotes; + this.backslashEscapesInSingleQuotes = backslashEscapesInSingleQuotes; + this.backslashEscapesInDoubleQuotes = backslashEscapesInDoubleQuotes; + this.postgresExtendedStrings = postgresExtendedStrings; + this.hashComments = hashComments; + this.mysqlDashDashComments = mysqlDashDashComments; + this.nestedBlockComments = nestedBlockComments; + } + } + + private static final class WordToken { + final String word; + final int start; + final int end; + final boolean canQualifyColumn; + + WordToken(String word, int start, int end, boolean canQualifyColumn) { + this.word = word; + this.start = start; + this.end = end; + this.canQualifyColumn = canQualifyColumn; + } + } + + private SelectQueryValidator() { + } + + /** + * Validates that the given query is a single read-only {@code SELECT} statement under the {@link Dialect#UNKNOWN} + * conservative rules (or {@link Dialect#TSQL} if {@code squareBracketQuotes} is true). + * + * @param query the query to validate + * @param squareBracketQuotes whether square brackets quote identifiers, as in SQL Server + * @throws IllegalArgumentException if the query is not a single read-only SELECT statement + */ + public static void validate(String query, boolean squareBracketQuotes) { + validate(query, squareBracketQuotes ? Dialect.TSQL : Dialect.UNKNOWN); + } + + /** + * Validates that the given query is a single read-only {@code SELECT} statement for the specified {@link Dialect}. + * + * @param query the query to validate + * @param dialect the database SQL dialect, or {@link Dialect#UNKNOWN} when unknown + * @throws IllegalArgumentException if the query is not a single read-only SELECT statement + */ + public static void validate(String query, Dialect dialect) { + if (query == null) { + throw new IllegalArgumentException("Query must not be null."); + } + Objects.requireNonNull(dialect, "Dialect must not be null."); + String parsableQuery = query.replace(CONDITIONS_PLACEHOLDER, "1 = 1"); + boolean hasDoubleQuote = parsableQuery.indexOf('"') >= 0; + + switch (dialect) { + case ANSI: + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, false, false, false, false)); + break; + case TSQL: + validateWithConfig(parsableQuery, + new ScanConfig(true, false, false, false, false, false, true)); + break; + case POSTGRESQL: + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, true, false, false, true)); + break; + case MYSQL: + validateWithConfig(parsableQuery, + new ScanConfig(false, true, true, false, true, true, false)); + if (hasDoubleQuote) { + // Under ANSI_QUOTES a double quote is an identifier where '\' does not escape '"'. + validateWithConfig(parsableQuery, + new ScanConfig(false, true, false, false, true, true, false)); + } + break; + case MYSQL_AMBIGUOUS_BACKSLASH: + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, false, true, true, false)); + validateWithConfig(parsableQuery, + new ScanConfig(false, true, true, false, true, true, false)); + if (hasDoubleQuote) { + validateWithConfig(parsableQuery, + new ScanConfig(false, true, false, false, true, true, false)); + } + break; + case DATABRICKS: + validateWithConfig(parsableQuery, + new ScanConfig(false, true, true, false, false, false, false)); + if (hasDoubleQuote) { + validateWithConfig(parsableQuery, + new ScanConfig(false, true, false, false, false, false, false)); + } + break; + case UNKNOWN: + default: + // Must pass under ANSI, PostgreSQL (per-literal E'...' and nested comments), and MySQL (both backslash modes). + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, false, false, false, false)); + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, true, false, false, true)); + validateWithConfig(parsableQuery, + new ScanConfig(false, false, false, false, true, true, false)); + validateWithConfig(parsableQuery, + new ScanConfig(false, true, true, false, true, true, false)); + if (hasDoubleQuote) { + validateWithConfig(parsableQuery, + new ScanConfig(false, true, false, false, true, true, false)); + } + break; + } + } + + private static void validateWithConfig(String query, ScanConfig config) { + boolean usedExtendedStrings = lexicalCheck(query, config); + boolean parserBackslash = config.backslashEscapesInSingleQuotes || usedExtendedStrings; + confirmWithParser(query, config.squareBracketQuotes, parserBackslash); + } + + /** + * Best-effort secondary check via JSqlParser: if JSqlParser can parse the query, verify that it produces no more + * than one statement and that the statement is a {@link Select}. Dialect-specific syntax that JSqlParser cannot + * parse is left to the authoritative lexical scan. + */ + @VisibleForTesting + static void confirmWithParser(String query, boolean squareBracketQuotes, boolean backslashEscapes) { + Statements statements; + try { + statements = CCJSqlParserUtil.parseStatements(query, PARSER_EXECUTOR, parser -> parser + .withSquareBracketQuotation(squareBracketQuotes) + .withBackslashEscapeCharacter(backslashEscapes) + .withTimeOut(PARSE_TIMEOUT_MS)); + } catch (JSQLParserException | TokenMgrException e) { + return; + } + List nonEmpty = statements == null ? Collections.emptyList() + : statements.stream().filter(Objects::nonNull).collect(Collectors.toList()); + if (nonEmpty.size() == 1 && !(nonEmpty.get(0) instanceof Select)) { + throw new IllegalArgumentException(String.format( + "Only a SELECT statement is allowed. Found a %s statement.", nonEmpty.get(0).getClass().getSimpleName())); + } + } + + /** + * Scans the query, ignoring the contents of string literals, comments, and quoted identifiers, and enforces that it + * is a single read-only SELECT statement. Returns true if any PostgreSQL {@code E'...'} literal was scanned. + */ + private static boolean lexicalCheck(String query, ScanConfig config) { + StringBuilder bareText = new StringBuilder(query.length() + 16); + int statementsWithContent = 0; + boolean currentStatementHasContent = false; + boolean sawExtendedString = false; + int i = 0; + int n = query.length(); + + while (i < n) { + char c = query.charAt(i); + if (c == '-' && i + 1 < n && query.charAt(i + 1) == '-' && isDashDashCommentStart(query, i, config)) { + i = skipToEndOfLine(query, i + 2); + bareText.append(' '); + } else if (config.hashComments && c == '#') { + i = skipToEndOfLine(query, i + 1); + bareText.append(' '); + } else if (c == '/' && i + 1 < n && query.charAt(i + 1) == '*') { + if (isExecutableCommentStart(query, i)) { + throw new IllegalArgumentException( + "Executable SQL comments ('/*!' or '/*M!') are not allowed in the query."); + } + i = skipBlockComment(query, i, config.nestedBlockComments); + bareText.append(' '); + } else if (isOracleAlternativeQuoteStart(query, i)) { + throw new IllegalArgumentException("Alternative-quoted string literals (q'...') are not allowed in the query."); + } else if (c == '$' && isDollarQuoteStart(query, i)) { + throw new IllegalArgumentException( + "Dollar-quoted string literals ($$...$$ or $tag$...$tag$) are not allowed in the query."); + } else if (config.postgresExtendedStrings && isPostgresExtendedStringStart(query, i)) { + i = skipPostgresExtendedString(query, i + 1, config.nestedBlockComments); + sawExtendedString = true; + currentStatementHasContent = true; + bareText.append(' ').append(STRING_PLACEHOLDER).append(' '); + } else if (c == '\'') { + i = skipString(query, i, '\'', config.backslashEscapesInSingleQuotes); + currentStatementHasContent = true; + bareText.append(' ').append(STRING_PLACEHOLDER).append(' '); + } else if (c == '"') { + if (config.backslashEscapesInDoubleQuotes) { + i = skipString(query, i, '"', true); + } else { + i = skipQuotedIdentifier(query, i, '"'); + } + currentStatementHasContent = true; + bareText.append(' ').append(QUOTED_IDENT_PLACEHOLDER).append(' '); + } else if (c == '`') { + i = skipQuotedIdentifier(query, i, '`'); + currentStatementHasContent = true; + bareText.append(' ').append(QUOTED_IDENT_PLACEHOLDER).append(' '); + } else if (config.squareBracketQuotes && c == '[') { + i = skipBracketIdentifier(query, i); + currentStatementHasContent = true; + bareText.append(' ').append(QUOTED_IDENT_PLACEHOLDER).append(' '); + } else if (c == ';') { + if (currentStatementHasContent) { + statementsWithContent++; + } + currentStatementHasContent = false; + bareText.append(' '); + i++; + } else { + if (!isSqlWhitespace(c)) { + currentStatementHasContent = true; + } + bareText.append(c); + i++; + } + } + + if (currentStatementHasContent) { + statementsWithContent++; + } + if (statementsWithContent > 1) { + throw new IllegalArgumentException(String.format( + "Only a single SELECT statement is allowed. Found %d statements.", statementsWithContent)); + } + checkKeywords(bareText.toString()); + return sawExtendedString; + } + + private static boolean isDashDashCommentStart(String query, int i, ScanConfig config) { + if (!config.mysqlDashDashComments) { + return true; + } + // MySQL and MariaDB treat '--' as a comment only before whitespace, a control character, or end of input. + if (i + 2 >= query.length()) { + return true; + } + char after = query.charAt(i + 2); + return isSqlWhitespace(after) || Character.isISOControl(after); + } + + private static boolean isExecutableCommentStart(String query, int i) { + int n = query.length(); + if (i + 2 < n && query.charAt(i + 2) == '!') { + return true; + } + return i + 3 < n && (query.charAt(i + 2) == 'M' || query.charAt(i + 2) == 'm') && query.charAt(i + 3) == '!'; + } + + private static boolean isOracleAlternativeQuoteStart(String query, int i) { + if (i > 0 && isIdentifierChar(query.charAt(i - 1))) { + return false; + } + char c = query.charAt(i); + int n = query.length(); + if ((c == 'q' || c == 'Q') && i + 1 < n && query.charAt(i + 1) == '\'') { + return true; + } + return (c == 'n' || c == 'N') && i + 2 < n + && (query.charAt(i + 1) == 'q' || query.charAt(i + 1) == 'Q') + && query.charAt(i + 2) == '\''; + } + + private static boolean isPostgresExtendedStringStart(String query, int i) { + if (i > 0 && isIdentifierChar(query.charAt(i - 1))) { + return false; + } + char c = query.charAt(i); + return (c == 'e' || c == 'E') && i + 1 < query.length() && query.charAt(i + 1) == '\''; + } + + private static boolean isDollarQuoteStart(String query, int i) { + // In PostgreSQL, '$' is a valid continuation character inside an identifier (e.g. v$session, dbms_aq$aq$_jms). + // Walk back across any preceding identifier characters: if the token began with a letter or '_', it is an + // ordinary identifier rather than a dollar-quote opener. + if (i > 0 && isIdentifierChar(query.charAt(i - 1))) { + int k = i - 1; + while (k > 0 && isIdentifierChar(query.charAt(k - 1))) { + k--; + } + char tokenFirst = query.charAt(k); + if (!Character.isDigit(tokenFirst) && tokenFirst != '$') { + return false; + } + } + if (!DOLLAR_QUOTE_AT_START.matcher(query.substring(i)).find()) { + return false; + } + int closingDollar = query.indexOf('$', i + 1); + for (int p = i + 1; p < closingDollar; p++) { + if (!isIdentifierChar(query.charAt(p)) || query.charAt(p) == '$') { + return false; + } + } + return true; + } + + private static void checkKeywords(String bareText) { + List tokens = new ArrayList<>(); + int i = 0; + int n = bareText.length(); + while (i < n) { + char c = bareText.charAt(i); + if (isIdentifierChar(c)) { + int start = i; + while (i < n && isIdentifierChar(bareText.charAt(i))) { + i++; + } + String raw = bareText.substring(start, i); + String upper = raw.toUpperCase(Locale.ROOT); + boolean canQualify = QUOTED_IDENT_PLACEHOLDER.equals(upper) + || (!STRING_PLACEHOLDER.equals(upper) && !Character.isDigit(c)); + tokens.add(new WordToken(upper, start, i, canQualify)); + } else { + i++; + } + } + + checkJdbcCallEscape(bareText, tokens); + + for (int w = 0; w < tokens.size(); w++) { + WordToken token = tokens.get(w); + String word = token.word; + if (STRING_PLACEHOLDER.equals(word) || QUOTED_IDENT_PLACEHOLDER.equals(word)) { + continue; + } + if (isDotQualifiedIdentifier(bareText, tokens, w)) { + continue; + } + if ("CALL".equals(word) && w == 0) { + throw new IllegalArgumentException("The 'CALL' keyword is not allowed. The query must only read data."); + } + if ("FOR".equals(word) && w + 1 < tokens.size() && LOCKING_AFTER_FOR.contains(tokens.get(w + 1).word)) { + throw new IllegalArgumentException( + "FOR UPDATE/SHARE locking clauses are not allowed. The query must only read data."); + } + if ("LOCK".equals(word) && w + 2 < tokens.size() + && "IN".equals(tokens.get(w + 1).word) && "SHARE".equals(tokens.get(w + 2).word)) { + throw new IllegalArgumentException( + "LOCK IN SHARE MODE clauses are not allowed. The query must only read data."); + } + if (WRITE_KEYWORDS.contains(word) && !isExemptWriteKeyword(bareText, tokens, w)) { + throw new IllegalArgumentException(String.format( + "The '%s' keyword is not allowed. The query must only read data.", word)); + } + } + + boolean hasSelect = tokens.stream().anyMatch(t -> "SELECT".equals(t.word)); + if (!hasSelect) { + throw new IllegalArgumentException("Only a SELECT statement is allowed. No SELECT keyword was found."); + } + + WordToken first = tokens.get(0); + if (!ALLOWED_START_KEYWORDS.contains(first.word)) { + throw new IllegalArgumentException(String.format( + "Only a SELECT statement is allowed. Query cannot start with '%s'.", first.word)); + } + if ("LOCKING".equals(first.word) && !hasReadOnlyTeradataLockingPrefix(tokens)) { + throw new IllegalArgumentException( + "Only LOCKING ... FOR ACCESS or FOR READ modifiers are allowed before SELECT."); + } + } + + private static void checkJdbcCallEscape(String bareText, List tokens) { + // Reject JDBC callable escape syntax: {call proc(...)} or {?= call proc(...)}. + for (WordToken token : tokens) { + if (!"CALL".equals(token.word)) { + continue; + } + int p = skipWhitespaceBackward(bareText, token.start - 1); + if (p >= 0 && bareText.charAt(p) == '=') { + p = skipWhitespaceBackward(bareText, p - 1); + if (p >= 0 && bareText.charAt(p) == '?') { + p = skipWhitespaceBackward(bareText, p - 1); + } + } + if (p >= 0 && bareText.charAt(p) == '{') { + throw new IllegalArgumentException("The 'CALL' keyword is not allowed. The query must only read data."); + } + } + } + + private static boolean isDotQualifiedIdentifier(String bareText, List tokens, int w) { + if (w == 0) { + return false; + } + WordToken prev = tokens.get(w - 1); + if (!prev.canQualifyColumn) { + return false; + } + int dotPos = skipWhitespaceBackward(bareText, tokens.get(w).start - 1); + if (dotPos < 0 || bareText.charAt(dotPos) != '.') { + return false; + } + int beforeDot = skipWhitespaceBackward(bareText, dotPos - 1); + return beforeDot == prev.end - 1; + } + + private static boolean isExemptWriteKeyword(String bareText, List tokens, int w) { + String word = tokens.get(w).word; + if (FUNCTION_COLLIDING_WRITE_KEYWORDS.contains(word) + && w > 0 + && isFollowedByOpenParen(bareText, tokens.get(w).end)) { + return true; + } + if ("MERGE".equals(word) && w > 0 && w + 1 < tokens.size()) { + String next = tokens.get(w + 1).word; + if ("JOIN".equals(next) || "UNION".equals(next)) { + return true; + } + } + if ("SET".equals(word) && w > 0 && "CHARACTER".equals(tokens.get(w - 1).word)) { + // Allow SQL standard / MySQL 'CAST(x AS CHAR CHARACTER SET utf8mb4)', while rejecting 'SELECT 1 AS character SET' + boolean precededByAs = w >= 2 && "AS".equals(tokens.get(w - 2).word); + return !precededByAs; + } + return false; + } + + private static boolean hasReadOnlyTeradataLockingPrefix(List tokens) { + for (int w = 1; w < tokens.size(); w++) { + String word = tokens.get(w).word; + if ("SELECT".equals(word) || "WITH".equals(word)) { + return false; + } + if ("FOR".equals(word) && w + 1 < tokens.size()) { + String mode = tokens.get(w + 1).word; + return "ACCESS".equals(mode) || "READ".equals(mode); + } + } + return false; + } + + private static int skipWhitespaceBackward(String text, int fromIndex) { + int i = fromIndex; + while (i >= 0 && isSqlWhitespace(text.charAt(i))) { + i--; + } + return i; + } + + private static boolean isFollowedByOpenParen(String text, int fromOffset) { + int i = fromOffset; + while (i < text.length() && isSqlWhitespace(text.charAt(i))) { + i++; + } + return i < text.length() && text.charAt(i) == '('; + } + + private static boolean isIdentifierChar(char c) { + if (isSqlWhitespace(c) || Character.isISOControl(c)) { + return false; + } + return Character.isLetterOrDigit(c) || c == '_' || c == '$' || c >= 0x80; + } + + private static boolean isSqlWhitespace(char c) { + return Character.isWhitespace(c) || Character.isSpaceChar(c) || c == '\uFEFF'; + } + + private static int skipToEndOfLine(String q, int i) { + while (i < q.length() && q.charAt(i) != '\n' && q.charAt(i) != '\r') { + i++; + } + return i; + } + + private static int skipBlockComment(String q, int i, boolean nested) { + i += 2; + int depth = 1; + while (i + 1 < q.length()) { + if (nested && q.charAt(i) == '/' && q.charAt(i + 1) == '*') { + if (isExecutableCommentStart(q, i)) { + throw new IllegalArgumentException( + "Executable SQL comments ('/*!' or '/*M!') are not allowed in the query."); + } + depth++; + i += 2; + continue; + } + if (q.charAt(i) == '*' && q.charAt(i + 1) == '/') { + depth--; + i += 2; + if (depth == 0) { + return i; + } + continue; + } + i++; + } + throw new IllegalArgumentException("Unterminated block comment in the query."); + } + + private static int skipPostgresExtendedString(String q, int quoteIdx, boolean nestedBlockComments) { + int next = skipString(q, quoteIdx, '\'', true); + // PostgreSQL's lexer concatenates adjacent string literals in mode when separated by whitespace/comments + // containing at least one newline ('quotecontinue'). + while (next < q.length()) { + int p = next; + boolean sawNewline = false; + while (p < q.length()) { + char c = q.charAt(p); + if (c == '\n' || c == '\r') { + sawNewline = true; + p++; + } else if (isSqlWhitespace(c)) { + p++; + } else if (c == '-' && p + 1 < q.length() && q.charAt(p + 1) == '-') { + p = skipToEndOfLine(q, p + 2); + } else if (c == '/' && p + 1 < q.length() && q.charAt(p + 1) == '*') { + int commentEnd = skipBlockComment(q, p, nestedBlockComments); + for (int k = p + 2; k < commentEnd - 2; k++) { + if (q.charAt(k) == '\n' || q.charAt(k) == '\r') { + sawNewline = true; + break; + } + } + p = commentEnd; + } else { + break; + } + } + if (sawNewline && p < q.length() && q.charAt(p) == '\'') { + next = skipString(q, p, '\'', true); + } else { + break; + } + } + return next; + } + + private static int skipString(String q, int i, char quote, boolean backslashEscapes) { + i++; + while (i < q.length()) { + char c = q.charAt(i); + if (backslashEscapes && c == '\\' && i + 1 < q.length()) { + i += 2; + continue; + } + if (c == quote) { + if (i + 1 < q.length() && q.charAt(i + 1) == quote) { + i += 2; + continue; + } + return i + 1; + } + i++; + } + throw new IllegalArgumentException("Unterminated string literal in the query."); + } + + private static int skipQuotedIdentifier(String q, int i, char quote) { + i++; + while (i < q.length()) { + char c = q.charAt(i); + if (c == quote) { + if (i + 1 < q.length() && q.charAt(i + 1) == quote) { + i += 2; + continue; + } + return i + 1; + } + i++; + } + throw new IllegalArgumentException("Unterminated quoted identifier in the query."); + } + + private static int skipBracketIdentifier(String q, int i) { + i++; + while (i < q.length()) { + char c = q.charAt(i); + if (c == ']') { + if (i + 1 < q.length() && q.charAt(i + 1) == ']') { + i += 2; + continue; + } + return i + 1; + } + i++; + } + throw new IllegalArgumentException("Unterminated quoted identifier in the query."); + } + + /** + * Validates that the given value is a single, bare column reference (optionally qualified and/or quoted) and + * nothing more. Used for inputs such as the split-by field, which are concatenated into generated SQL at runtime + * and so must not carry operators, function calls, comments, or additional statements. + * + * @param value the value to validate + * @param dialect the database SQL dialect + * @throws IllegalArgumentException if the value is not a single column reference + */ + public static void validateColumnReference(String value, Dialect dialect) { + Objects.requireNonNull(dialect, "Dialect must not be null."); + validateColumnReference(value, dialect.usesSquareBracketQuotes()); + } + + /** + * Validates that the given value is a single, bare column reference (optionally qualified and/or quoted). + * + * @param value the value to validate + * @param squareBracketQuotes whether square brackets quote identifiers, as in SQL Server + * @throws IllegalArgumentException if the value is not a single column reference + */ + public static void validateColumnReference(String value, boolean squareBracketQuotes) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("Must not be empty."); + } + if (value.indexOf(';') >= 0 || value.indexOf('\\') >= 0 || value.indexOf('#') >= 0 + || value.indexOf('\r') >= 0 || value.indexOf('\n') >= 0 + || value.contains("--") || value.contains("/*") + || containsUnquotedDollarQuote(value, squareBracketQuotes)) { + throw new IllegalArgumentException("Must be a single column name."); + } + Expression expression; + try { + // allowPartialParse=false so trailing content (operators, extra tokens) is rejected rather than ignored. + expression = CCJSqlParserUtil.parseExpression(value, false, + parser -> parser.withSquareBracketQuotation(squareBracketQuotes).withTimeOut(PARSE_TIMEOUT_MS)); + } catch (JSQLParserException | TokenMgrException e) { + throw new IllegalArgumentException("Must be a single column name.", e); + } + if (!(expression instanceof Column) || "*".equals(((Column) expression).getColumnName())) { + throw new IllegalArgumentException("Must be a single column name."); + } + } + + private static boolean containsUnquotedDollarQuote(String value, boolean squareBracketQuotes) { + if (value.indexOf('$') < 0) { + return false; + } + int i = 0; + int n = value.length(); + try { + while (i < n) { + char c = value.charAt(i); + if (c == '"' || c == '`') { + i = skipQuotedIdentifier(value, i, c); + } else if (squareBracketQuotes && c == '[') { + i = skipBracketIdentifier(value, i); + } else if (c == '$' && isDollarQuoteStart(value, i)) { + return true; + } else { + i++; + } + } + } catch (IllegalArgumentException e) { + return false; + } + return false; + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java index a8be38b46..afeb1ac39 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java @@ -17,18 +17,37 @@ package io.cdap.plugin.db.source; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.plugin.PluginProperties; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.validation.CauseAttributes; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import io.cdap.plugin.db.ConnectionConfigAccessor; +import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import org.mockito.Mockito; +import java.io.IOException; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; /** - * Test class for source schema validation. + * Test class for {@link AbstractDBSource} schema, query validation, and read-only schema fetch. */ public class AbstractDBSourceTest { private static final String MOCK_STAGE = "mockStage"; @@ -94,6 +113,124 @@ public void testValidateSourceSchemaInvalidFieldType() { assertPropertyValidationFailed(collector, "boolean_column"); } + @Test + public void testConfigurePipelineValidatesImportBoundingAndSplitByWithExplicitSchema() { + TestDBSourceConfig config = new TestDBSourceConfig( + "SELECT * FROM t WHERE $CONDITIONS; DELETE FROM t", + "SELECT MIN(id), MAX(id) FROM t; DROP TABLE t", + "id'", + 2, + SCHEMA, + null); + TestDBSource source = new TestDBSource(config); + MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); + PipelineConfigurer configurer = mockPipelineConfigurer(collector); + + source.configurePipeline(configurer); + + List failures = collector.getValidationFailures(); + Assert.assertEquals(3, failures.size()); + assertConfigFailure(failures.get(0), AbstractDBSource.DBSourceConfig.IMPORT_QUERY, + "Invalid Import Query: Only a single SELECT statement is allowed. Found 2 statements."); + assertConfigFailure(failures.get(1), AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, + "Invalid Bounding Query: Only a single SELECT statement is allowed. Found 2 statements."); + assertConfigFailure(failures.get(2), AbstractDBSource.DBSourceConfig.SPLIT_BY, + "Invalid Split-By Field Name: Must be a single column name."); + } + + @Test + public void testExecuteSchemaQuerySetsReadOnlyDisablesAutoCommitAndRollsBack() throws SQLException { + Connection connection = Mockito.mock(Connection.class); + Statement statement = Mockito.mock(Statement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(connection.createStatement()).thenReturn(statement); + Mockito.when(statement.executeQuery("SELECT * FROM t")).thenReturn(resultSet); + Mockito.when(resultSet.getMetaData()).thenReturn(metaData); + Mockito.when(metaData.getColumnCount()).thenReturn(1); + Mockito.when(metaData.getColumnName(1)).thenReturn("id"); + Mockito.when(metaData.getColumnType(1)).thenReturn(Types.INTEGER); + Mockito.when(metaData.getColumnTypeName(1)).thenReturn("INTEGER"); + Mockito.when(metaData.isSigned(1)).thenReturn(true); + Mockito.when(metaData.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls); + + TestDBSourceConfig config = new TestDBSourceConfig("SELECT * FROM t WHERE $CONDITIONS", null, null, 1, null, false); + TestDBSource source = new TestDBSource(config); + Schema fetched = source.executeSchemaQuery(connection); + Assert.assertNotNull(fetched.getField("id")); + + InOrder inOrder = Mockito.inOrder(connection, statement, resultSet); + inOrder.verify(connection).setReadOnly(true); + inOrder.verify(connection).setAutoCommit(false); + inOrder.verify(connection).createStatement(); + inOrder.verify(statement).setMaxRows(1); + inOrder.verify(statement).executeQuery("SELECT * FROM t"); + inOrder.verify(resultSet).close(); + inOrder.verify(statement).close(); + inOrder.verify(connection).rollback(); + Mockito.verify(connection, Mockito.never()).commit(); + } + + @Test + public void testDisableQueryValidationAndReadOnlyFlagOnLegacyAndConnectorConfigs() throws IOException { + TestDBSourceConfig enabledConfig = new TestDBSourceConfig( + "SELECT * FROM t WHERE $CONDITIONS", + "SELECT MIN(id), MAX(id) FROM t", + "id", + 2, + SCHEMA, + false); + TestDBSource enabledSource = new TestDBSource(enabledConfig); + MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); + ConnectionConfigAccessor enabledAccessor = + enabledSource.getConnectionConfigAccessor(Driver.class.getName(), SCHEMA, collector); + Assert.assertTrue(enabledAccessor.isReadOnly()); + + TestDBSourceConfig disabledConfig = new TestDBSourceConfig( + "SELECT * FROM t WHERE $CONDITIONS; DELETE FROM t", + "SELECT MIN(id), MAX(id) FROM t; DROP TABLE t", + "id'", + 2, + SCHEMA, + true); + TestDBSource disabledSource = new TestDBSource(disabledConfig); + MockFailureCollector disabledCollector = new MockFailureCollector(MOCK_STAGE); + disabledSource.configurePipeline(mockPipelineConfigurer(disabledCollector)); + Assert.assertTrue(disabledCollector.getValidationFailures().isEmpty()); + ConnectionConfigAccessor disabledAccessor = + disabledSource.getConnectionConfigAccessor(Driver.class.getName(), SCHEMA, disabledCollector); + Assert.assertFalse(disabledAccessor.isReadOnly()); + + TestConnectorConfig connectorWithoutOptOut = new TestConnectorConfig(false); + TestConnectorSourceConfig connectorSourceEnabled = new TestConnectorSourceConfig(connectorWithoutOptOut); + Assert.assertFalse(connectorSourceEnabled.isQueryValidationDisabled()); + + TestConnectorConfig connectorWithOptOut = new TestConnectorConfig(true); + TestConnectorSourceConfig connectorSourceDisabled = new TestConnectorSourceConfig(connectorWithOptOut); + Assert.assertTrue(connectorSourceDisabled.isQueryValidationDisabled()); + } + + private static PipelineConfigurer mockPipelineConfigurer(MockFailureCollector collector) { + PipelineConfigurer pipelineConfigurer = Mockito.mock(PipelineConfigurer.class); + StageConfigurer stageConfigurer = Mockito.mock(StageConfigurer.class); + Mockito.when(pipelineConfigurer.getStageConfigurer()).thenReturn(stageConfigurer); + Mockito.when(stageConfigurer.getFailureCollector()).thenReturn(collector); + Mockito.doReturn(Driver.class).when(pipelineConfigurer).usePluginClass( + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), + ArgumentMatchers.any(PluginProperties.class)); + return pipelineConfigurer; + } + + private static void assertConfigFailure(ValidationFailure failure, String expectedProperty, String expectedMessage) { + Assert.assertEquals(expectedMessage, failure.getMessage()); + List causes = getCauses(failure, CauseAttributes.STAGE_CONFIG); + Assert.assertEquals(1, causes.size()); + Assert.assertEquals(expectedProperty, causes.get(0).getAttribute(CauseAttributes.STAGE_CONFIG)); + } + private static void assertPropertyValidationFailed(MockFailureCollector failureCollector, String paramName) { List failureList = failureCollector.getValidationFailures(); Assert.assertEquals(1, failureList.size()); @@ -105,10 +242,90 @@ private static void assertPropertyValidationFailed(MockFailureCollector failureC } @Nonnull - private static List getCauses(ValidationFailure failure, String stacktrace) { + private static List getCauses(ValidationFailure failure, String attribute) { return failure.getCauses() .stream() - .filter(cause -> cause.getAttribute(stacktrace) != null) + .filter(cause -> cause.getAttribute(attribute) != null) .collect(Collectors.toList()); } + + private static final class TestDBSource extends AbstractDBSource { + TestDBSource(TestDBSourceConfig sourceConfig) { + super(sourceConfig); + } + + @Override + protected String createConnectionString() { + return sourceConfig.getConnectionString(); + } + } + + private static final class TestDBSourceConfig extends AbstractDBSource.DBSourceConfig { + private final Schema explicitSchema; + + TestDBSourceConfig(String importQuery, String boundingQuery, String splitBy, Integer numSplits, + Schema explicitSchema, Boolean disableQueryValidation) { + this.referenceName = "testRef"; + this.importQuery = importQuery; + this.boundingQuery = boundingQuery; + this.splitBy = splitBy; + this.numSplits = numSplits; + this.explicitSchema = explicitSchema; + this.disableQueryValidation = disableQueryValidation; + } + + @Override + public String getJdbcPluginName() { + return "testDriver"; + } + + @Override + public String getConnectionString() { + return "jdbc:hsqldb:mem:test"; + } + + @Override + public Schema getSchema() { + return explicitSchema; + } + + @Override + public Integer getMaxRetryCount() { + return 0; + } + } + + private static final class TestConnectorConfig extends AbstractDBConnectorConfig { + TestConnectorConfig(Boolean disableValidation) { + this.disableQueryValidation = disableValidation; + } + + @Override + public String getConnectionString() { + return "jdbc:postgresql://localhost:5432/db"; + } + + @Override + public boolean canConnect() { + return false; + } + } + + private static final class TestConnectorSourceConfig extends AbstractDBSpecificSourceConfig { + private final AbstractDBConnectorConfig connection; + + TestConnectorSourceConfig(AbstractDBConnectorConfig connection) { + this.connection = connection; + } + + @Override + protected Map getDBSpecificArguments() { + return Collections.emptyMap(); + } + + @Override + protected AbstractDBConnectorConfig getConnection() { + return connection; + } + } } diff --git a/database-commons/src/test/java/io/cdap/plugin/util/SelectQueryValidatorTest.java b/database-commons/src/test/java/io/cdap/plugin/util/SelectQueryValidatorTest.java new file mode 100644 index 000000000..0671140d0 --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/util/SelectQueryValidatorTest.java @@ -0,0 +1,477 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.util; + +import net.sf.jsqlparser.parser.TokenMgrException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Locale; + +/** + * Unit tests for {@link SelectQueryValidator}. + */ +public class SelectQueryValidatorTest { + + @Test + public void testSingleReadOnlySelectAllowed() { + String[] queries = { + "SELECT * FROM my_table", + "select id, name from my_table where id > 10 order by name", + "SELECT * FROM my_table WHERE $CONDITIONS", + "SELECT * FROM my_table WHERE id > 10 AND $CONDITIONS", + "SELECT MIN(id), MAX(id) FROM my_table", + "SELECT * FROM \"my schema\".\"my;table\"", + "SELECT * FROM \"pg_class\"\"; DELETE FROM t; --\"", + "SELECT * FROM `db`.`my;table`", + "SELECT a.id, b.name FROM a JOIN b ON a.id = b.id", + "SELECT * FROM (SELECT id FROM my_table) t", + "SELECT id FROM a UNION ALL SELECT id FROM b", + "WITH t AS (SELECT id FROM my_table) SELECT * FROM t", + "SELECT * FROM my_table WHERE name = 'a;b' -- trailing ; comment", + "SELECT * FROM my_table /* ; DELETE FROM t */", + "SELECT * FROM my_table WHERE name = 'O;Brien -- not a comment'", + "SELECT * FROM my_table;", + "SELECT TOP 10 * FROM dbo.my_table", + "SELECT * FROM my_table WHERE ROWNUM <= 10", + "SELECT * FROM my_table LIMIT 10", + "SELECT TRUNCATE(col, 2) FROM my_table", + "SELECT INSERT(name, 1, 2, 'x') FROM my_table", + "SELECT REPLACE(name, 'a', 'b') FROM my_table", + "SELECT CAST(name AS CHAR CHARACTER SET utf8mb4) FROM my_table", + "SELECT * FROM t TABLESAMPLE BERNOULLI (100.0 * 10 / (SELECT COUNT(*) FROM t))", + "SELECT * FROM t FOR READ ONLY", + "SELECT * FROM t FOR FETCH ONLY", + "LOCKING ROW FOR ACCESS SELECT * FROM t", + "LOCKING TABLE my_table FOR READ SELECT * FROM my_table", + "SELECT * FROM `my``db`.`my``table`", + "SELECT * FROM my_table WHERE ts <= '${logicalStartTime(yyyy-MM-dd,1d)}' AND $CONDITIONS", + }; + for (String query : queries) { + for (SelectQueryValidator.Dialect dialect : SelectQueryValidator.Dialect.values()) { + try { + SelectQueryValidator.validate(query, dialect); + } catch (IllegalArgumentException e) { + Assert.fail(String.format("Query '%s' should be valid under %s but failed with: %s", + query, dialect, e.getMessage())); + } + } + } + } + + @Test + public void testOrdinaryLiteralsAndDollarIdentifiersAllowed() { + String[] queries = { + "SELECT * FROM t WHERE grade = 'Q'", + "SELECT * FROM t WHERE code IN ('p','q')", + "SELECT * FROM t WHERE name LIKE '%q'", + "SELECT TO_CHAR(d, 'Q') FROM t", + "SELECT * FROM t WHERE label = 'nq''test'", + "SELECT * FROM t WHERE code = '$A$'", + "SELECT * FROM t WHERE code = '$$not_dollar_quoted$$'", + "SELECT * FROM t WHERE note = '/*!50000 NOT EXECUTABLE IN STRING */'", + "SELECT * FROM v$session", + "SELECT * FROM sys.dbms_aq$aq$_jms", + }; + for (String query : queries) { + for (SelectQueryValidator.Dialect dialect : SelectQueryValidator.Dialect.values()) { + SelectQueryValidator.validate(query, dialect); + } + } + } + + @Test + public void testUnquotedCallDotQualifiedColumnsAndMergeJoinAllowed() { + SelectQueryValidator.validate("SELECT call FROM calls", SelectQueryValidator.Dialect.UNKNOWN); + SelectQueryValidator.validate( + "SELECT c.call, c.grant, c.update, c.delete, c.set, c.into, c.lock FROM calls c", + SelectQueryValidator.Dialect.UNKNOWN); + SelectQueryValidator.validate( + "SELECT \"c\".grant, \"c\".update FROM calls \"c\"", + SelectQueryValidator.Dialect.POSTGRESQL); + SelectQueryValidator.validate( + "SELECT [c].grant, [c].merge FROM [calls] [c]", + SelectQueryValidator.Dialect.TSQL); + SelectQueryValidator.validate( + "SELECT * FROM a INNER MERGE JOIN b ON a.id = b.id", + SelectQueryValidator.Dialect.TSQL); + SelectQueryValidator.validate( + "SELECT * FROM a JOIN b ON a.id = b.id OPTION (MERGE JOIN)", + SelectQueryValidator.Dialect.TSQL); + + // Top-level CALL, JDBC {call ...} escapes, and non-identifier dot prefixes must still be rejected. + assertRejected("CALL my_procedure()", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("{call my_procedure()}", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("{?= call my_procedure()}", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT 1. DELETE FROM t", SelectQueryValidator.Dialect.TSQL); + assertRejected("SELECT 'a'. DELETE FROM t", SelectQueryValidator.Dialect.TSQL); + assertRejected("SELECT 1 AS character SET IDENTITY_INSERT t ON", SelectQueryValidator.Dialect.TSQL); + } + + @Test + public void testSquareBracketIdentifiersHandledPerDialect() { + SelectQueryValidator.validate("SELECT * FROM [dbo].[my;table] WHERE [id] > 1", true); + SelectQueryValidator.validate("SELECT * FROM [dbo].[my;table] WHERE [id] > 1", + SelectQueryValidator.Dialect.TSQL); + Assert.assertThrows(IllegalArgumentException.class, + () -> SelectQueryValidator.validate("SELECT * FROM [dbo].[t]; DELETE FROM [dbo].[t]", true)); + assertRejected("SELECT * FROM [dbo].[t]; DELETE FROM [dbo].[t]", SelectQueryValidator.Dialect.TSQL); + assertRejected("SELECT * FROM [dbo].[my;table]", SelectQueryValidator.Dialect.ANSI); + } + + @Test + public void testBackslashLiteralsAcrossDialects() { + String likeEscape = "SELECT * FROM t WHERE name LIKE '%\\_%' ESCAPE '\\'"; + String windowsPath = "SELECT * FROM t WHERE dir = 'C:\\data\\'"; + for (SelectQueryValidator.Dialect dialect : new SelectQueryValidator.Dialect[]{ + SelectQueryValidator.Dialect.ANSI, + SelectQueryValidator.Dialect.TSQL, + SelectQueryValidator.Dialect.POSTGRESQL + }) { + SelectQueryValidator.validate(likeEscape, dialect); + SelectQueryValidator.validate(windowsPath, dialect); + } + + // MySQL and Databricks use backslash escapes inside string literals. + SelectQueryValidator.validate("SELECT * FROM users WHERE last_name = 'O\\'Brien' AND id > 1", + SelectQueryValidator.Dialect.MYSQL); + SelectQueryValidator.validate("SELECT * FROM users WHERE last_name = 'O\\'Brien' AND id > 1", + SelectQueryValidator.Dialect.DATABRICKS); + + // Under UNKNOWN or MYSQL_AMBIGUOUS_BACKSLASH (when sqlMode may enable NO_BACKSLASH_ESCAPES), a backslash before a + // single quote is ambiguous and must be rejected so it cannot break out of the literal. + assertRejected("SELECT 'a\\'; DELETE FROM my_table; -- '", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT 'a\\'; DELETE FROM my_table; -- '", SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH); + } + + @Test + public void testPostgresqlPerLiteralExtendedStringsAndNestedComments() { + SelectQueryValidator.validate("SELECT E'O\\'Brien', 'C:\\data\\' FROM t", + SelectQueryValidator.Dialect.POSTGRESQL); + SelectQueryValidator.validate("SELECT e'line1\\nline2', 'C:\\', '; SELECT 1 -- ' FROM t", + SelectQueryValidator.Dialect.POSTGRESQL); + SelectQueryValidator.validate("SELECT 'a' /* outer /* inner ; DELETE */ still comment */ FROM t", + SelectQueryValidator.Dialect.POSTGRESQL); + SelectQueryValidator.validate("SELECT 'a' /* outer /* inner ; DELETE */ still comment */ FROM t", + SelectQueryValidator.Dialect.TSQL); + + // Second literal has no E prefix, so '\' does not escape the quote in PostgreSQL. + assertRejected("SELECT E'a\\'b', 'c\\'; DELETE FROM t; --'", SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected("SELECT E'a\\'b', 'c\\'; DELETE FROM t; --'", SelectQueryValidator.Dialect.UNKNOWN); + + // PostgreSQL quotecontinue concatenates adjacent newline-separated string segments in E'...' mode. + assertRejected("SELECT E'a'\n'b\\'', 'x'; DELETE FROM t; --'", SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected("SELECT E'a'\n'b\\'', 'x'; DELETE FROM t; --'", SelectQueryValidator.Dialect.UNKNOWN); + + // Nested block comment breakout must be rejected in POSTGRESQL, TSQL, and UNKNOWN. + String nestedCommentBreakout = "SELECT 'a' /* /* */ , ' */ ; DELETE FROM t; --'"; + assertRejected(nestedCommentBreakout, SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected(nestedCommentBreakout, SelectQueryValidator.Dialect.TSQL); + assertRejected(nestedCommentBreakout, SelectQueryValidator.Dialect.UNKNOWN); + + assertRejected("SELECT * FROM t /* unterminated comment", SelectQueryValidator.Dialect.ANSI); + assertRejected("SELECT * FROM t /* outer /* inner */", SelectQueryValidator.Dialect.POSTGRESQL); + } + + @Test + public void testMysqlDashDashAndHashComments() { + // MySQL requires whitespace, a control character, or EOF after '--' to start a comment. + String[] nonCommentsInMysql = { + "SELECT name FROM t WHERE id > 0 --1 INTO OUTFILE '/tmp/x'", + "SELECT name FROM t WHERE id > 0 --1 FOR UPDATE", + "SELECT 1 --1; DELETE FROM t", + }; + for (String query : nonCommentsInMysql) { + assertRejected(query, SelectQueryValidator.Dialect.MYSQL); + assertRejected(query, SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH); + assertRejected(query, SelectQueryValidator.Dialect.UNKNOWN); + } + + SelectQueryValidator.validate("SELECT name FROM t WHERE id > 0 -- 1trailing comment", + SelectQueryValidator.Dialect.MYSQL); + SelectQueryValidator.validate("SELECT name FROM t WHERE id > 0 --", + SelectQueryValidator.Dialect.MYSQL); + + // '#' starts a line comment in MySQL, but ends at \n or \r and is not a comment in ANSI/PostgreSQL. + SelectQueryValidator.validate("SELECT * FROM t WHERE id = 1 # comment ; DELETE FROM t", + SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM t # comment\n; DELETE FROM t", SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM t # comment\r; DELETE FROM t", SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM t WHERE id = 1 # comment ; DELETE FROM t", SelectQueryValidator.Dialect.ANSI); + } + + @Test + public void testDoubleQuoteBackslashBreakoutUnderAnsiQuotesRejected() { + // Valid single string in backslash mode, but breaks out as an identifier if ANSI_QUOTES is active. + String ansiQuotesBreakout = "SELECT \"a\\\"; DELETE FROM t; -- \" FROM t"; + assertRejected(ansiQuotesBreakout, SelectQueryValidator.Dialect.MYSQL); + assertRejected(ansiQuotesBreakout, SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH); + assertRejected(ansiQuotesBreakout, SelectQueryValidator.Dialect.DATABRICKS); + assertRejected(ansiQuotesBreakout, SelectQueryValidator.Dialect.UNKNOWN); + + String backslashModeBreakout = "SELECT \"a\\\"\"; DELETE FROM my_table; --\" FROM my_table"; + assertRejected(backslashModeBreakout, SelectQueryValidator.Dialect.MYSQL); + assertRejected(backslashModeBreakout, SelectQueryValidator.Dialect.UNKNOWN); + } + + @Test + public void testCarriageReturnLineCommentTerminationRejected() { + for (SelectQueryValidator.Dialect dialect : SelectQueryValidator.Dialect.values()) { + assertRejected("SELECT 1 -- comment\r; DELETE FROM t", dialect); + assertRejected("SELECT * FROM t -- comment\rINTO OUTFILE '/tmp/x'", dialect); + } + } + + @Test + public void testTurkishDefaultLocaleDoesNotBypassKeywordOrUrlChecks() { + Locale previous = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + try { + assertRejected("select * into backup_t from my_table", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("select * from my_table into outfile '/tmp/x'", SelectQueryValidator.Dialect.MYSQL); + assertRejected("select * from my_table kill 52", SelectQueryValidator.Dialect.TSQL); + assertRejected("with d as (insert into my_table values (1)) select * from d", + SelectQueryValidator.Dialect.POSTGRESQL); + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + SelectQueryValidator.Dialect.fromJdbcUrl("JDBC:SQLSERVER://HOST:1433", null)); + Assert.assertEquals( + SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + SelectQueryValidator.Dialect.fromJdbcUrl("JDBC:MYSQL://HOST:3306/DB", + "sessionVariables=sql_mode='NO_BACKSLASH_ESCAPES'")); + } finally { + Locale.setDefault(previous); + } + } + + @Test + public void testNonAsciiWhitespaceDoesNotGlueKeywords() { + assertRejected("DELETE\u00A0FROM t WHERE id IN (SELECT 1)", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT * INTO\u00A0x FROM t", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT * FROM t FOR\u00A0UPDATE", SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT * FROM t FOR\u3000KEY\u2000SHARE", SelectQueryValidator.Dialect.POSTGRESQL); + } + + @Test + public void testLexerMismatchConstructsOutsideLiteralsRejected() { + assertRejected("SELECT $$ ' $$; DELETE FROM my_table; -- '", SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected("SELECT $tag$ ' $tag$; UPDATE my_table SET v=$$x$$ WHERE id=1; SELECT * FROM t WHERE 1=1 -- '", + SelectQueryValidator.Dialect.UNKNOWN); + assertRejected("SELECT $$x$$ AS c FROM my_table", SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected("SELECT 123$tag$x$tag$ FROM my_table", SelectQueryValidator.Dialect.POSTGRESQL); + assertRejected("SELECT q'['; DELETE FROM my_table; --]' FROM dual", SelectQueryValidator.Dialect.ANSI); + assertRejected("SELECT nq'['; DELETE FROM my_table; --]' FROM dual", SelectQueryValidator.Dialect.ANSI); + assertRejected("SELECT * FROM my_table /*!50000 INTO OUTFILE '/tmp/out.csv' */", + SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM my_table /*!50000 FOR UPDATE */", SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT 1; /*! DELETE FROM my_table */", SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM my_table /*M!50000 FOR UPDATE */", SelectQueryValidator.Dialect.MYSQL); + assertRejected("SELECT * FROM my_table /* outer /*!50000 INTO OUTFILE '/tmp/x' */ */", + SelectQueryValidator.Dialect.POSTGRESQL); + } + + @Test + public void testStackedWriteLockingAndTsqlBatchStatementsRejected() { + String[] rejected = { + "SELECT 1; DELETE FROM t", + "SELECT * FROM t; UPDATE t SET v = 1", + "SELECT * FROM \"pg_catalog\".\"pg_class\" LIMIT 0; UPDATE t SET v = 'x'; SELECT 1", + "UPDATE my_table SET v = 1", + "DELETE FROM my_table", + "INSERT INTO my_table VALUES (1)", + "DROP TABLE my_table", + "TRUNCATE TABLE my_table", + "CREATE TABLE t (id INT)", + "ALTER TABLE my_table ADD COLUMN c INT", + "MERGE INTO t USING s ON (t.id = s.id) WHEN MATCHED THEN UPDATE SET t.v = s.v", + "GRANT SELECT ON my_table TO someone", + "EXEC my_procedure", + "CALL my_procedure()", + "VALUES (1)", + "TABLE my_table", + "COPY (SELECT * FROM t) TO PROGRAM 'id'", + "EXPLAIN SELECT * FROM t", + "EXPLAIN (ANALYZE) SELECT * FROM t", + "REPLACE t (a) SELECT 1", + "LOCKING TABLE t FOR WRITE SELECT * FROM t", + "SELECT * INTO new_table FROM my_table", + "SELECT * FROM my_table INTO OUTFILE '/tmp/out.csv'", + "SELECT * FROM my_table FOR UPDATE", + "SELECT * FROM my_table FOR SHARE", + "SELECT * FROM my_table FOR NO KEY UPDATE", + "SELECT * FROM my_table FOR KEY SHARE", + "SELECT * FROM my_table LOCK IN SHARE MODE", + "SELECT * FROM (SELECT * INTO new_table FROM my_table) t", + "SELECT * FROM (SELECT * FROM my_table FOR UPDATE) t", + "SELECT * FROM my_table WHERE id IN (SELECT id FROM other_table FOR KEY SHARE)", + "SELECT id FROM a UNION ALL (SELECT id INTO t2 FROM b)", + "WITH d AS (DELETE FROM my_table RETURNING *) SELECT * FROM d", + "SELECT 1 DENY SELECT ON t TO public", + "SELECT * FROM t DBCC CHECKIDENT('t', RESEED, 0)", + "SELECT 1 SET IDENTITY_INSERT t ON", + "SELECT * FROM t SHUTDOWN WITH NOWAIT", + "SELECT 1 BACKUP DATABASE db TO DISK = 'x'", + "SELECT 1 WAITFOR DELAY '00:00:10'", + "SELECT * INTO/**/new_table FROM my_table", + "SELECT * FROM my_table INTO/**/OUTFILE '/tmp/out.csv'", + "SELECT * FROM my_table FOR/**/UPDATE", + "SELECT * FROM \"my_table\"FOR\"x\"UPDATE", + "SELECT a\"alias\"INTO new_table FROM my_table", + "WITH d AS (DELETE/**/FROM my_table RETURNING *) SELECT * FROM d", + "SELECT * INTO \"new_table\"(col1) FROM my_table", + "UPDATE (SELECT * FROM my_table) SET v = 1", + "EXEC (SELECT 'DROP TABLE x')", + "EXECUTE (SELECT 'DROP TABLE x')", + "", + ";", + null, + }; + for (String query : rejected) { + assertRejected(query, SelectQueryValidator.Dialect.UNKNOWN); + } + } + + @Test + public void testConfirmWithParserRejectsNonSelectStatement() { + SelectQueryValidator.confirmWithParser("SELECT * FROM my_table", false, false); + + IllegalArgumentException explainErr = Assert.assertThrows( + IllegalArgumentException.class, + () -> SelectQueryValidator.confirmWithParser("EXPLAIN SELECT * FROM t", false, false)); + Assert.assertTrue(explainErr.getMessage().startsWith("Only a SELECT statement is allowed. Found a ")); + + IllegalArgumentException replaceErr = Assert.assertThrows( + IllegalArgumentException.class, + () -> SelectQueryValidator.confirmWithParser("REPLACE t (a) SELECT 1", false, false)); + Assert.assertEquals("Only a SELECT statement is allowed. Found a Upsert statement.", replaceErr.getMessage()); + } + + @Test + public void testInvalidQueriesReportExpectedErrorMessages() { + assertError("SELECT 1; DELETE FROM t", "Only a single SELECT statement is allowed. Found 2 statements."); + assertError("DELETE FROM t", "The 'DELETE' keyword is not allowed. The query must only read data."); + assertError("SELECT * INTO t2 FROM t", "The 'INTO' keyword is not allowed. The query must only read data."); + assertError("SELECT * FROM t FOR UPDATE", + "FOR UPDATE/SHARE locking clauses are not allowed. The query must only read data."); + assertError("SELECT * FROM t FOR KEY SHARE", + "FOR UPDATE/SHARE locking clauses are not allowed. The query must only read data."); + assertError("SELECT * FROM t LOCK IN SHARE MODE", + "LOCK IN SHARE MODE clauses are not allowed. The query must only read data."); + assertError("TABLE my_table", "Only a SELECT statement is allowed. No SELECT keyword was found."); + assertError("EXPLAIN SELECT * FROM t", "Only a SELECT statement is allowed. Query cannot start with 'EXPLAIN'."); + } + + @Test + public void testDialectFromJdbcUrl() { + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:sqlserver://localhost:1433", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:jtds:sqlserver://localhost:1433/db", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.POSTGRESQL, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:postgresql://localhost:5432/db", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.UNKNOWN, + SelectQueryValidator.Dialect.fromJdbcUrl( + "jdbc:postgresql://localhost:5432/db?options=-c%20standard_conforming_strings=off", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:mysql://localhost:3306/db", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:mariadb://localhost:3306/db", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + SelectQueryValidator.Dialect.fromJdbcUrl( + "jdbc:mysql://localhost:3306/db", "sessionVariables=sql_mode='NO_BACKSLASH_ESCAPES'")); + Assert.assertEquals(SelectQueryValidator.Dialect.DATABRICKS, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:databricks://host:443/default", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.ANSI, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:oracle:thin:@localhost:1521:xe", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.ANSI, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:hsqldb:hsql://localhost/testdb", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.UNKNOWN, + SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:redshift://cluster:5439/dev", null)); + Assert.assertEquals(SelectQueryValidator.Dialect.UNKNOWN, + SelectQueryValidator.Dialect.fromJdbcUrl(null, null)); + } + + @Test + public void testValidateColumnReferenceAllowed() { + String[] columns = {"id", "my_col", "t.id", "\"My Col\"", "`id`", "schema.t.id", "v$col"}; + for (String col : columns) { + SelectQueryValidator.validateColumnReference(col, SelectQueryValidator.Dialect.UNKNOWN); + } + SelectQueryValidator.validateColumnReference("[my id]", SelectQueryValidator.Dialect.TSQL); + } + + @Test + public void testValidateColumnReferenceWrapsTokenMgrExceptionCleanly() { + for (String malformedQuote : new String[]{"id'", "\"id", "id`"}) { + IllegalArgumentException ex = Assert.assertThrows( + IllegalArgumentException.class, + () -> SelectQueryValidator.validateColumnReference(malformedQuote, SelectQueryValidator.Dialect.UNKNOWN)); + Assert.assertEquals("Must be a single column name.", ex.getMessage()); + Assert.assertTrue("Expected TokenMgrException cause for input: " + malformedQuote, + ex.getCause() instanceof TokenMgrException); + } + } + + @Test + public void testValidateColumnReferenceRejectsEmptyAndInjections() { + for (String blank : new String[]{"", " ", null}) { + IllegalArgumentException ex = Assert.assertThrows( + IllegalArgumentException.class, + () -> SelectQueryValidator.validateColumnReference(blank, SelectQueryValidator.Dialect.UNKNOWN)); + Assert.assertEquals("Must not be empty.", ex.getMessage()); + } + + String[] injections = { + "1 = 1); UPDATE audit_vault.write_marker SET value='INJECTED' WHERE id=1; SELECT * FROM \"my_table\" WHERE (1", + "id; DROP TABLE t", + "id)", + "f(id)", + "id, name", + "id OR 1=1", + "id /*! FOR UPDATE */", + "\"a\\\"\"; DROP TABLE t; --\"", + "id -- comment", + "id /* comment */", + "id # comment", + "id\nOR 1=1", + "'id'", + "*", + "t.*", + "$$id$$", + }; + for (String col : injections) { + IllegalArgumentException ex = Assert.assertThrows( + IllegalArgumentException.class, + () -> SelectQueryValidator.validateColumnReference(col, SelectQueryValidator.Dialect.UNKNOWN)); + Assert.assertEquals("Must be a single column name.", ex.getMessage()); + } + } + + private static void assertRejected(String query, SelectQueryValidator.Dialect dialect) { + Assert.assertThrows("Expected query to be rejected under " + dialect + ": " + query, + IllegalArgumentException.class, + () -> SelectQueryValidator.validate(query, dialect)); + } + + private static void assertError(String query, String expectedMessage) { + try { + SelectQueryValidator.validate(query, SelectQueryValidator.Dialect.UNKNOWN); + Assert.fail(String.format("Query '%s' should have been rejected", query)); + } catch (IllegalArgumentException e) { + Assert.assertEquals(expectedMessage, e.getMessage()); + } + } +} diff --git a/databricks-plugin/docs/Databricks-batchsource.md b/databricks-plugin/docs/Databricks-batchsource.md index 7f225b66e..3f10e3d68 100644 --- a/databricks-plugin/docs/Databricks-batchsource.md +++ b/databricks-plugin/docs/Databricks-batchsource.md @@ -12,5 +12,7 @@ Properties * **HTTP Path**: The HTTP Path for the Databricks cluster or SQL warehouse. * **Reference Name**: Name used to identify this source for lineage. * **Database / Catalog**: Optional catalog or database name. -* **Import Query**: SQL query to execute against Databricks. +* **Import Query**: Single read-only `SELECT` query to execute against Databricks. Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses, dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. +* **Bounding Query**: Single read-only `SELECT` query returning the min and max of the `splitBy` field. Not required if `numSplits` is set to one. +* **Split-By Field Name**: Single column name (optionally table-qualified and/or quoted) used to generate splits. Not required if `numSplits` is set to one. * **Transaction Isolation Level**: Select `TRANSACTION_READ_UNCOMMITTED` when using Databricks JDBC 2.6.40, or `TRANSACTION_REPEATABLE_READ` when using Databricks JDBC 3.4.3. The 3.4.3 driver requires Java 11 or later. diff --git a/databricks-plugin/docs/Databricks-connector.md b/databricks-plugin/docs/Databricks-connector.md index 66a77571a..051aa5027 100644 --- a/databricks-plugin/docs/Databricks-connector.md +++ b/databricks-plugin/docs/Databricks-connector.md @@ -14,3 +14,5 @@ Properties * **Password / Token**: Personal Access Token (PAT) or password. * **Transaction Isolation Level**: Select `TRANSACTION_READ_UNCOMMITTED` when using Databricks JDBC 2.6.40, or `TRANSACTION_REPEATABLE_READ` when using Databricks JDBC 3.4.3. The 3.4.3 driver requires Java 11 or later. * **Connection Arguments**: Arbitrary key-value pairs to pass as connection arguments to the JDBC driver (e.g. `AuthMech=11;Auth_Flow=2`). +* **Disable Source Query Validation**: Disables read-only `SELECT` validation and read-only session enforcement for source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source queries are rejected by the validator. Defaults to `false`. + diff --git a/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksSource.java b/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksSource.java index 524dec5ee..922012e00 100644 --- a/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksSource.java +++ b/databricks-plugin/src/main/java/io/cdap/plugin/databricks/DatabricksSource.java @@ -37,6 +37,7 @@ import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.io.IOException; @@ -78,6 +79,11 @@ protected String createConnectionString() { return connection == null ? null : connection.getConnectionString(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.DATABRICKS; + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { DatabricksConnectorConfig connection = databricksSourceConfig.getConnection(); diff --git a/databricks-plugin/widgets/Databricks-batchsource.json b/databricks-plugin/widgets/Databricks-batchsource.json index 36be4064e..dadc10ce8 100644 --- a/databricks-plugin/widgets/Databricks-batchsource.json +++ b/databricks-plugin/widgets/Databricks-batchsource.json @@ -205,6 +205,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/databricks-plugin/widgets/Databricks-connector.json b/databricks-plugin/widgets/Databricks-connector.json index 4c90db0c5..1d580cf85 100644 --- a/databricks-plugin/widgets/Databricks-connector.json +++ b/databricks-plugin/widgets/Databricks-connector.json @@ -87,6 +87,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/db2-plugin/docs/Db2-batchsource.md b/db2-plugin/docs/Db2-batchsource.md index 53592f3c4..232dbf3ca 100644 --- a/db2-plugin/docs/Db2-batchsource.md +++ b/db2-plugin/docs/Db2-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** DB2 database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/db2-plugin/src/main/java/io/cdap/plugin/db2/Db2Source.java b/db2-plugin/src/main/java/io/cdap/plugin/db2/Db2Source.java index ffe9afa82..44239b895 100644 --- a/db2-plugin/src/main/java/io/cdap/plugin/db2/Db2Source.java +++ b/db2-plugin/src/main/java/io/cdap/plugin/db2/Db2Source.java @@ -23,6 +23,7 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; @@ -58,6 +59,11 @@ protected String createConnectionString() { db2SourceConfig.database); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.ANSI; + } + /** * DB2 source config. */ diff --git a/db2-plugin/widgets/Db2-batchsource.json b/db2-plugin/widgets/Db2-batchsource.json index df8e93bdf..07dffe1a5 100644 --- a/db2-plugin/widgets/Db2-batchsource.json +++ b/db2-plugin/widgets/Db2-batchsource.json @@ -148,6 +148,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/generic-database-plugin/docs/Database-batchsource.md b/generic-database-plugin/docs/Database-batchsource.md index 577aba8f3..75d797da9 100644 --- a/generic-database-plugin/docs/Database-batchsource.md +++ b/generic-database-plugin/docs/Database-batchsource.md @@ -22,16 +22,20 @@ Properties **Connection String:** JDBC connection string including database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/generic-database-plugin/src/main/java/io/cdap/plugin/jdbc/DatabaseSource.java b/generic-database-plugin/src/main/java/io/cdap/plugin/jdbc/DatabaseSource.java index b35b025b7..fec7adae0 100644 --- a/generic-database-plugin/src/main/java/io/cdap/plugin/jdbc/DatabaseSource.java +++ b/generic-database-plugin/src/main/java/io/cdap/plugin/jdbc/DatabaseSource.java @@ -22,6 +22,7 @@ import io.cdap.cdap.api.annotation.Plugin; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.SelectQueryValidator; import javax.annotation.Nullable; @@ -46,6 +47,16 @@ protected String createConnectionString() { return databaseSourceConfig.connectionString; } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (databaseSourceConfig.containsMacro(ConnectionConfig.CONNECTION_STRING)) { + return SelectQueryValidator.Dialect.UNKNOWN; + } + String args = databaseSourceConfig.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + ? "sessionvariables" : databaseSourceConfig.connectionArguments; + return SelectQueryValidator.Dialect.fromJdbcUrl(databaseSourceConfig.connectionString, args); + } + /** * Generic database source configuration. */ diff --git a/generic-database-plugin/src/test/java/io/cdap/plugin/db/sink/DBSourceTestRun.java b/generic-database-plugin/src/test/java/io/cdap/plugin/db/sink/DBSourceTestRun.java index bedb720a2..3a1fde727 100644 --- a/generic-database-plugin/src/test/java/io/cdap/plugin/db/sink/DBSourceTestRun.java +++ b/generic-database-plugin/src/test/java/io/cdap/plugin/db/sink/DBSourceTestRun.java @@ -19,17 +19,21 @@ import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.datapipeline.SmartWorkflow; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.mock.batch.MockSink; import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; import io.cdap.cdap.etl.proto.v2.ETLPlugin; import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; import io.cdap.cdap.proto.artifact.AppRequest; import io.cdap.cdap.proto.id.ApplicationId; import io.cdap.cdap.proto.id.NamespaceId; import io.cdap.cdap.test.ApplicationManager; import io.cdap.cdap.test.DataSetManager; +import io.cdap.cdap.test.WorkflowManager; import io.cdap.plugin.GenericDatabasePluginTestBase; import io.cdap.plugin.common.Constants; import io.cdap.plugin.db.ConnectionConfig; @@ -41,7 +45,13 @@ import java.math.BigDecimal; import java.math.MathContext; import java.nio.ByteBuffer; +import java.sql.Connection; import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.sql.Time; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -52,6 +62,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * Test for ETL using databases. @@ -329,6 +340,309 @@ public void testUserNamePasswordCombinations() throws Exception { deployApplication(appId, appRequest); } + @Test + public void testNonSelectImportQueryFailsDeployment() throws Exception { + int rowCount = countRows("my_table"); + String importQuery = "SELECT * FROM \"my_table\" WHERE $CONDITIONS; UPDATE \"my_table\" SET NAME='INJECTED'"; + ETLPlugin sourceConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, "SELECT MIN(ID),MAX(ID) from \"my_table\"") + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, "ID") + .put(Constants.Reference.REFERENCE_NAME, "DBNonSelectQueryTest") + .build(), + null); + ETLBatchConfig etlConfig = getETLBatchConfig(sourceConfig, MockSink.getPlugin("nonSelectQueryOutput")); + assertDeploymentFailureContains(NamespaceId.DEFAULT.app("dbSourceNonSelectQueryTest"), etlConfig, + "Invalid Import Query"); + Assert.assertEquals(rowCount, countRows("my_table")); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } + + @Test + public void testNonSelectImportQueryWithExplicitSchemaFailsDeployment() throws Exception { + // When an explicit schema is configured, configurePipeline returns before opening any DB connection, so only + // SelectQueryValidator can fail deployment here. + Schema explicitSchema = Schema.recordOf( + "explicit", + Schema.Field.of("ID", Schema.of(Schema.Type.INT)), + Schema.Field.of("NAME", Schema.of(Schema.Type.STRING))); + String importQuery = "SELECT ID, NAME FROM \"my_table\" WHERE $CONDITIONS; UPDATE \"my_table\" SET NAME='INJECTED'"; + ETLPlugin sourceConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "1") + .put(AbstractDBSource.DBSourceConfig.SCHEMA, explicitSchema.toString()) + .put(Constants.Reference.REFERENCE_NAME, "DBExplicitSchemaNonSelectTest") + .build(), + null); + ETLBatchConfig etlConfig = getETLBatchConfig(sourceConfig, MockSink.getPlugin("explicitSchemaNonSelectOutput")); + assertDeploymentFailureContains(NamespaceId.DEFAULT.app("dbSourceExplicitSchemaNonSelectTest"), etlConfig, + "Invalid Import Query"); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } + + @Test + public void testNonSelectImportQueryMacroFailsRun() throws Exception { + int rowCount = countRows("my_table"); + ETLPlugin sourceConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, "${importQuery}") + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, "SELECT MIN(ID),MAX(ID) from \"my_table\"") + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, "ID") + .put(Constants.Reference.REFERENCE_NAME, "DBNonSelectQueryMacroTest") + .build(), + null); + ApplicationManager appManager = deployETL(sourceConfig, MockSink.getPlugin("nonSelectQueryMacroOutput"), + DATAPIPELINE_ARTIFACT, "dbSourceNonSelectQueryMacroTest"); + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.start(ImmutableMap.of( + "importQuery", + "SELECT * FROM \"my_table\" WHERE $CONDITIONS; DELETE FROM \"my_table\"; UPDATE \"my_table\" SET NAME='INJECTED'" + )); + workflowManager.waitForRun(ProgramRunStatus.FAILED, 5, TimeUnit.MINUTES); + Assert.assertEquals(rowCount, countRows("my_table")); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } + + @Test + public void testNonSelectBoundingQueryFailsDeploymentAndMacroFailsRun() throws Exception { + int rowCount = countRows("my_table"); + ETLPlugin nonMacroBoundingConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, "SELECT * FROM \"my_table\" WHERE $CONDITIONS") + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, + "SELECT MIN(ID), MAX(ID) FROM \"my_table\"; DELETE FROM \"my_table\"") + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, "ID") + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "2") + .put(Constants.Reference.REFERENCE_NAME, "DBNonSelectBoundingQueryDeployTest") + .build(), + null); + ETLBatchConfig deployConfig = getETLBatchConfig(nonMacroBoundingConfig, + MockSink.getPlugin("nonSelectBoundingDeployOutput")); + assertDeploymentFailureContains(NamespaceId.DEFAULT.app("dbSourceNonSelectBoundingDeployTest"), deployConfig, + "Invalid Bounding Query"); + Assert.assertEquals(rowCount, countRows("my_table")); + + ETLPlugin macroBoundingConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, "SELECT * FROM \"my_table\" WHERE $CONDITIONS") + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, "${boundingQuery}") + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, "ID") + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "2") + .put(Constants.Reference.REFERENCE_NAME, "DBNonSelectBoundingQueryMacroTest") + .build(), + null); + ApplicationManager appManager = deployETL(macroBoundingConfig, MockSink.getPlugin("nonSelectBoundingMacroOutput"), + DATAPIPELINE_ARTIFACT, "dbSourceNonSelectBoundingMacroTest"); + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.start(ImmutableMap.of( + "boundingQuery", + "SELECT MIN(ID), MAX(ID) FROM \"my_table\"; DELETE FROM \"my_table\"; UPDATE \"my_table\" SET NAME='INJECTED'")); + workflowManager.waitForRun(ProgramRunStatus.FAILED, 5, TimeUnit.MINUTES); + Assert.assertEquals(rowCount, countRows("my_table")); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } + + @Test + public void testInjectionSplitByFailsDeployment() throws Exception { + int rowCount = countRows("my_table"); + String splitBy = "ID); UPDATE \"my_table\" SET NAME='INJECTED' WHERE (1=1"; + ETLPlugin sourceConfig = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, "SELECT * FROM \"my_table\" WHERE $CONDITIONS") + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, "SELECT MIN(ID),MAX(ID) FROM \"my_table\"") + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "2") + .put(Constants.Reference.REFERENCE_NAME, "DBInjectionSplitByTest") + .build(), + null); + ETLBatchConfig etlConfig = getETLBatchConfig(sourceConfig, MockSink.getPlugin("injectionSplitByOutput")); + assertDeploymentFailureContains(NamespaceId.DEFAULT.app("dbSourceInjectionSplitByTest"), etlConfig, + "Invalid Split-By Field Name"); + Assert.assertEquals(rowCount, countRows("my_table")); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } + + private static volatile String sideEffectJdbcUrl; + + public static int writeSideEffect(Connection sessionConn) throws SQLException { + if (sessionConn.isReadOnly()) { + throw new SQLException("SQL data change not permitted in read-only session", "25006"); + } + String url = sideEffectJdbcUrl; + if (url != null) { + try (Connection rwConn = DriverManager.getConnection(url, "SA", ""); + Statement stmt = rwConn.createStatement()) { + stmt.executeUpdate("UPDATE \"my_table\" SET NAME = 'INJECTED' WHERE ID = 1"); + } + } + return 1; + } + + @Test + public void testReadOnlyConnectionBlocksWritingFunctionInSelect() throws Exception { + sideEffectJdbcUrl = getConnectionURL(); + try (Connection conn = getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("CREATE FUNCTION WRITE_SIDE_EFFECT() RETURNS INT " + + "LANGUAGE JAVA NOT DETERMINISTIC READS SQL DATA " + + "EXTERNAL NAME 'CLASSPATH:io.cdap.plugin.db.sink.DBSourceTestRun.writeSideEffect'"); + } + try { + String queryWithWritingFn = "SELECT WRITE_SIDE_EFFECT() AS SIDE_EFFECT, ID FROM \"my_table\" WHERE $CONDITIONS"; + + // 1. Configure-time schema fetch opens a read-only connection, blocking the writing function. + ETLPlugin deployTimeSource = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, queryWithWritingFn) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "1") + .put(Constants.Reference.REFERENCE_NAME, "DBReadOnlyDeployBackstopTest") + .build(), + null); + ETLBatchConfig deployConfig = getETLBatchConfig(deployTimeSource, MockSink.getPlugin("readOnlyDeployOutput")); + assertDeploymentFailure(NamespaceId.DEFAULT.app("dbSourceReadOnlyDeployTest"), deployConfig, + DATAPIPELINE_ARTIFACT, + "Read-only connection should have blocked writing function during schema fetch."); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + + // 2. Even with an explicit schema (skipping configure-time fetch), runtime execution is also read-only. + Schema explicitSchema = Schema.recordOf( + "explicitFn", + Schema.Field.of("SIDE_EFFECT", Schema.of(Schema.Type.INT)), + Schema.Field.of("ID", Schema.of(Schema.Type.INT))); + ETLPlugin runTimeSource = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, queryWithWritingFn) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "1") + .put(AbstractDBSource.DBSourceConfig.SCHEMA, explicitSchema.toString()) + .put(Constants.Reference.REFERENCE_NAME, "DBReadOnlyRunBackstopTest") + .build(), + null); + ApplicationManager appManager = deployETL(runTimeSource, MockSink.getPlugin("readOnlyRunOutput"), + DATAPIPELINE_ARTIFACT, "dbSourceReadOnlyRunTest"); + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.start(); + workflowManager.waitForRun(ProgramRunStatus.FAILED, 5, TimeUnit.MINUTES); + Assert.assertEquals(0, countRowsWithName("my_table", "INJECTED")); + } finally { + sideEffectJdbcUrl = null; + try (Connection conn = getConnection(); + Statement stmt = conn.createStatement()) { + stmt.execute("DROP FUNCTION IF EXISTS WRITE_SIDE_EFFECT"); + } + } + } + + @Test + public void testDisableQueryValidationAllowsTrustedLegacyQuery() throws Exception { + // HSQLDB treats '/*M! ... */' as a regular comment, whereas SelectQueryValidator rejects it by default. + String legacyQuery = "SELECT ID, NAME FROM \"my_table\" /*M! legacy hint */ WHERE ID = 1 AND $CONDITIONS"; + + ETLPlugin defaultSource = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, legacyQuery) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "1") + .put(Constants.Reference.REFERENCE_NAME, "DBLegacyRejectedByDefaultTest") + .build(), + null); + ETLBatchConfig rejectedConfig = getETLBatchConfig(defaultSource, MockSink.getPlugin("legacyRejectedOutput")); + assertDeploymentFailureContains(NamespaceId.DEFAULT.app("dbSourceLegacyRejectedTest"), rejectedConfig, + "Invalid Import Query"); + + ETLPlugin optOutSource = new ETLPlugin( + DatabaseConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.CONNECTION_STRING, getConnectionURL()) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(ConnectionConfig.DISABLE_QUERY_VALIDATION, "true") + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, legacyQuery) + .put(AbstractDBSource.DBSourceConfig.NUM_SPLITS, "1") + .put(Constants.Reference.REFERENCE_NAME, "DBLegacyOptOutAllowedTest") + .build(), + null); + ApplicationManager appManager = deployETL(optOutSource, MockSink.getPlugin("legacyOptOutOutput"), + DATAPIPELINE_ARTIFACT, "dbSourceLegacyOptOutTest"); + runETLOnce(appManager); + DataSetManager outputManager = getDataset("legacyOptOutOutput"); + List records = MockSink.readOutput(outputManager); + Assert.assertEquals(1, records.size()); + Assert.assertEquals("user1", records.get(0).get("NAME")); + } + + private void assertDeploymentFailureContains(ApplicationId appId, ETLBatchConfig etlConfig, + String expectedMessageSubstring) throws Exception { + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + try { + deployApplication(appId, appRequest); + Assert.fail("Expected deployment failure containing: " + expectedMessageSubstring); + } catch (Exception e) { + String message = e.getMessage() == null ? "" : e.getMessage(); + Assert.assertTrue( + String.format("Expected deployment failure message to contain '%s', but got: '%s'", + expectedMessageSubstring, message), + message.contains(expectedMessageSubstring)); + } + } + + private int countRows(String table) throws SQLException { + try (Connection connection = getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(String.format("SELECT COUNT(*) FROM \"%s\"", table))) { + resultSet.next(); + return resultSet.getInt(1); + } + } + + private int countRowsWithName(String table, String name) throws SQLException { + try (Connection connection = getConnection(); + PreparedStatement statement = connection.prepareStatement( + String.format("SELECT COUNT(*) FROM \"%s\" WHERE NAME = ?", table))) { + statement.setString(1, name); + try (ResultSet resultSet = statement.executeQuery()) { + resultSet.next(); + return resultSet.getInt(1); + } + } + } + @Test public void testNonExistentDBTable() throws Exception { // source diff --git a/generic-database-plugin/src/test/java/io/cdap/plugin/jdbc/DatabaseSourceTest.java b/generic-database-plugin/src/test/java/io/cdap/plugin/jdbc/DatabaseSourceTest.java new file mode 100644 index 000000000..352655733 --- /dev/null +++ b/generic-database-plugin/src/test/java/io/cdap/plugin/jdbc/DatabaseSourceTest.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jdbc; + +import io.cdap.plugin.util.SelectQueryValidator; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Locale; + +/** + * Unit tests for {@link DatabaseSource}. + */ +public class DatabaseSourceTest { + + @Test + public void testGetSqlDialect() { + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + createSource("jdbc:sqlserver://localhost:1433;databaseName=db", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + createSource("jdbc:jtds:sqlserver://localhost:1433/db", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + createSource("JDBC:SQLSERVER://localhost:1433", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.POSTGRESQL, + createSource("jdbc:postgresql://localhost:5432/db", null).getSqlDialect()); + Assert.assertEquals( + SelectQueryValidator.Dialect.UNKNOWN, + createSource("jdbc:postgresql://localhost:5432/db", "options=-c standard_conforming_strings=off") + .getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL, + createSource("jdbc:mysql://localhost:3306/db", null).getSqlDialect()); + Assert.assertEquals( + SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + createSource("jdbc:mysql://localhost:3306/db", "sessionVariables=sql_mode='NO_BACKSLASH_ESCAPES'") + .getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.DATABRICKS, + createSource("jdbc:databricks://host:443/default", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.ANSI, + createSource("jdbc:oracle:thin:@localhost:1521:xe", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.ANSI, + createSource("jdbc:hsqldb:hsql://localhost/testdb", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.UNKNOWN, + createSource("jdbc:redshift://cluster:5439/dev", null).getSqlDialect()); + Assert.assertEquals(SelectQueryValidator.Dialect.UNKNOWN, + createSource(null, null).getSqlDialect()); + } + + @Test + public void testGetSqlDialectTurkishLocale() { + Locale previous = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + try { + Assert.assertEquals(SelectQueryValidator.Dialect.TSQL, + createSource("JDBC:SQLSERVER://localhost:1433", null).getSqlDialect()); + Assert.assertEquals( + SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + createSource("JDBC:MYSQL://localhost:3306/db", "SESSIONVARIABLES=sql_mode='NO_BACKSLASH_ESCAPES'") + .getSqlDialect()); + } finally { + Locale.setDefault(previous); + } + } + + private static DatabaseSource createSource(String connectionString, String connectionArguments) { + DatabaseSource.DatabaseSourceConfig config = new DatabaseSource.DatabaseSourceConfig(); + config.connectionString = connectionString; + config.connectionArguments = connectionArguments; + return new DatabaseSource(config); + } +} diff --git a/generic-database-plugin/widgets/Database-batchsource.json b/generic-database-plugin/widgets/Database-batchsource.json index bf363027d..009e6af9b 100644 --- a/generic-database-plugin/widgets/Database-batchsource.json +++ b/generic-database-plugin/widgets/Database-batchsource.json @@ -163,6 +163,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/mariadb-plugin/docs/Mariadb-batchsource.md b/mariadb-plugin/docs/Mariadb-batchsource.md index 713af2ee8..dd1ca399b 100644 --- a/mariadb-plugin/docs/Mariadb-batchsource.md +++ b/mariadb-plugin/docs/Mariadb-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** MariaDB database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 6361ba86c..59ab6f992 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -26,10 +26,12 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.HashMap; @@ -64,6 +66,15 @@ protected String createConnectionString() { mariadbSourceConfig.host, mariadbSourceConfig.port, mariadbSourceConfig.database); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (mariadbSourceConfig.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + || SelectQueryValidator.Dialect.mayOverrideMysqlSqlMode(mariadbSourceConfig.connectionArguments)) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + return SelectQueryValidator.Dialect.MYSQL; + } + @Override protected Class getDBRecordType() { return MariadbDBRecord.class; diff --git a/mariadb-plugin/widgets/Mariadb-batchsource.json b/mariadb-plugin/widgets/Mariadb-batchsource.json index 8d197c659..6a56c8eb4 100644 --- a/mariadb-plugin/widgets/Mariadb-batchsource.json +++ b/mariadb-plugin/widgets/Mariadb-batchsource.json @@ -246,6 +246,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/memsql-plugin/docs/Memsql-batchsource.md b/memsql-plugin/docs/Memsql-batchsource.md index b076b885a..0f3569a7d 100644 --- a/memsql-plugin/docs/Memsql-batchsource.md +++ b/memsql-plugin/docs/Memsql-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** MemSQL database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/memsql-plugin/src/main/java/io/cdap/plugin/memsql/source/MemsqlSource.java b/memsql-plugin/src/main/java/io/cdap/plugin/memsql/source/MemsqlSource.java index b9d003441..cccdbe5f3 100644 --- a/memsql-plugin/src/main/java/io/cdap/plugin/memsql/source/MemsqlSource.java +++ b/memsql-plugin/src/main/java/io/cdap/plugin/memsql/source/MemsqlSource.java @@ -19,9 +19,11 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.memsql.MemsqlConstants; import io.cdap.plugin.memsql.MemsqlDBRecord; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; /** @@ -50,4 +52,13 @@ protected String createConnectionString() { return String.format(MemsqlConstants.MEMSQL_CONNECTION_STRING_FORMAT, memsqlSourceConfig.host, memsqlSourceConfig.port, memsqlSourceConfig.database); } + + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (memsqlSourceConfig.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + || SelectQueryValidator.Dialect.mayOverrideMysqlSqlMode(memsqlSourceConfig.connectionArguments)) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + return SelectQueryValidator.Dialect.MYSQL; + } } diff --git a/memsql-plugin/widgets/Memsql-batchsource.json b/memsql-plugin/widgets/Memsql-batchsource.json index 81644861b..89c64c6e0 100644 --- a/memsql-plugin/widgets/Memsql-batchsource.json +++ b/memsql-plugin/widgets/Memsql-batchsource.json @@ -234,6 +234,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/mssql-plugin/docs/SQL Server-connector.md b/mssql-plugin/docs/SQL Server-connector.md index 6f0038715..41b8ffbec 100644 --- a/mssql-plugin/docs/SQL Server-connector.md +++ b/mssql-plugin/docs/SQL Server-connector.md @@ -40,6 +40,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/mssql-plugin/docs/SqlServer-batchsource.md b/mssql-plugin/docs/SqlServer-batchsource.md index 5c917621c..660e4e963 100644 --- a/mssql-plugin/docs/SqlServer-batchsource.md +++ b/mssql-plugin/docs/SqlServer-batchsource.md @@ -35,16 +35,20 @@ error is thrown if it does not match the port. **Database:** SQL Server database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 1f328f4d6..fb7cd254c 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -22,6 +22,6 @@ errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage errorMessage: SQL Exception occurred: [Message='Incorrect syntax near the keyword 'table'.', SQLState='S0001', ErrorCode='156'].. Please check the system logs for more details. errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Could not find stored procedure +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Incorrect syntax near the keyword 'table'. blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: 'S0001', errorCode: '18456', errorMessage: SQL Exception occurred: [Message='Login failed for user ' diff --git a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties index 4421c0c72..37c24ce4b 100644 --- a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -89,3 +89,4 @@ outputDatatypesSchema4=[{"key":"ID","value":"string"},{"key":"COL1","value":"dat {"key":"COL2","value":"datetime"},{"key":"COL3","value":"datetime"}, {"key":"COL4","value":"timestamp"}] mssqlOutputDatatypesSchema=[{"key":"Description","value":"string"}, {"key":"FileData","value":"bytes"}, \ {"key":"Notes","value":"string"}, {"key":"CustomValue","value":"string"}] +invalidBoundingQueryValue=SELECT MIN(id),MAX(id) FROM table diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index db1970549..70c1a389e 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -37,6 +37,7 @@ import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Collections; @@ -81,6 +82,11 @@ protected String getErrorDetailsProviderClassName() { return SqlServerErrorDetailsProvider.class.getName(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.TSQL; + } + @Override protected DBErrorDetailsProvider getErrorDetailsProvider() { if (dbErrorDetailsProvider == null) { diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 198ac1fd4..162857426 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -95,6 +95,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 3f3ad5887..8fed78694 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -286,6 +286,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index d35db27ae..748855bae 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -305,6 +305,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index f586084c1..10f83ffa1 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -36,6 +36,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 552bb5504..09dd1894e 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -32,16 +32,20 @@ You also can use the macro function ${conn(connection-name)}. **Database:** MySQL database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index c3e7f479b..419b02193 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -32,10 +32,12 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.ArrayList; @@ -65,6 +67,21 @@ protected String createConnectionString() { return mysqlSourceConfig.getConnectionString(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + if (!Strings.isNullOrEmpty(mysqlSourceConfig.sqlMode) + || mysqlSourceConfig.containsMacro(MysqlConstants.SQL_MODE)) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + MysqlConnectorConfig connection = mysqlSourceConfig.getConnection(); + if (connection != null + && (connection.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS) + || SelectQueryValidator.Dialect.mayOverrideMysqlSqlMode(connection.getConnectionArguments()))) { + return SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH; + } + return SelectQueryValidator.Dialect.MYSQL; + } + @Override protected Class getDBRecordType() { return MysqlDBRecord.class; diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSourceUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSourceUnitTest.java new file mode 100644 index 000000000..bd850b349 --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSourceUnitTest.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.plugin.util.SelectQueryValidator; +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit tests for {@link MysqlSource} SQL dialect selection. + */ +public class MysqlSourceUnitTest { + + @Test + public void testGetSqlDialectSwitchesToAmbiguousBackslashWhenSqlModeMayBeOverridden() { + MysqlSource.MysqlSourceConfig defaultConfig = new MysqlSource.MysqlSourceConfig(); + defaultConfig.referenceName = "mysqlRef"; + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL, new MysqlSource(defaultConfig).getSqlDialect()); + + MysqlSource.MysqlSourceConfig withSqlMode = new MysqlSource.MysqlSourceConfig(); + withSqlMode.referenceName = "mysqlRef"; + withSqlMode.sqlMode = "NO_BACKSLASH_ESCAPES"; + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + new MysqlSource(withSqlMode).getSqlDialect()); + + MysqlConnectorConfig connWithSessionVars = new MysqlConnectorConfig( + "localhost", 3306, "user", "pass", "mysql", "sessionVariables=sql_mode='NO_BACKSLASH_ESCAPES'"); + MysqlSource.MysqlSourceConfig withConnArgs = new MysqlSource.MysqlSourceConfig() { + @Override + public MysqlConnectorConfig getConnection() { + return connWithSessionVars; + } + }; + withConnArgs.referenceName = "mysqlRef"; + Assert.assertEquals(SelectQueryValidator.Dialect.MYSQL_AMBIGUOUS_BACKSLASH, + new MysqlSource(withConnArgs).getSqlDialect()); + } +} diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 52982aa32..b971d2f37 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -76,6 +76,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index ca23f71c8..3d56a78b9 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -246,6 +246,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index ce6cd047e..aa133eef2 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -281,6 +281,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/netezza-plugin/docs/Netezza-batchsource.md b/netezza-plugin/docs/Netezza-batchsource.md index 6974ba4c4..493015807 100644 --- a/netezza-plugin/docs/Netezza-batchsource.md +++ b/netezza-plugin/docs/Netezza-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** Netezza database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/netezza-plugin/widgets/Netezza-batchsource.json b/netezza-plugin/widgets/Netezza-batchsource.json index 196546791..807224611 100644 --- a/netezza-plugin/widgets/Netezza-batchsource.json +++ b/netezza-plugin/widgets/Netezza-batchsource.json @@ -139,6 +139,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/oracle-plugin/docs/Oracle-batchsource.md b/oracle-plugin/docs/Oracle-batchsource.md index 860801d19..904d1e299 100644 --- a/oracle-plugin/docs/Oracle-batchsource.md +++ b/oracle-plugin/docs/Oracle-batchsource.md @@ -44,16 +44,20 @@ the full TNS Connect Descriptor in the text field. For example: - TRANSACTION_SERIALIZABLE (default): No dirty reads. Non-repeatable and phantom reads are prevented. - Note: If the user role selected is SYSDBA or SYSOPER, the plugin will default to TRANSACTION_READ_COMMITTED to prevent ORA-08178 errors -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/oracle-plugin/docs/Oracle-connector.md b/oracle-plugin/docs/Oracle-connector.md index 9650a2fc0..b38673868 100644 --- a/oracle-plugin/docs/Oracle-connector.md +++ b/oracle-plugin/docs/Oracle-connector.md @@ -42,6 +42,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 24ef25d19..5cb2d2c5d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -35,6 +35,7 @@ import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Map; @@ -62,6 +63,11 @@ protected String createConnectionString() { return oracleSourceConfig.getConnectionString(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.ANSI; + } + @Override protected SchemaReader getSchemaReader() { // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 2b8e2a52a..ffe00a459 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -349,6 +349,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index f5565d78e..936db32f5 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -353,6 +353,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 4aaddc358..4a7a26c52 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -222,6 +222,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/pom.xml b/pom.xml index c739d8855..a04552767 100644 --- a/pom.xml +++ b/pom.xml @@ -310,6 +310,12 @@ failsafe 3.3.2 + + + com.github.jsqlparser + jsqlparser + 4.9 + org.hsqldb hsqldb diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index fe442cbf1..fa5438234 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -38,6 +38,10 @@ This is a semicolon-separated list of key-value pairs, where each pair is separa the key and value for the argument. For example, 'key1=value1;key2=value' specifies that the connection will be given arguments 'key1' mapped to 'value1' and the argument 'key2' mapped to 'value2'. +**Disable Source Query Validation:** Disables read-only `SELECT` validation and read-only session enforcement for +source queries using this connection. Enable only for backward compatibility with trusted pipelines whose source +queries are rejected by the validator. Defaults to `false`. + Path of the connection ---------------------- To browse, get a sample from, or get the specification for this connection through diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index 559723526..96ca5139c 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -32,16 +32,20 @@ You also can use the macro function ${conn(connection-name)}. **Database:** PostgreSQL database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless **Disable Source Query Validation** (`disableQueryValidation`) is set to `true` on the connection. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 661c22a71..711d330ea 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -32,10 +32,12 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import java.util.Map; @@ -63,6 +65,16 @@ protected String createConnectionString() { return postgresSourceConfig.getConnectionString(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + PostgresConnectorConfig connection = postgresSourceConfig.getConnection(); + if (connection != null && connection.containsMacro(ConnectionConfig.CONNECTION_ARGUMENTS)) { + return SelectQueryValidator.Dialect.UNKNOWN; + } + String args = connection == null ? null : connection.getConnectionArguments(); + return SelectQueryValidator.Dialect.fromJdbcUrl("jdbc:postgresql:", args); + } + @Override protected SchemaReader getSchemaReader() { return new PostgresSchemaReader(); diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 88a1714fa..c79141cca 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -80,6 +80,25 @@ "kv-delimiter": "=", "delimiter": ";" } + }, + { + "widget-type": "radio-group", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "True" + }, + { + "id": "false", + "label": "False" + } + ] + } } ] }, diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index f58cb1995..ea5cbab59 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -198,6 +198,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 6d0656e39..3d40d7f80 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -201,6 +201,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/saphana-plugin/docs/SapHana-batchsource.md b/saphana-plugin/docs/SapHana-batchsource.md index e80dda3af..831373df7 100644 --- a/saphana-plugin/docs/SapHana-batchsource.md +++ b/saphana-plugin/docs/SapHana-batchsource.md @@ -27,16 +27,20 @@ Properties **Database:** SAP HANA database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/saphana-plugin/src/main/java/io/cdap/plugin/saphana/SapHanaSource.java b/saphana-plugin/src/main/java/io/cdap/plugin/saphana/SapHanaSource.java index 398724ef2..f23546dda 100644 --- a/saphana-plugin/src/main/java/io/cdap/plugin/saphana/SapHanaSource.java +++ b/saphana-plugin/src/main/java/io/cdap/plugin/saphana/SapHanaSource.java @@ -24,6 +24,7 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; /** @@ -48,6 +49,11 @@ protected String createConnectionString() { sapHanaSourceConfig.port); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.ANSI; + } + @Override protected SchemaReader getSchemaReader() { return new SapHanaSchemaReader(); diff --git a/saphana-plugin/widgets/SapHana-batchsource.json b/saphana-plugin/widgets/SapHana-batchsource.json index 7df341a7e..b017a6a00 100644 --- a/saphana-plugin/widgets/SapHana-batchsource.json +++ b/saphana-plugin/widgets/SapHana-batchsource.json @@ -156,6 +156,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] } diff --git a/teradata-plugin/docs/Teradata-batchsource.md b/teradata-plugin/docs/Teradata-batchsource.md index 1c9eef0a6..36164a848 100644 --- a/teradata-plugin/docs/Teradata-batchsource.md +++ b/teradata-plugin/docs/Teradata-batchsource.md @@ -26,16 +26,20 @@ Properties **Database:** Teradata database name. -**Import Query:** The SELECT query to use to import data from the specified table. +**Import Query:** The single read-only `SELECT` query to use to import data from the specified table. You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. The '$CONDITIONS' string is not required if numSplits is set to one. +Multi-statement queries, write/DDL/administrative statements, `INTO`, locking clauses (such as `FOR UPDATE`/`SHARE`), +dollar-quoted or `q'...'` literals, and executable comments are rejected, and the query is executed on a read-only +connection unless `disableQueryValidation` is set to `true`. -**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +**Bounding Query:** Single read-only `SELECT` query returning the min and max of the values of the 'splitBy' field. For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. -**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. +**Split-By Field Name:** Single column name (optionally table-qualified and/or quoted) which will be used to generate +splits. Not required if numSplits is set to one. **Number of Splits to Generate:** Number of splits to generate. diff --git a/teradata-plugin/src/main/java/io/cdap/plugin/teradata/source/TeradataSource.java b/teradata-plugin/src/main/java/io/cdap/plugin/teradata/source/TeradataSource.java index 9b412c5b6..9b94e0c43 100644 --- a/teradata-plugin/src/main/java/io/cdap/plugin/teradata/source/TeradataSource.java +++ b/teradata-plugin/src/main/java/io/cdap/plugin/teradata/source/TeradataSource.java @@ -25,6 +25,7 @@ import io.cdap.plugin.teradata.TeradataConstants; import io.cdap.plugin.teradata.TeradataDBRecord; import io.cdap.plugin.teradata.TeradataSchemaReader; +import io.cdap.plugin.util.SelectQueryValidator; import org.apache.hadoop.mapreduce.lib.db.DBWritable; /** @@ -52,6 +53,11 @@ protected String createConnectionString() { return config.getConnectionString(); } + @Override + protected SelectQueryValidator.Dialect getSqlDialect() { + return SelectQueryValidator.Dialect.ANSI; + } + @Override protected SchemaReader getSchemaReader() { return new TeradataSchemaReader(); diff --git a/teradata-plugin/widgets/Teradata-batchsource.json b/teradata-plugin/widgets/Teradata-batchsource.json index 2d10020a5..65e339e7d 100644 --- a/teradata-plugin/widgets/Teradata-batchsource.json +++ b/teradata-plugin/widgets/Teradata-batchsource.json @@ -144,6 +144,14 @@ "default": 5, "minimum": 0 } + }, + { + "widget-type": "hidden", + "label": "Disable Source Query Validation", + "name": "disableQueryValidation", + "widget-attributes": { + "default": "false" + } } ] }