diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature index 1385cc8c1..a3b751371 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature @@ -171,3 +171,93 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Enter input plugin property: "referenceName" with value: "targetRef" And Click on the Validate button Then Verify that the Plugin is displaying an error message: "errormessageBlankHost" on the header + + @Mssql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | tableName | + + @Mssql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET @Mssql_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET @Mssql_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature index 31e731858..60d951651 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature @@ -60,3 +60,55 @@ Feature: Mssql - Verify Mssql sink data transfer Then Close the pipeline logs Then Validate the values of records transferred to target MsSql table is equal to the values from source BigQuery table + @MSSQL_SOURCE @MSSQL_TARGET @CONNECTION @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to Mssql successfully with use connection using MAX type datatypes + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "mssqlOutputDatatypesSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Mssql sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table are equal to number of records from the source table diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature index 775df040e..ddfb3d2a3 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature @@ -203,3 +203,87 @@ Feature: Mssql Sink - Run time scenarios (macro) Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorMessageInvalidCredentials | + + @MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST @MSSQL_TARGET_DATATYPES_UIDTYPE_TEST @Mssql_Required + Scenario: To verify data is getting transferred from MsSQL to MsSQL successfully when macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Macro button of Property: "jdbcPluginName" and set the value to: "SQLServerDriverName" + Then Click on the Macro button of Property: "host" and set the value to: "SQLServerHost" + Then Click on the Macro button of Property: "port" and set the value to: "SQLServerPort" + Then Click on the Macro button of Property: "user" and set the value to: "SQLServerUsername" + Then Click on the Macro button of Property: "password" and set the value to: "SQLServerPassword" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSource" + Then Click on the Macro button of Property: "database" and set the value to: "SQLServerDatabaseName" + Then Click on the Macro button of Property: "importQuery" and set the value in textarea: "SQLServerImportQuery" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Click on the Macro button of Property: "numSplits" and set the value to: "SQLServerNumSplits" + Then Click on the Macro button of Property: "fetchSize" and set the value to: "SQLServerFetchSize" + Then Click on the Macro button of Property: "splitBy" and set the value to: "SQLServerSplitByColumn" + Then Click on the Macro button of Property: "boundingQuery" and set the value in textarea: "SQLServerBoundingQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Click on the Macro button of Property: "jdbcPluginName" and set the value to: "SQLServerDriverName" + Then Click on the Macro button of Property: "host" and set the value to: "SQLServerHost" + Then Click on the Macro button of Property: "port" and set the value to: "SQLServerPort" + Then Click on the Macro button of Property: "user" and set the value to: "SQLServerUsername" + Then Click on the Macro button of Property: "password" and set the value to: "SQLServerPassword" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "database" and set the value to: "SQLServerDatabaseName" + Then Enter input plugin property: "referenceName" with value: "targetRef" + And Click on the Macro button of Property: "tableName" and set the value to: "SQLServerTableName" + And Click on the Macro button of Property: "dbSchemaName" and set the value to: "SQLServerSchemaName" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "driverName" for key "SQLServerDriverName" + Then Enter runtime argument value from environment variable "host" for key "SQLServerHost" + Then Enter runtime argument value from environment variable "port" for key "SQLServerPort" + Then Enter runtime argument value from environment variable "username" for key "SQLServerUsername" + Then Enter runtime argument value from environment variable "password" for key "SQLServerPassword" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "fetchSize" for key "SQLServerFetchSize" + Then Enter runtime argument value "splitByField" for key "SQLServerSplitByColumn" + Then Enter runtime argument value "numberOfSplits" for key "SQLServerNumSplits" + Then Enter runtime argument value "selectQuery" for key "SQLServerImportQuery" + Then Enter runtime argument value "databaseName" for key "SQLServerDatabaseName" + Then Enter runtime argument value "boundingQuery" for key "SQLServerBoundingQuery" + Then Enter runtime argument value "targetTable" for key "SQLServerTableName" + Then Enter runtime argument value "schema" for key "SQLServerSchemaName" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "driverName" for key "SQLServerDriverName" + Then Enter runtime argument value from environment variable "host" for key "SQLServerHost" + Then Enter runtime argument value from environment variable "port" for key "SQLServerPort" + Then Enter runtime argument value from environment variable "username" for key "SQLServerUsername" + Then Enter runtime argument value from environment variable "password" for key "SQLServerPassword" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "fetchSize" for key "SQLServerFetchSize" + Then Enter runtime argument value "splitByField" for key "SQLServerSplitByColumn" + Then Enter runtime argument value "numberOfSplits" for key "SQLServerNumSplits" + Then Enter runtime argument value "selectQuery" for key "SQLServerImportQuery" + Then Enter runtime argument value "databaseName" for key "SQLServerDatabaseName" + Then Enter runtime argument value "boundingQuery" for key "SQLServerBoundingQuery" + Then Enter runtime argument value "targetTable" for key "SQLServerTableName" + Then Enter runtime argument value "schema" for key "SQLServerSchemaName" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate records transferred to target table are equal to number of records from the source table diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature index 74277fa77..cc3bf8562 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature @@ -200,3 +200,31 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "errorMessageBoundingQuery" Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessagenumofSplit" + + @Mssql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | importQuery | + + @Mssql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header \ No newline at end of file diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature index 342a576d3..75d2d810d 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature @@ -184,3 +184,152 @@ Feature: Mssql - Verify Mssql source data transfer Then Wait till pipeline is in running state Then Open and capture logs And Verify the pipeline status is "Failed" + + @MSSQL_AS_SOURCE @BQ_SINK_TEST @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to BigQuery successfully with bounding Query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Enter textarea plugin property: "boundingQuery" with value: "boundingQuery" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqtarget.table" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + Then Save the pipeline + And Preview and run the pipeline + And Wait till pipeline preview is in running state + And Open and capture pipeline preview logs + And Verify the preview run status of pipeline in the logs is "succeeded" + And Close the pipeline logs + And Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table + + @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_TARGET_DATATYPES_TEST @CONNECTION @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to Mssql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Mssql sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table are equal to number of records from the source table + + @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_TARGET_DATATYPES_TEST @CONNECTION @Mssql_Required + Scenario: To verify pipeline failure message in logs when an invalid bounding query is provided + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "splitBy" with value: "splitBy" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "numberOfSplits" + Then Enter textarea plugin property: "boundingQuery" with value: "invalidBoundingQueryValue" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Failed" + And Close the pipeline logs + Then Open Pipeline logs and verify Log entries having below listed Level and Message: + | Level | Message | + | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java index 0d1712658..92c354c48 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java @@ -255,5 +255,36 @@ private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarg rsTarget.next()); return true; } + + public static void createMssqlSourceTable(String sourceTable, String schema) + throws SQLException, ClassNotFoundException { + try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + " (" + + "Description VARCHAR(MAX), " + + "FileData VARBINARY(MAX), " + + "Notes NVARCHAR(MAX), " + + "CustomValue NVARCHAR(MAX))"; + statement.executeUpdate(createSourceTableQuery); + String insertDataQuery = "INSERT INTO " + schema + "." + sourceTable + + " (Description, FileData, Notes, CustomValue) VALUES (" + + "'This is a long description stored in VARCHAR(MAX)', " + + "CONVERT(VARBINARY(MAX), 'BinaryDataExample'), " + + "N'This is a Unicode note stored in NVARCHAR(MAX)', " + + "'123.45')"; + statement.executeUpdate(insertDataQuery); + } + } + + public static void createMssqlTargetTable(String targetTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) { + String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable + " (" + + "Description VARCHAR(MAX), " + + "FileData VARBINARY(MAX), " + + "Notes NVARCHAR(MAX), " + + "CustomValue NVARCHAR(MAX))"; + statement.executeUpdate(createTargetTableQuery); + } + } } diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java index 74ae9f911..902c82ff7 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java @@ -17,6 +17,8 @@ package io.cdap.plugin.common.stepsdesign; import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.pages.actions.CdfConnectionActions; +import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MssqlClient; @@ -51,8 +53,10 @@ public static void setTableName() { String targetTableName = String.format("TARGETTABLE_%s", randomString); PluginPropertyUtils.addPluginProp("targetTable", targetTableName); targetTable = targetTableName; - PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s.%s", schema, - sourceTable)); + PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s.%s" + + " WHERE $CONDITIONS", schema, sourceTable)); + PluginPropertyUtils.addPluginProp("boundingQuery", String.format("select MIN(ID),MAX(ID)" + + " from %s.%s", schema, sourceTable)); } @Before(order = 2, value = "@MSSQL_AS_SOURCE") @@ -264,4 +268,49 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce } } } + + @Before(order = 1, value = "@CONNECTION") + public static void setNewConnectionName() { + String connectionName = "SQL Server" + RandomStringUtils.randomAlphanumeric(10); + PluginPropertyUtils.addPluginProp("connection.name", connectionName); + BeforeActions.scenario.write("New Connection name: " + connectionName); + } + + private static void deleteConnection(String connectionType, String connectionName) throws IOException { + CdfConnectionActions.openWranglerConnectionsPage(); + CdfConnectionActions.expandConnections(connectionType); + CdfConnectionActions.openConnectionActionMenu(connectionType, connectionName); + CdfConnectionActions.selectConnectionAction(connectionType, connectionName, "Delete"); + CdfPluginPropertiesActions.clickPluginPropertyButton("Delete"); + } + + @After(order = 1, value = "@CONNECTION") + public static void deleteTestConnection() throws IOException { + deleteConnection("SQL Server", "connection.name"); + PluginPropertyUtils.removePluginProp("connection.name"); + } + + @Before(order = 2, value = "@MSSQL_SOURCE") + public static void createMssqlSourceTables() throws SQLException, ClassNotFoundException { + MssqlClient.createMssqlSourceTable(sourceTable, schema); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + sourceTable + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_SOURCE") + public static void dropMssqlSourceTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(schema, sourceTable); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + sourceTable + " deleted successfully"); + } + + @Before(order = 2, value = "@MSSQL_TARGET") + public static void createMssqlTargetTables() throws SQLException, ClassNotFoundException { + MssqlClient.createMssqlTargetTable(targetTable, schema); + BeforeActions.scenario.write("MSSQL TARGET Table - " + targetTable + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_TARGET") + public static void dropMssqlTargetTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(schema, targetTable); + BeforeActions.scenario.write("MSSQL TARGET Table - " + targetTable + " deleted successfully"); + } } diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index c752d6ec1..e66697fd8 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -21,3 +21,6 @@ errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Could not find stored procedure +blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. +blank.connection.message=Exception while trying to validate schema of database table diff --git a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties index ede609c0e..4421c0c72 100644 --- a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -39,6 +39,9 @@ invalid.Host=MSSQL_HOST1 invalid.Port=MSSQL_PORT1 invalid.Username=MSSQL_USERNAME1 invalid.Password=MSSQL_PASSWORD1 +splitByField=ID +numSplits=1 +connectionArguments=queryTimeout=50 #bq queries file path CreateBQTableQueryFile=testData/BigQuery/BigQueryCreateTableQuery.txt @@ -84,3 +87,5 @@ dateTimeValues=VALUES ('User1', '2023-01-01 01:00:00.000', '2023-01-01 01:00:00. '2025-12-10 12:32:10.000 +01:00') outputDatatypesSchema4=[{"key":"ID","value":"string"},{"key":"COL1","value":"datetime"},\ {"key":"COL2","value":"datetime"},{"key":"COL3","value":"datetime"}, {"key":"COL4","value":"timestamp"}] +mssqlOutputDatatypesSchema=[{"key":"Description","value":"string"}, {"key":"FileData","value":"bytes"}, \ + {"key":"Notes","value":"string"}, {"key":"CustomValue","value":"string"}]