Merge pull request #534 from cloudsufi/e2e-mysqlsink-tests

e2e mysql sink additional tests
data-integrations · Jan 6, 2025 · 9f13a64 · 9f13a64
2 parents 80e17e5 + 954447b
commit 9f13a64
Show file tree

Hide file tree

Showing 4 changed files with 196 additions and 0 deletions.
diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature
@@ -167,3 +167,91 @@ Feature: MySQL Sink - Design time validation scenarios
     Then Enter input plugin property: "referenceName" with value: "targetRef"
     Then Click on the Validate button
     Then Verify that the Plugin is displaying an error message: "invalid.host.message" on the header
+
+  @Mysql_Required
+  Scenario: Verify required fields missing validation messages
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Click on the Validate button
+    Then Verify mandatory property error for below listed properties:
+      | jdbcPluginName |
+      | referenceName  |
+      | database       |
+      | tableName      |
+
+  @Mysql_Required
+  Scenario: Verify the validation error message with missing jdbc plugin name
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Click plugin property: "switch-useConnection"
+    Then Click on the Validate button
+    Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header
+
+  @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required
+  Scenario: Verify the validation error message with blank password value
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "MySQL" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Connect plugins: "MySQL" and "MySQL2" to establish connection
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Enter input plugin property: "referenceName" with value: "sourceRef"
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
+    Then Validate "MySQL" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "MySQL2"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Replace input plugin property: "tableName" with value: "targetTable"
+    Then Enter input plugin property: "referenceName" with value: "targetRef"
+    Then Click on the Validate button
+    Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header
+
+  @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required
+  Scenario: Verify the validation error message with blank host value
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "MySQL" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Connect plugins: "MySQL" and "MySQL2" to establish connection
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Enter input plugin property: "referenceName" with value: "sourceRef"
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
+    Then Validate "MySQL" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "MySQL2"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Replace input plugin property: "tableName" with value: "targetTable"
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Enter input plugin property: "referenceName" with value: "targetRef"
+    Then Click on the Validate button
+    Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header
diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature
@@ -147,3 +147,57 @@ Feature: MySQL Sink - Run time scenarios
     Then Open and capture logs
     Then Verify the pipeline status is "Succeeded"
     Then Validate the values of records transferred to target table is equal to the values from source table
+
+  @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @CONNECTION @Mysql_Required
+  Scenario: To verify data is getting transferred from BigQuery to Mysql successfully with use connection
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    And Navigate to the properties page of plugin: "BigQuery"
+    And Enter input plugin property: "referenceName" with value: "Reference"
+    And Replace input plugin property: "project" with value: "projectId"
+    And Enter input plugin property: "datasetProject" with value: "datasetprojectId"
+    And Enter input plugin property: "dataset" with value: "dataset"
+    And Enter input plugin property: "table" with value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema"
+    Then Validate "BigQuery" plugin properties
+    And Close the Plugin Properties page
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "MySQL" to establish connection
+    Then Navigate to the properties page of plugin: "MySQL"
+    And Click plugin property: "switch-useConnection"
+    And Click on the Browse Connections button
+    And Click on the Add Connection button
+    And Select Mysql Connection
+    And Enter input plugin property: "name" with value: "connection.name"
+    Then Enter input plugin property: "referenceName" with value: "sinkRef"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Click on the Test Connection button
+    And Verify the test connection is successful
+    Then Click on the Create button
+    And Use new connection
+    Then Enter input plugin property: "referenceName" with value: "targetRef"
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Replace input plugin property: "tableName" with value: "targetTable"
+    Then Click plugin property: "useCompression"
+    Then Click plugin property: "autoReconnect"
+    Then Validate "MySQL" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Verify the preview of pipeline is "success"
+    Then Click on preview data for MySQL sink
+    Then Verify preview output schema matches the outputSchema captured in properties
+    Then Close the preview data
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate the values of records transferred to target MySQL table is equal to the values from source BigQuery table
diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature
@@ -139,3 +139,55 @@ Feature: MySQL Sink - Run time scenarios (macro)
     And Open and capture logs
     And Verify the pipeline status is "Failed"
     And Close the pipeline logs
+
+  @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required
+  Scenario: To verify data is getting transferred from Mysql to Mysql successfully with connection arguments, databasename and tablename macro enabled
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "MySQL" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Connect plugins: "MySQL" and "MySQL2" to establish connection
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Enter input plugin property: "referenceName" with value: "sourceRef"
+    Then Replace input plugin property: "database" with value: "databaseName"
+    Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
+    Then Click on the Get Schema button
+    Then Verify the Output Schema matches the Expected Schema: "outputSchema"
+    Then Validate "MySQL" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "MySQL2"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
+    Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
+    Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
+    And Click on the Macro button of Property: "database" and set the value to: "databaseName"
+    And Click on the Macro button of Property: "tableName" and set the value to: "tableName"
+    Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink"
+    Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
+    Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
+    Then Enter input plugin property: "referenceName" with value: "targetRef"
+    Then Validate "MySQL2" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Enter runtime argument value "databaseName" for key "databaseName"
+    Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink"
+    Then Enter runtime argument value "targetTable" for key "tableName"
+    And Run the preview of pipeline with runtime arguments
+    Then Verify the preview of pipeline is "success"
+    And Close the preview
+    And Deploy the pipeline
+    And Run the Pipeline in Runtime
+    Then Enter runtime argument value "databaseName" for key "databaseName"
+    Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink"
+    Then Enter runtime argument value "targetTable" for key "tableName"
+    And Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate the values of records transferred to target table is equal to the values from source table
diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties
@@ -14,3 +14,5 @@ blank.username.message=Username is required when password is given.
 errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer.
 errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1.
 errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'.
+blank.connection.message=Exception while trying to validate schema of database table 
+blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value.