From caae1f4aba79aedc30eaaeb50485af29f4a80fdc Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Thu, 16 Jan 2025 18:10:52 +0530 Subject: [PATCH] Additional e2e scenarios for Wrangler --- .../features/WranglerErrorScenarios.feature | 76 +++++++++++++++++ .../e2e-test/features/Wrangler_Macro.feature | 83 +++++++++++++++++++ .../resources/errorMessage.properties | 2 + .../resources/pluginParameters.properties | 4 + 4 files changed, 165 insertions(+) create mode 100644 wrangler-transform/src/e2e-test/features/WranglerErrorScenarios.feature create mode 100644 wrangler-transform/src/e2e-test/features/Wrangler_Macro.feature diff --git a/wrangler-transform/src/e2e-test/features/WranglerErrorScenarios.feature b/wrangler-transform/src/e2e-test/features/WranglerErrorScenarios.feature new file mode 100644 index 000000000..8b8bc4503 --- /dev/null +++ b/wrangler-transform/src/e2e-test/features/WranglerErrorScenarios.feature @@ -0,0 +1,76 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Wrangler_Required +Feature: Wrangler - Verify Wrangler Plugin Error scenarios + + @BQ_SOURCE_CSV_TEST @BQ_SOURCE_TEST @BQ_CONNECTION + Scenario: Verify Wrangler Plugin error when user selects Precondition Language as SQL + Given Open Wrangler connections page + Then Click plugin property: "addConnection" button + Then Click plugin property: "bqConnectionRow" + Then Enter input plugin property: "name" with value: "bqConnectionName" + Then Replace input plugin property: "projectId" with value: "projectId" + Then Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details in Wrangler connection page if set in environment variables + Then Click plugin property: "testConnection" button + Then Verify the test connection is successful + Then Click plugin property: "connectionCreate" button + Then Verify the connection with name: "bqConnectionName" is created successfully + Then Select connection data row with name: "dataset" + Then Select connection data row with name: "bqSourceTable" + Then Verify connection datatable is displayed for the data: "bqSourceTable" + Then Click Create Pipeline button and choose the type of pipeline as: "Batch pipeline" + Then Verify plugin: "BigQueryTable" node is displayed on the canvas with a timeout of 120 seconds + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Wrangler" + Then Select radio button plugin property: "expressionLanguage" with value: "sql" + Then Click on the Validate button + Then Verify that the Plugin Property: "directives" is displaying an in-line error message: "errorMessageSqlError" + Given Open Wrangler connections page + Then Expand connections of type: "BigQuery" + Then Open action menu for connection: "bqConnectionName" of type: "BigQuery" + Then Select action: "Delete" for connection: "bqConnectionName" of type: "BigQuery" + Then Click plugin property: "Delete" button + Then Verify connection: "bqConnectionName" of type: "BigQuery" is deleted successfully + + @BQ_SOURCE_CSV_TEST @BQ_SOURCE_TEST @BQ_CONNECTION + Scenario: Verify Wrangler Plugin error when user provides invalid input field Name + Given Open Wrangler connections page + Then Click plugin property: "addConnection" button + Then Click plugin property: "bqConnectionRow" + Then Enter input plugin property: "name" with value: "bqConnectionName" + Then Replace input plugin property: "projectId" with value: "projectId" + Then Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details in Wrangler connection page if set in environment variables + Then Click plugin property: "testConnection" button + Then Verify the test connection is successful + Then Click plugin property: "connectionCreate" button + Then Verify the connection with name: "bqConnectionName" is created successfully + Then Select connection data row with name: "dataset" + Then Select connection data row with name: "bqSourceTable" + Then Verify connection datatable is displayed for the data: "bqSourceTable" + Then Click Create Pipeline button and choose the type of pipeline as: "Batch pipeline" + Then Verify plugin: "BigQueryTable" node is displayed on the canvas with a timeout of 120 seconds + Then Navigate to the properties page of plugin: "Wrangler" + Then Replace input plugin property: "field" with value: "invalid" + Then Click on the Validate button + Then Verify that the Plugin Property: "field" is displaying an in-line error message: "errorMessageInvalidInputFieldName" + Given Open Wrangler connections page + Then Expand connections of type: "BigQuery" + Then Open action menu for connection: "bqConnectionName" of type: "BigQuery" + Then Select action: "Delete" for connection: "bqConnectionName" of type: "BigQuery" + Then Click plugin property: "Delete" button + Then Verify connection: "bqConnectionName" of type: "BigQuery" is deleted successfully + diff --git a/wrangler-transform/src/e2e-test/features/Wrangler_Macro.feature b/wrangler-transform/src/e2e-test/features/Wrangler_Macro.feature new file mode 100644 index 000000000..c2aa9e2f8 --- /dev/null +++ b/wrangler-transform/src/e2e-test/features/Wrangler_Macro.feature @@ -0,0 +1,83 @@ +# Copyright © 2024 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Wrangler +Feature: Wrangler - Run time scenarios for wrangler plugin using macro arguments + + @BQ_SOURCE_CSV_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST @BQ_CONNECTION + Scenario: To verify User is able to run a pipeline using macro arguments + Given Open Wrangler connections page + Then Click plugin property: "addConnection" button + Then Click plugin property: "bqConnectionRow" + Then Enter input plugin property: "name" with value: "bqConnectionName" + Then Replace input plugin property: "projectId" with value: "projectId" + Then Enter input plugin property: "datasetProjectId" with value: "projectId" + Then Override Service account details in Wrangler connection page if set in environment variables + Then Click plugin property: "testConnection" button + Then Verify the test connection is successful + Then Click plugin property: "connectionCreate" button + Then Verify the connection with name: "bqConnectionName" is created successfully + Then Select connection data row with name: "dataset" + Then Select connection data row with name: "bqSourceTable" + Then Verify connection datatable is displayed for the data: "bqSourceTable" + Then Expand dropdown column: "body" and apply directive: "Parse" as "CSV" with: "Comma" option + Then Expand dropdown column: "body_3" and apply directive: "FillNullOrEmptyCells" as "shubh" + Then Enter directive from CLI "rename body_1 new_id" + Then Enter directive from CLI "quantize body_4 body_q 1:2=20,3:4=40" + Then Expand dropdown column: "body_4" and apply directive: "ChangeDataType" as "Integer" + Then Enter directive from CLI "columns-replace s/^new_//g" + Then Enter directive from CLI "set-headers :abc" + Then Enter directive from CLI "change-column-case uppercase" + Then Enter directive from CLI "cleanse-column-names " + Then Enter directive from CLI "split-to-rows :id '#'" + Then Click Create Pipeline button and choose the type of pipeline as: "Batch pipeline" + Then Verify plugin: "BigQueryTable" node is displayed on the canvas with a timeout of 120 seconds + Then Navigate to the properties page of plugin: "Wrangler" + Then Click on the Macro button of Property: "field" and set the value to: "fields" + Then Click on the Macro button of Property: "expressionLanguage" and set the value to: "expressionLanguage" + Then Click on the Macro button of Property: "precondition" and set the value to: "precondition" + Then Click on the Macro button of Property: "on-error" and set the value to: "on-error" + Then Validate "Wrangler" plugin properties + Then Close the Plugin Properties page + Then Expand Plugin group in the LHS plugins list: "Sink" + Then Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "BigQuery2" + Then Click plugin property: "useConnection" + Then Click on the Browse Connections button + Then Select connection: "bqConnectionName" + Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqTargetTable" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Connect plugins: "Wrangler" and "BigQuery2" to establish connection + Then Save the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "expressionLanguage" for key "expressionLanguage" + Then Enter runtime argument value "fields" for key "fields" + Then Enter runtime argument value "precondition" for key "precondition" + Then Enter runtime argument value "onError" for key "on-error" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_csv" + Given Open Wrangler connections page + Then Expand connections of type: "BigQuery" + Then Open action menu for connection: "bqConnectionName" of type: "BigQuery" + Then Select action: "Delete" for connection: "bqConnectionName" of type: "BigQuery" + Then Click plugin property: "Delete" button + Then Verify connection: "bqConnectionName" of type: "BigQuery" is deleted successfully diff --git a/wrangler-transform/src/e2e-test/resources/errorMessage.properties b/wrangler-transform/src/e2e-test/resources/errorMessage.properties index 87c823507..22b9f5edc 100644 --- a/wrangler-transform/src/e2e-test/resources/errorMessage.properties +++ b/wrangler-transform/src/e2e-test/resources/errorMessage.properties @@ -1 +1,3 @@ validationSuccessMessage=No errors found. +errorMessageInvalidInputFieldName=Field 'invalid' must be present in input schema. +errorMessageSqlError=Directives are not supported for precondition of type SQL diff --git a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties index ea5e3efd3..17ec4b64a 100644 --- a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties +++ b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties @@ -17,6 +17,10 @@ bqTargetTable=dummy sourcePath=example/hello.csv gcsSourceBucket=dummy testFile=BQtesdata/BigQuery/test1.xlsx +fields=* +expressionLanguage=JEXL +onError=Fail pipeline +precondition=false #bq queries file path CreateBQDataQueryFileFxdLen=BQtesdata/BigQuery/BigQueryCreateTableQueryFxdlen.txt