From 43a4ed74a586a89c93abcf5855381ad3203b3d00 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Thu, 11 Jan 2024 19:59:13 +0530 Subject: [PATCH] addressed comments for version range --- .../features/Wrangler/DataTypeParsers.feature | 4 +--- .../parseAsAvro-cdap-data-pipeline (1).json | 14 ++++---------- .../parse_HL7_Wrangler-cdap-data-pipeline (1).json | 14 ++++---------- .../parse_csv_wrangle-cdap-data-pipeline.json | 14 ++++---------- .../parse_datetime_wrangle-cdap-data-pipeline.json | 14 ++++---------- ...rse_fixedlength_wrangle-cdap-data-pipeline.json | 14 ++++---------- ...parse_log_wrangler_copy-cdap-data-pipeline.json | 14 ++++---------- ...parse_timestamp_wrangle-cdap-data-pipeline.json | 14 ++++---------- 8 files changed, 29 insertions(+), 73 deletions(-) diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature b/wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature index fefe29a8c..c299b2735 100644 --- a/wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature +++ b/wrangler-transform/src/e2e-test/features/Wrangler/DataTypeParsers.feature @@ -15,7 +15,7 @@ @Wranglers Feature: datatype parsers - @BQ_SOURCE_TS_TEST @BQ_SINK_TEST + @BQ_SOURCE_TS_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST Scenario: To verify User is able to run a pipeline using parse timestamp directive Given Open Datafusion Project to configure pipeline Then Click on the Plus Green Button to import the pipelines @@ -24,8 +24,6 @@ Feature: datatype parsers Then Replace input plugin property: "project" with value: "projectId" Then Replace input plugin property: "dataset" with value: "dataset" Then Replace input plugin property: "table" with value: "bqSourceTable" - Then Click on the Get Schema button - Then Click on the Validate button Then Close the Plugin Properties page Then Navigate to the properties page of plugin: "BigQuery2" Then Replace input plugin property: "project" with value: "projectId" diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json index bfea9fd59..7977f2c52 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -258,9 +256,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-avro-file :body\nfilter-rows-on condition-true name == null || name =~ \"^\\W*$\"\ncut-character name nameNew 2-3\nset-type :value integer ", @@ -377,9 +373,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json index 550be7934..59af75ce2 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -76,9 +74,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-hl7 :Body\nhash :Body MD5\nset-type :Body string \nkeep address,Body,Body_hl7_MSH_12,Body_hl7_MSH_9_1\nfind-and-replace address s/address1/test/g\nmask-shuffle :Body_hl7_MSH_9_1\nsend-to-error empty(address)\nrename :Body_hl7_MSH_12 :id ", @@ -119,9 +115,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json index dfad86afe..9ca1207c2 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -76,9 +74,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-csv :body ',' false\nrename body_1 new_id\nquantize body_4 body_q 1:2=20,3:4=40\nset-type :body_4 integer \ncolumns-replace s/^new_//g\nfill-null-or-empty :body_3 'shubh'\nset-headers :abc\nchange-column-case uppercase\ncleanse-column-names\nsplit-to-rows :id '#'", @@ -119,9 +115,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json index 99dc55803..a21c5adca 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangle-cdap-data-pipeline.json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -257,9 +255,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-datetime :timestamp \"yyyy-MM-dd'T'HH:mm:ssX'['z']'\"\ncurrent-datetime :create_date\ndatetime-to-timestamp :timestamp\nformat-datetime :create_date 'y'\nformat-date :timestamp yyyy-mm-dd\nrename timestamp timecolumn", @@ -301,9 +297,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json index c1e5abead..4bfe5d838 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangle-cdap-data-pipeline.json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -76,9 +74,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-fixed-length :fixedlength 2,4,5,3\nsplit-url :url\nwrite-as-csv :url_protocol\nurl-encode :url\nurl-decode :url\nencode base32 :fixedlength\ndecode base32 :fixedlength_encode_base32\nsplit-to-columns :url_query '='\nrename :fixedlength_2 :id\nfilter-rows-on condition-true fixedlength_4 !~ 'XYZ'", @@ -119,9 +115,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json index 8c4348dc1..62585c49c 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.11.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -76,9 +74,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-as-log :Body 'common'\ndrop :number_connection_client_logname_last\ndrop :number_connection_client_logname\ndrop :http_querystring_request_firstline_uri_query\nfill-null-or-empty :http_ref_request_firstline_uri_ref 'no value'\nfilter-rows-on condition-false ip_connection_client_host =$ \"1\"\nfilter-rows-on regex-match string_connection_client_user_last ^Tryck$\nextract-regex-groups :http_firstline_request_firstline GET*\nsplit-to-columns :ip_connection_client_host ,\ncatalog-lookup ICD-9 :Body\nfill-null-or-empty :Body_icd_9_description 'Body Post Catalog lookup'\nset-variable Pass string_request_status_last == \"200\" ? string_request_status_last : Pass\nset-type :time_day_request_receive_time_last_day integer ", @@ -112,9 +108,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json index 805e58cc6..4bf09c0c1 100644 --- a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json @@ -3,7 +3,7 @@ "description": "Data Pipeline Application", "artifact": { "name": "cdap-data-pipeline", - "version": "6.10.0-SNAPSHOT", + "version": "[6.0.0, 7.0.0]", "scope": "SYSTEM" }, "config": { @@ -37,9 +37,7 @@ "type": "batchsource", "label": "BigQueryTable", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false", @@ -76,9 +74,7 @@ "type": "transform", "label": "Wrangler", "artifact": { - "name": "wrangler-transform", - "version": "4.11.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "wrangler-transform" }, "properties": { "directives": "parse-timestamp :time\nparse-as-currency :price :newprice\nformat-as-currency :newprice :format_price\nparse-as-simple-date :create_date yyyy-MM-dd\nparse-as-simple-date :update_date yyyy-MM-dd\ndiff-date :create_date :update_date :diff_date\ntimestamp-to-datetime :update_date\nrename :newprice :id", @@ -119,9 +115,7 @@ "type": "batchsink", "label": "BigQuery2", "artifact": { - "name": "google-cloud", - "version": "0.24.0-SNAPSHOT", - "scope": "SYSTEM" + "name": "google-cloud" }, "properties": { "useConnection": "false",