Skip to content

Commit

Permalink
addressed comments for version range
Browse files Browse the repository at this point in the history
  • Loading branch information
@AnkitCLI
AnkitCLI committed Jan 11, 2024
1 parent 17bb5a9 commit b6a4ed0
Show file tree
Hide file tree
Showing 8 changed files with 32 additions and 80 deletions.
14 changes: 4 additions & 10 deletions ...m/src/e2e-test/resources/testData/Wrangler/parse_HL7_Wrangler-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-hl7 :Body\nhash :Body MD5\nset-type :Body string \nkeep address,Body,Body_hl7_MSH_12,Body_hl7_MSH_9_1\nfind-and-replace address s/address1/test/g\nmask-shuffle :Body_hl7_MSH_9_1\nsend-to-error empty(address)\nrename :Body_hl7_MSH_12 :id ",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...nsform/src/e2e-test/resources/testData/Wrangler/parse_csv_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-csv :body ',' false\nrename body_1 new_id\nquantize body_4 body_q 1:2=20,3:4=40\nset-type :body_4 integer \ncolumns-replace s/^new_//g\nfill-null-or-empty :body_3 'shubh'\nset-headers :abc\nchange-column-case uppercase\ncleanse-column-names\nsplit-to-rows :id '#'",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions .../src/e2e-test/resources/testData/Wrangler/parse_datetime_wrangler-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.11.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-datetime :timestamp \"yyyy-MM-dd'T'HH:mm:ssX'['z']'\"\ncurrent-datetime :create_date\ndatetime-to-timestamp :timestamp\nformat-datetime :create_date 'y'\nformat-date :timestamp yyyy-mm-dd\nrename timestamp timecolumn",
Expand Down Expand Up @@ -112,9 +108,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...rc/e2e-test/resources/testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "GCSFile",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"format": "blob",
Expand Down Expand Up @@ -82,9 +80,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-excel :body '0' true\ncopy name copiedname\nmerge name bkd uniquenum ','\nrename bkd rollno\ndrop fwd\nswap id rollno\nsplit-to-rows :name 'o'\nfilter-rows-on condition-false rollno !~ '2.0'",
Expand Down Expand Up @@ -125,9 +121,7 @@
"type": "batchsink",
"label": "BigQuery",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...c/e2e-test/resources/testData/Wrangler/parse_fixedlength_wrangler-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.11.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-fixed-length :fixedlength 2,4,5,3\nsplit-url url\nwrite-as-csv :url_protocol\nurl-encode :url\nurl-decode :url\nencode base32 fixedlength\ndecode base32 fixedlength_encode_base32\nsplit-to-columns :url_query '='\nrename fixedlength_2 id\nfilter-rows-on condition-true fixedlength_4 !~ 'XYZ'",
Expand Down Expand Up @@ -112,9 +108,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...orm/src/e2e-test/resources/testData/Wrangler/parse_json_wrangler1-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.11.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -77,9 +75,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-as-json :json 1\nltrim :body\nset-column :desc concat(json_pet,body)\ncopy :json_name :copied\nswap :json_id :json_age\nmerge :json_id :json_name :json_id_json_name ,\nmask-number :json_pet 'testing'\ndrop json_height\nwrite-as-json-map :json_age\nrename json_id id",
Expand Down Expand Up @@ -114,9 +110,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions .../src/e2e-test/resources/testData/Wrangler/parse_timestamp_wrangle-cdap-data-pipeline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.10.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -76,9 +74,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-timestamp :time\nparse-as-currency :price :newprice\nformat-as-currency :newprice :format_price\nparse-as-simple-date :create_date yyyy-MM-dd\nparse-as-simple-date :update_date yyyy-MM-dd\ndiff-date :create_date :update_date :diff_date\ntimestamp-to-datetime :update_date\nrename :newprice :id",
Expand Down Expand Up @@ -119,9 +115,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down
14 changes: 4 additions & 10 deletions ...e2e-test/resources/testData/Wrangler/parse_xmltojson_wrangler-cdap-data-pipeline (1).json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"description": "Data Pipeline Application",
"artifact": {
"name": "cdap-data-pipeline",
"version": "6.11.0-SNAPSHOT",
"version": "[4.0.0, 5.0.0]",
"scope": "SYSTEM"
},
"config": {
Expand Down Expand Up @@ -37,9 +37,7 @@
"type": "batchsource",
"label": "BigQueryTable",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down Expand Up @@ -257,9 +255,7 @@
"type": "transform",
"label": "Wrangler",
"artifact": {
"name": "wrangler-transform",
"version": "4.11.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "wrangler-transform"
},
"properties": {
"directives": "parse-xml-to-json :xmldata 1\nsplit-email :email\ntext-distance block email email_account distance\ntext-metric longest-common-subsequence email email_account distance2\nwrite-as-json-object :email_domain distance,email_account\nstemming :email\nsplit-to-rows :email_account '0'\nrename :email_account id",
Expand Down Expand Up @@ -302,9 +298,7 @@
"type": "batchsink",
"label": "BigQuery2",
"artifact": {
"name": "google-cloud",
"version": "0.24.0-SNAPSHOT",
"scope": "SYSTEM"
"name": "google-cloud"
},
"properties": {
"useConnection": "false",
Expand Down

0 comments on commit b6a4ed0

Please sign in to comment.