diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsAvro.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsAvro.feature new file mode 100644 index 000000000..1f17583fb --- /dev/null +++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsAvro.feature @@ -0,0 +1,43 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Wrangler +Feature: Wrangler - Run time scenarios for Parse Avro + + @BQ_SOURCE_AVRO_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST + Scenario: To verify User is able to run a pipeline using parse avro directive + Given Open Datafusion Project to configure pipeline + Then Click on the Plus Green Button to import the pipelines + Then Select the file for importing the pipeline for the plugin "Directive_parse_avro" + Then Navigate to the properties page of plugin: "BigQueryTable" + Then Replace input plugin property: "project" with value: "projectId" + Then Replace input plugin property: "dataset" with value: "dataset" + Then Replace input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Validate "BigQueryTable" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "BigQuery2" + Then Replace input plugin property: "project" with value: "projectId" + Then Replace input plugin property: "table" with value: "bqTargetTable" + Then Replace input plugin property: "dataset" with value: "dataset" + Then Validate "BigQuery2" plugin properties + Then Close the Plugin Properties page + Then Rename the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_avro" diff --git a/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsLog.feature b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsLog.feature new file mode 100644 index 000000000..b9e044ee8 --- /dev/null +++ b/wrangler-transform/src/e2e-test/features/Wrangler/ParseAsLog.feature @@ -0,0 +1,43 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@Wrangler +Feature: Wrangler - Run time scenarios for Parse Log + + @BQ_SOURCE_LOG_TEST @BQ_SOURCE_TEST @BQ_SINK_TEST + Scenario: To verify User is able to run a pipeline using parse log directive + Given Open Datafusion Project to configure pipeline + Then Click on the Plus Green Button to import the pipelines + Then Select the file for importing the pipeline for the plugin "Directive_parse_log" + Then Navigate to the properties page of plugin: "BigQueryTable" + Then Replace input plugin property: "project" with value: "projectId" + Then Replace input plugin property: "dataset" with value: "dataset" + Then Replace input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Validate "BigQueryTable" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "BigQuery2" + Then Replace input plugin property: "project" with value: "projectId" + Then Replace input plugin property: "table" with value: "bqTargetTable" + Then Replace input plugin property: "dataset" with value: "dataset" + Then Validate "BigQuery2" plugin properties + Then Close the Plugin Properties page + Then Rename the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate The Data From BQ To BQ With Actual And Expected File for: "ExpectedDirective_parse_log" diff --git a/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 16e4213ee..47b122a8c 100644 --- a/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/wrangler-transform/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -107,6 +107,16 @@ public static void deleteTempSourceBQTable() throws IOException, InterruptedExce BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully"); PluginPropertyUtils.removePluginProp("bqSourceTable"); } + @Before(order = 1, value = "@BQ_SOURCE_AVRO_TEST") + public static void createTempSourceBQTableAvro() throws IOException, InterruptedException { + createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileAvro"), + PluginPropertyUtils.pluginProp("InsertBQDataQueryFileAvro")); + } + @Before(order = 1, value = "@BQ_SOURCE_LOG_TEST") + public static void createTempSourceBQTableLog() throws IOException, InterruptedException { + createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("CreateBQTableQueryFileLog"), + PluginPropertyUtils.pluginProp("InsertBQDataQueryFileLog")); + } @Before(order = 1, value = "@BQ_SOURCE_JSON_TEST") diff --git a/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_avro b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_avro new file mode 100644 index 000000000..f7ad4b503 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_avro @@ -0,0 +1,3 @@ +{"id":1,"name":"ABC","nameNew":"BC","value":1} +{"id":2,"name":"DEF","nameNew":"EF","value":0} +{"id":4,"name":"XYZ","nameNew":"YZ","value":null} \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_log b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_log new file mode 100644 index 000000000..eedc04369 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQValidationExpectedFiles/Directive_parse_log @@ -0,0 +1 @@ +{"Body_icd_9_description":"Body Post Catalog lookup","body":"127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326","bytes_response_body_bytes":"2326","bytes_response_body_bytes_last":"2326","http_firstline_request_firstline":"GET /apache_pb.gif HTTP/1.0","http_method_request_firstline_method":"GET","http_path_request_firstline_uri_path":"/apache_pb.gif","http_protocol_request_firstline_protocol":"HTTP","http_protocol_version_request_firstline_protocol":"HTTP/1.0","http_protocol_version_request_firstline_protocol_version":"1.0","http_ref_request_firstline_uri_ref":"no value","http_uri_request_firstline_uri":"/apache_pb.gif","id":1,"ip_connection_client_host":"127.0.0.1","ip_connection_client_host_1":"127.0.0.1","ip_connection_client_host_last":"127.0.0.1","string_connection_client_user":"frank","string_connection_client_user_last":"frank","string_request_status_last":"200","time_date_request_receive_time_date":"2000-10-10","time_date_request_receive_time_date_utc":"2000-10-10","time_date_request_receive_time_last_date":"2000-10-10","time_date_request_receive_time_last_date_utc":"2000-10-10","time_day_request_receive_time_day":"10","time_day_request_receive_time_day_utc":"10","time_day_request_receive_time_last_day":10,"time_day_request_receive_time_last_day_utc":"10","time_epoch_request_receive_time_epoch":"971211336000","time_epoch_request_receive_time_last_epoch":"971211336000","time_hour_request_receive_time_hour":"13","time_hour_request_receive_time_hour_utc":"20","time_hour_request_receive_time_last_hour":"13","time_hour_request_receive_time_last_hour_utc":"20","time_millisecond_request_receive_time_last_millisecond":"0","time_millisecond_request_receive_time_last_millisecond_utc":"0","time_millisecond_request_receive_time_millisecond":"0","time_millisecond_request_receive_time_millisecond_utc":"0","time_minute_request_receive_time_last_minute":"55","time_minute_request_receive_time_last_minute_utc":"55","time_minute_request_receive_time_minute":"55","time_minute_request_receive_time_minute_utc":"55","time_month_request_receive_time_last_month":"10","time_month_request_receive_time_last_month_utc":"10","time_month_request_receive_time_month":"10","time_month_request_receive_time_month_utc":"10","time_monthname_request_receive_time_last_monthname":"October","time_monthname_request_receive_time_last_monthname_utc":"October","time_monthname_request_receive_time_monthname":"October","time_monthname_request_receive_time_monthname_utc":"October","time_second_request_receive_time_last_second":"36","time_second_request_receive_time_last_second_utc":"36","time_second_request_receive_time_second":"36","time_second_request_receive_time_second_utc":"36","time_stamp_request_receive_time":"10/Oct/2000:13:55:36 -0700","time_stamp_request_receive_time_last":"10/Oct/2000:13:55:36 -0700","time_time_request_receive_time_last_time":"13:55:36","time_time_request_receive_time_last_time_utc":"20:55:36","time_time_request_receive_time_time":"13:55:36","time_time_request_receive_time_time_utc":"20:55:36","time_week_request_receive_time_last_weekofweekyear":"41","time_week_request_receive_time_last_weekofweekyear_utc":"41","time_week_request_receive_time_weekofweekyear":"41","time_week_request_receive_time_weekofweekyear_utc":"41","time_year_request_receive_time_last_weekyear":"2000","time_year_request_receive_time_last_weekyear_utc":"2000","time_year_request_receive_time_last_year":"2000","time_year_request_receive_time_last_year_utc":"2000","time_year_request_receive_time_weekyear":"2000","time_year_request_receive_time_weekyear_utc":"2000","time_year_request_receive_time_year":"2000","time_year_request_receive_time_year_utc":"2000"} \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryAvro.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryAvro.txt new file mode 100644 index 000000000..1fc92efa6 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryAvro.txt @@ -0,0 +1 @@ +create table `DATASET.TABLE_NAME` (id INTEGER, body BYTES) \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryLog.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryLog.txt new file mode 100644 index 000000000..06b7075ec --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryCreateTableQueryLog.txt @@ -0,0 +1 @@ +create table `DATASET.TABLE_NAME` (id INTEGER, body STRING(100)) \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryAvro.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryAvro.txt new file mode 100644 index 000000000..c0004e366 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryAvro.txt @@ -0,0 +1,3 @@ +INSERT INTO DATASET.TABLE_NAME (body) +VALUES +(FROM_BASE64("T2JqAQQUYXZyby5jb2RlYwhudWxsFmF2cm8uc2NoZW1htgJ7InR5cGUiOiJyZWNvcmQiLCJuYW1lIjoiUmVjb3JkIiwiZmllbGRzIjpbeyJuYW1lIjoiaWQiLCJ0eXBlIjoibG9uZyJ9LHsibmFtZSI6Im5hbWUiLCJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0seyJuYW1lIjoidmFsdWUiLCJ0eXBlIjpbIm51bGwiLCJkb3VibGUiXX1dfQDPLTa6WtteXtkAuF/3puB+CGACAgZBQkMCWDm0yHa+8z8EAgZERUYCAAAAAAAAAAAGAAKuR+E6gskyQQgCBlhZWgDPLTa6WtteXtkAuF/3puB+")) \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryLog.txt b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryLog.txt new file mode 100644 index 000000000..c5d4137f3 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/BQtesdata/BigQuery/BigQueryInsertDataQueryLog.txt @@ -0,0 +1,3 @@ +INSERT INTO DATASET.TABLE_NAME (id,body) +VALUES +(1,'127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326'); diff --git a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties index a49e18d75..9396489e4 100644 --- a/wrangler-transform/src/e2e-test/resources/pluginParameters.properties +++ b/wrangler-transform/src/e2e-test/resources/pluginParameters.properties @@ -8,6 +8,8 @@ Directive_parse_xml=testData/Wrangler/parse_xmltojson_wrangler-cdap-data-pipelin Directive_parse_excel=testData/Wrangler/parse_excel_wrangler_copy-cdap-data-pipeline.json Directive_parse_csv=testData/Wrangler\ /parse_csv_wrangle-cdap-data-pipeline.json +Directive_parse_avro=testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json +Directive_parse_log=testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json Directive_GroupBy=testData/Wrangler/BQ2BQwithWrnglerNGrpby-cdap-data-pipeline (1).json bqSourceTable=dummy bqTargetTable=dummy @@ -30,6 +32,10 @@ CreateBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryCreateTableQueryXml.txt InsertBQDataQueryFileXml=BQtesdata/BigQuery/BigQueryInsertDataQueryXml.txt CreateBQTableQueryFileCsv=BQtesdata/BigQuery/BigQueryCreateTableQueryCsv.txt InsertBQDataQueryFileCsv=BQtesdata/BigQuery/BigQueryInsertDataQueryCsv.txt +CreateBQTableQueryFileAvro=BQtesdata/BigQuery/BigQueryCreateTableQueryAvro.txt +InsertBQDataQueryFileAvro=BQtesdata/BigQuery/BigQueryInsertDataQueryAvro.txt +CreateBQTableQueryFileLog=BQtesdata/BigQuery/BigQueryCreateTableQueryLog.txt +InsertBQDataQueryFileLog=BQtesdata/BigQuery/BigQueryInsertDataQueryLog.txt CreateBQTableQueryFile=BQtesdata/BigQuery/BigQueryCreateTableQuery.txt InsertBQDataQueryFile=BQtesdata/BigQuery/BigQueryInsertDataQuery.txt @@ -48,3 +54,5 @@ ExpectedDirective_parse_json=BQValidationExpectedFiles/Directive_parse_json ExpectedDirective_parse_xml=BQValidationExpectedFiles/Directive_parse_xmltojson ExpectedDirective_parse_excel=BQValidationExpectedFiles/Directive_parse_excel ExpectedDirective_parse_csv=BQValidationExpectedFiles/Directive_parse_csv +ExpectedDirective_parse_avro=BQValidationExpectedFiles/Directive_parse_avro +ExpectedDirective_parse_log=BQValidationExpectedFiles/Directive_parse_log diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json new file mode 100644 index 000000000..60aedd429 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parseAsAvro-cdap-data-pipeline (1).json @@ -0,0 +1,704 @@ +{ + "name": "parseAsAvro", + "description": "Data Pipeline Application", + "artifact": { + "name": "cdap-data-pipeline", + "version": "[6.0.0, 7.0.0)", + "scope": "SYSTEM" + }, + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "BigQueryTable", + "to": "Wrangler" + }, + { + "from": "Wrangler", + "to": "BigQuery2" + } + ], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": true, + "stages": [ + { + "name": "BigQueryTable", + "plugin": { + "name": "BigQueryTable", + "type": "batchsource", + "label": "BigQueryTable", + "artifact": { + "name": "google-cloud" + }, + "properties": { + "useConnection": "false", + "dataset": "wrangler_ankit", + "table": "avroTable", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"bytes\",\"null\"]}]}", + "enableQueryingViews": "false", + "referenceName": "ref", + "project": "auto-detect", + "serviceAccountType": "filePath", + "serviceFilePath": "auto-detect" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"bytes\",\"null\"]}]}" + } + ], + "id": "BigQueryTable", + "type": "batchsource", + "label": "BigQueryTable", + "icon": "fa-plug", + "$$hashKey": "object:31", + "isPluginAvailable": true, + "_uiPosition": { + "left": "496px", + "top": "343px" + }, + "_backendProperties": { + "schema": { + "name": "schema", + "description": "The schema of the table to read.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "viewMaterializationDataset": { + "name": "viewMaterializationDataset", + "description": "The dataset in the specified project where the temporary table should be created. Defaults to the same dataset in which the table is located.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "enableQueryingViews": { + "name": "enableQueryingViews", + "description": "Whether to allow querying views. Since BigQuery views are not materialized by default, querying them may have a performance overhead.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceAccountJSON": { + "name": "serviceAccountJSON", + "description": "Content of the service account file.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitionTo": { + "name": "partitionTo", + "description": "It's inclusive partition end date. It should be a String with format \"yyyy-MM-dd\". This value is ignored if the table does not support partitioning.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "cmekKey": { + "name": "cmekKey", + "description": "The GCP customer managed encryption key (CMEK) name used to encrypt data written to any bucket, dataset or table created by the plugin. If the bucket, dataset or table already exists, this is ignored. More information can be found at https://cloud.google.com/data-fusion/docs/how-to/customer-managed-encryption-keys", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "useConnection": { + "name": "useConnection", + "description": "Whether to use an existing connection.", + "type": "boolean", + "required": false, + "macroSupported": false, + "macroEscapingEnabled": false, + "children": [] + }, + "project": { + "name": "project", + "description": "Google Cloud Project ID. It can be found on the Dashboard in the Google Cloud Platform Console.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "viewMaterializationProject": { + "name": "viewMaterializationProject", + "description": "The project name where the temporary table should be created. Defaults to the same project in which the table is located.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "filter": { + "name": "filter", + "description": "The WHERE clause filters out rows by evaluating each row against boolean expression, and discards all rows that do not return TRUE (that is, rows that return FALSE or NULL).", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "bucket": { + "name": "bucket", + "description": "The Google Cloud Storage bucket to store temporary data in. Cloud Storage data will be deleted after it is loaded into BigQuery. If it is not provided, a unique bucket will be automatically created and then deleted after the run finishes. The service account must have permission to create buckets in the configured project.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitionFrom": { + "name": "partitionFrom", + "description": "It's inclusive partition start date. It should be a String with format \"yyyy-MM-dd\". This value is ignored if the table does not support partitioning.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceFilePath": { + "name": "serviceFilePath", + "description": "Path on the local file system of the service account key used for authorization. Can be set to 'auto-detect' when running on a Dataproc cluster. When running on other clusters, the file must be present on every node in the cluster.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceAccountType": { + "name": "serviceAccountType", + "description": "Service account type, file path where the service account is located or the JSON content of the service account.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "connection": { + "name": "connection", + "description": "The existing connection to use.", + "type": "bigqueryconnectorconfig", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [ + "serviceAccountJSON", + "serviceFilePath", + "project", + "serviceAccountType", + "datasetProject" + ] + }, + "datasetProject": { + "name": "datasetProject", + "description": "The project the dataset belongs to. This is only required if the dataset is not in the same project that the BigQuery job will run in. If no value is given, it will default to the configured project ID.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "dataset": { + "name": "dataset", + "description": "The dataset to write to. A dataset is contained within a specific project. Datasets are top-level containers that are used to organize and control access to tables and views.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "table": { + "name": "table", + "description": "The table to read from. A table contains individual records organized in rows. Each record is composed of columns (also called fields). Every table is defined by a schema that describes the column names, data types, and other information.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "referenceName": { + "name": "referenceName", + "description": "This will be used to uniquely identify this source for lineage, annotating metadata, etc.", + "type": "string", + "required": false, + "macroSupported": false, + "macroEscapingEnabled": false, + "children": [] + } + }, + "description": "This source reads the entire contents of a BigQuery table. BigQuery is Google's serverless, highly scalable, enterprise data warehouse.Data is first written to a temporary location on Google Cloud Storage, then read into the pipeline from there.", + "selected": false + }, + { + "name": "Wrangler", + "plugin": { + "name": "Wrangler", + "type": "transform", + "label": "Wrangler", + "artifact": { + "name": "wrangler-transform" + }, + "properties": { + "directives": "parse-as-avro-file :body\nfilter-rows-on condition-true name == null || name =~ \"^\\W*$\"\ncut-character name nameNew 2-3\nset-type :value integer ", + "field": "*", + "precondition": "false", + "workspaceId": "ceba48f3-ed03-411a-8d71-c92a39b7c250", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"value\",\"type\":[\"int\",\"null\"]},{\"name\":\"nameNew\",\"type\":[\"string\",\"null\"]}]}", + "on-error": "fail-pipeline" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"value\",\"type\":[\"int\",\"null\"]},{\"name\":\"nameNew\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "inputSchema": [ + { + "name": "BigQueryTable", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"bytes\",\"null\"]}]}" + } + ], + "id": "Wrangler", + "type": "transform", + "label": "Wrangler", + "icon": "icon-DataPreparation", + "$$hashKey": "object:32", + "isPluginAvailable": true, + "_uiPosition": { + "left": "796px", + "top": "343px" + }, + "selected": false, + "_backendProperties": { + "schema": { + "name": "schema", + "description": "Specifies the schema that has to be output.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "preconditionSQL": { + "name": "preconditionSQL", + "description": "SQL Precondition expression specifying filtering before applying directives (false to filter)", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "udd": { + "name": "udd", + "description": "List of User Defined Directives (UDD) that have to be loaded.", + "type": "string", + "required": false, + "macroSupported": false, + "macroEscapingEnabled": false, + "children": [] + }, + "field": { + "name": "field", + "description": "Name of the input field to be wrangled or '*' to wrangle all the fields.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "on-error": { + "name": "on-error", + "description": "How to handle error in record processing", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "directives": { + "name": "directives", + "description": "Recipe for wrangling the input records", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "expressionLanguage": { + "name": "expressionLanguage", + "description": "Toggle to configure precondition language between JEXL and SQL", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "precondition": { + "name": "precondition", + "description": "JEXL Precondition expression specifying filtering before applying directives (true to filter)", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + } + }, + "description": "Wrangler - A interactive tool for data cleansing and transformation." + }, + { + "name": "BigQuery2", + "plugin": { + "name": "BigQueryTable", + "type": "batchsink", + "label": "BigQuery2", + "artifact": { + "name": "google-cloud" + }, + "properties": { + "useConnection": "false", + "referenceName": "ref", + "dataset": "wrangler_ankit", + "table": "avroResult", + "operation": "insert", + "truncateTable": "false", + "allowSchemaRelaxation": "false", + "location": "US", + "createPartitionedTable": "false", + "partitioningType": "TIME", + "partitionFilterRequired": "false", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"value\",\"type\":[\"int\",\"null\"]},{\"name\":\"nameNew\",\"type\":[\"string\",\"null\"]}]}", + "project": "auto-detect", + "serviceAccountType": "filePath", + "serviceFilePath": "auto-detect" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"value\",\"type\":[\"int\",\"null\"]},{\"name\":\"nameNew\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "inputSchema": [ + { + "name": "Wrangler", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"name\",\"type\":[\"string\",\"null\"]},{\"name\":\"value\",\"type\":[\"int\",\"null\"]},{\"name\":\"nameNew\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "id": "BigQuery2", + "type": "batchsink", + "label": "BigQuery2", + "icon": "fa-plug", + "$$hashKey": "object:33", + "isPluginAvailable": true, + "_uiPosition": { + "left": "1096px", + "top": "343px" + }, + "selected": false, + "_backendProperties": { + "schema": { + "name": "schema", + "description": "The schema of the data to write. If provided, must be compatible with the table schema.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitionFilter": { + "name": "partitionFilter", + "description": "Partition filter that can be used for partition elimination during Update or Upsert operations.This value is ignored if operation is not UPDATE or UPSERT.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "rangeStart": { + "name": "rangeStart", + "description": "Start value for range partitioning. The start value is inclusive. Ignored when table already exists", + "type": "long", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceAccountJSON": { + "name": "serviceAccountJSON", + "description": "Content of the service account file.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "useConnection": { + "name": "useConnection", + "description": "Whether to use an existing connection.", + "type": "boolean", + "required": false, + "macroSupported": false, + "macroEscapingEnabled": false, + "children": [] + }, + "project": { + "name": "project", + "description": "Google Cloud Project ID. It can be found on the Dashboard in the Google Cloud Platform Console.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitioningType": { + "name": "partitioningType", + "description": "Specifies the partitioning type. Can either be Integer or Time or None. Ignored when table already exists", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "relationTableKey": { + "name": "relationTableKey", + "description": "List of fields that determines relation between tables during Update and Upsert operations.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "rangeEnd": { + "name": "rangeEnd", + "description": "End value for range partitioning. The end value is exclusive. Ignored when table already exists", + "type": "long", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "clusteringOrder": { + "name": "clusteringOrder", + "description": "List of fields that determines the sort order of the data. Fields must be of type INT, LONG, STRING, DATE, TIMESTAMP, BOOLEAN or DECIMAL. Tables cannot be clustered on more than 4 fields. This value is only used when the BigQuery table is automatically created and ignored if the table already exists.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitionFilterRequired": { + "name": "partitionFilterRequired", + "description": "Whether to create a table that requires a partition filter. This value is ignored if the table already exists.", + "type": "boolean", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceFilePath": { + "name": "serviceFilePath", + "description": "Path on the local file system of the service account key used for authorization. Can be set to 'auto-detect' when running on a Dataproc cluster. When running on other clusters, the file must be present on every node in the cluster.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "truncateTable": { + "name": "truncateTable", + "description": "Whether or not to truncate the table before writing to it. Should only be used with the Insert operation. This could overwrite the table schema", + "type": "boolean", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "partitionByField": { + "name": "partitionByField", + "description": "Partitioning column for the BigQuery table. This should be left empty if the BigQuery table is an ingestion-time partitioned table.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "connection": { + "name": "connection", + "description": "The existing connection to use.", + "type": "bigqueryconnectorconfig", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [ + "serviceAccountJSON", + "serviceFilePath", + "project", + "serviceAccountType", + "datasetProject" + ] + }, + "table": { + "name": "table", + "description": "The table to write to. A table contains individual records organized in rows. Each record is composed of columns (also called fields). Every table is defined by a schema that describes the column names, data types, and other information.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "referenceName": { + "name": "referenceName", + "description": "This will be used to uniquely identify this source/sink for lineage, annotating metadata, etc.", + "type": "string", + "required": false, + "macroSupported": false, + "macroEscapingEnabled": false, + "children": [] + }, + "cmekKey": { + "name": "cmekKey", + "description": "The GCP customer managed encryption key (CMEK) name used to encrypt data written to any bucket, dataset or table created by the plugin. If the bucket, dataset or table already exists, this is ignored. More information can be found at https://cloud.google.com/data-fusion/docs/how-to/customer-managed-encryption-keys", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "createPartitionedTable": { + "name": "createPartitionedTable", + "description": "DEPRECATED!. Whether to create the BigQuery table with time partitioning. This value is ignored if the table already exists. When this is set to false, value of Partitioning type will be used. Use 'Partitioning type' property", + "type": "boolean", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "dedupeBy": { + "name": "dedupeBy", + "description": "Column names and sort order used to choose which input record to update/upsert when there are multiple input records with the same key. For example, if this is set to 'updated_time desc', then if there are multiple input records with the same key, the one with the largest value for 'updated_time' will be applied.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "allowSchemaRelaxation": { + "name": "allowSchemaRelaxation", + "description": "Whether to modify the BigQuery table schema if it differs from the input schema.", + "type": "boolean", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "bucket": { + "name": "bucket", + "description": "The Google Cloud Storage bucket to store temporary data in. Cloud Storage data will be deleted after it is loaded into BigQuery. If it is not provided, a unique bucket will be automatically created and then deleted after the run finishes. The service account must have permission to create buckets in the configured project.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "rangeInterval": { + "name": "rangeInterval", + "description": "Interval value for range partitioning. The interval value must be a positive integer.Ignored when table already exists", + "type": "long", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "gcsChunkSize": { + "name": "gcsChunkSize", + "description": "Optional property to tune chunk size in gcs upload request. The value of this property should be in number of bytes. By default, 8388608 bytes (8MB) will be used as upload request chunk size.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "location": { + "name": "location", + "description": "The location where the big query dataset will get created. This value is ignored if the dataset or temporary bucket already exist.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "serviceAccountType": { + "name": "serviceAccountType", + "description": "Service account type, file path where the service account is located or the JSON content of the service account.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "datasetProject": { + "name": "datasetProject", + "description": "The project the dataset belongs to. This is only required if the dataset is not in the same project that the BigQuery job will run in. If no value is given, it will default to the configured project ID.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "operation": { + "name": "operation", + "description": "Type of write operation to perform. This can be set to Insert, Update or Upsert.", + "type": "string", + "required": false, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + }, + "dataset": { + "name": "dataset", + "description": "The dataset to write to. A dataset is contained within a specific project. Datasets are top-level containers that are used to organize and control access to tables and views.", + "type": "string", + "required": true, + "macroSupported": true, + "macroEscapingEnabled": false, + "children": [] + } + }, + "description": "This sink writes to a BigQuery table. BigQuery is Google's serverless, highly scalable, enterprise data warehouse. Data is first written to a temporary location on Google Cloud Storage, then loaded into BigQuery from there." + } + ], + "schedule": "0 1 */1 * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "rangeRecordsPreview": { + "min": 1, + "max": "5000" + }, + "description": "Data Pipeline Application", + "maxConcurrentRuns": 1, + "pushdownEnabled": false, + "transformationPushdown": {} + }, + "version": "5aeae257-5b78-11ee-9c85-00000078300d" +} \ No newline at end of file diff --git a/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json new file mode 100644 index 000000000..9c98f5c45 --- /dev/null +++ b/wrangler-transform/src/e2e-test/resources/testData/Wrangler/parse_log_wrangler_copy-cdap-data-pipeline.json @@ -0,0 +1,168 @@ +{ + "name": "parse_log_wrangler_copy", + "description": "Data Pipeline Application", + "artifact": { + "name": "cdap-data-pipeline", + "version": "[6.0.0, 7.0.0)", + "scope": "SYSTEM" + }, + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "BigQueryTable", + "to": "Wrangler" + }, + { + "from": "Wrangler", + "to": "BigQuery2" + } + ], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": true, + "stages": [ + { + "name": "BigQueryTable", + "plugin": { + "name": "BigQueryTable", + "type": "batchsource", + "label": "BigQueryTable", + "artifact": { + "name": "google-cloud" + }, + "properties": { + "useConnection": "false", + "dataset": "Wrangler_Test", + "table": "logTab", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]}", + "project": "auto-detect", + "serviceAccountType": "filePath", + "serviceFilePath": "auto-detect", + "enableQueryingViews": "false" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"output\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "id": "BigQueryTable", + "type": "batchsource", + "label": "BigQueryTable", + "icon": "fa-plug", + "$$hashKey": "object:329", + "isPluginAvailable": true, + "_uiPosition": { + "left": "496px", + "top": "342px" + } + }, + { + "name": "Wrangler", + "plugin": { + "name": "Wrangler", + "type": "transform", + "label": "Wrangler", + "artifact": { + "name": "wrangler-transform" + }, + "properties": { + "directives": "parse-as-log :Body 'common'\ndrop :number_connection_client_logname_last\ndrop :number_connection_client_logname\ndrop :http_querystring_request_firstline_uri_query\nfill-null-or-empty :http_ref_request_firstline_uri_ref 'no value'\nfilter-rows-on condition-false ip_connection_client_host =$ \"1\"\nfilter-rows-on regex-match string_connection_client_user_last ^Tryck$\nextract-regex-groups :http_firstline_request_firstline GET*\nsplit-to-columns :ip_connection_client_host ,\ncatalog-lookup ICD-9 :Body\nfill-null-or-empty :Body_icd_9_description 'Body Post Catalog lookup'\nset-variable Pass string_request_status_last == \"200\" ? string_request_status_last : Pass\nset-type :time_day_request_receive_time_last_day integer ", + "field": "*", + "precondition": "false", + "workspaceId": "c42a0f1b-26e2-4957-80ce-cd0f1a03f0d5", + "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_firstline_request_firstline\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_request_status_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_method_request_firstline_method\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_uri_request_firstline_uri\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day\",\"type\":[\"int\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_last_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_path_request_firstline_uri_path\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_ref_request_firstline_uri_ref\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol_version\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_1\",\"type\":[\"string\",\"null\"]},{\"name\":\"Body_icd_9_description\",\"type\":[\"string\",\"null\"]}]}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_firstline_request_firstline\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_request_status_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_method_request_firstline_method\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_uri_request_firstline_uri\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day\",\"type\":[\"int\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_last_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_path_request_firstline_uri_path\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_ref_request_firstline_uri_ref\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol_version\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_1\",\"type\":[\"string\",\"null\"]},{\"name\":\"Body_icd_9_description\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "id": "Wrangler", + "type": "transform", + "label": "Wrangler", + "icon": "icon-DataPreparation", + "$$hashKey": "object:330", + "isPluginAvailable": true, + "_uiPosition": { + "left": "796px", + "top": "342px" + } + }, + { + "name": "BigQuery2", + "plugin": { + "name": "BigQueryTable", + "type": "batchsink", + "label": "BigQuery2", + "artifact": { + "name": "google-cloud" + }, + "properties": { + "useConnection": "false", + "project": "auto-detect", + "serviceAccountType": "filePath", + "serviceFilePath": "auto-detect", + "dataset": "Wrangler", + "table": "logfintabs", + "operation": "insert", + "truncateTable": "false", + "allowSchemaRelaxation": "false", + "location": "US", + "createPartitionedTable": "false", + "partitioningType": "TIME", + "timePartitioningType": "DAY", + "partitionFilterRequired": "false", + "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_firstline_request_firstline\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_request_status_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_method_request_firstline_method\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_uri_request_firstline_uri\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day\",\"type\":[\"int\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_last_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_path_request_firstline_uri_path\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_ref_request_firstline_uri_ref\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol_version\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_1\",\"type\":[\"string\",\"null\"]},{\"name\":\"Body_icd_9_description\",\"type\":[\"string\",\"null\"]}]}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_firstline_request_firstline\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_request_status_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_method_request_firstline_method\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_uri_request_firstline_uri\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day\",\"type\":[\"int\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_last_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_path_request_firstline_uri_path\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_ref_request_firstline_uri_ref\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol_version\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_1\",\"type\":[\"string\",\"null\"]},{\"name\":\"Body_icd_9_description\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "inputSchema": [ + { + "name": "Wrangler", + "schema": "{\"type\":\"record\",\"name\":\"outputSchema\",\"fields\":[{\"name\":\"id\",\"type\":[\"long\",\"null\"]},{\"name\":\"body\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_connection_client_user_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_stamp_request_receive_time_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_firstline_request_firstline\",\"type\":[\"string\",\"null\"]},{\"name\":\"string_request_status_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_method_request_firstline_method\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_uri_request_firstline_uri\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"bytes_response_body_bytes_last\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day\",\"type\":[\"int\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_epoch_request_receive_time_last_epoch\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_day_request_receive_time_last_day_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_monthname_request_receive_time_last_monthname_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_month_request_receive_time_last_month_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_week_request_receive_time_last_weekofweekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_weekyear_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_year_request_receive_time_last_year_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_hour_request_receive_time_last_hour_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_minute_request_receive_time_last_minute_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_second_request_receive_time_last_second_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_millisecond_request_receive_time_last_millisecond_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_date_request_receive_time_last_date_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"time_time_request_receive_time_last_time_utc\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_path_request_firstline_uri_path\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_ref_request_firstline_uri_ref\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_request_firstline_protocol\",\"type\":[\"string\",\"null\"]},{\"name\":\"http_protocol_version_request_firstline_protocol_version\",\"type\":[\"string\",\"null\"]},{\"name\":\"ip_connection_client_host_1\",\"type\":[\"string\",\"null\"]},{\"name\":\"Body_icd_9_description\",\"type\":[\"string\",\"null\"]}]}" + } + ], + "id": "BigQuery2", + "type": "batchsink", + "label": "BigQuery2", + "icon": "fa-plug", + "$$hashKey": "object:331", + "isPluginAvailable": true, + "_uiPosition": { + "left": "1096px", + "top": "342px" + } + } + ], + "schedule": "0 1 */1 * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "rangeRecordsPreview": { + "min": 1, + "max": "5000" + }, + "description": "Data Pipeline Application", + "maxConcurrentRuns": 1, + "pushdownEnabled": false, + "transformationPushdown": {} + }, + "version": "d2fd2cff-afdb-11ee-84cf-000000df2f26" +} \ No newline at end of file