From ad65fdd9530f6fee04e8326513f15c9263d7fe06 Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Fri, 3 May 2024 18:35:10 +0530 Subject: [PATCH 1/9] Modified parameters for Stream_GCS_Text_to_BigQuery_Flex template --- .../transforms/JavascriptTextTransformer.java | 9 ++++---- .../v2/templates/TextIOToBigQuery.java | 23 ++++++++++--------- .../v2/templates/TextToBigQueryStreaming.java | 7 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java index 3e47ae19b5..45c8008898 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java @@ -78,9 +78,9 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to Javascript UDF source", helpText = - "The Cloud Storage path pattern for the JavaScript code containing your user-defined " - + "functions.", - example = "gs://your-bucket/your-function.js") + "\tThe Cloud Storage URI of the .js file that defines the JavaScript user-defined function (UDF) " + + "you want to use.", + example = "gs://my-bucket/my-udfs/my_file.js") String getJavascriptTextTransformGcsPath(); void setJavascriptTextTransformGcsPath(String javascriptTextTransformGcsPath); @@ -102,7 +102,8 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "JavaScript UDF auto-reload interval (minutes)", helpText = - "Define the interval that workers may check for JavaScript UDF changes to reload the files.") + "Define the interval that workers may check for JavaScript UDF changes to reload the files. Defaults " + + "to: 0.") @Default.Integer(0) Integer getJavascriptTextTransformReloadIntervalMinutes(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index aa1bb7a814..4e7b69c78f 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -215,8 +215,8 @@ public interface Options optional = false, regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "The GCS location of the text you'd like to process", - helpText = "The path to the Cloud Storage text to read.", - example = "gs://your-bucket/your-file.txt") + helpText = "Cloud Storage location of the text you'd like to process.", + example = "gs://my-bucket/my-files/text.txt") String getInputFilePattern(); void setInputFilePattern(String value); @@ -225,8 +225,8 @@ public interface Options order = 2, optional = false, description = "JSON file with BigQuery Schema description", - helpText = "The Cloud Storage path to the JSON file that defines your BigQuery schema.", - example = "gs://your-bucket/your-schema.json") + helpText = "Cloud Storage location of your BigQuery schema file, described as a JSON.", + example = "gs://path/to/my/schema.json") String getJSONPath(); void setJSONPath(String value); @@ -237,9 +237,8 @@ public interface Options regexes = {".+:.+\\..+"}, description = "Output table to write to", helpText = - "The location of the BigQuery table in which to store your processed data. If you reuse" - + " an existing table, it will be overwritten.", - example = "your-project:your-dataset.your-table") + "The fully qualified BigQuery table.", + example = "my-project:dataset.table") String getOutputTable(); void setOutputTable(String value); @@ -263,8 +262,10 @@ public interface Options regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the function to call from your JavaScript file. Use only letters, digits," - + " and underscores.", + "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if your " + + "JavaScript function code is myTransform(inJson) { /*...do stuff...*/ }, then the function name " + + "is myTransform. For sample JavaScript UDFs, see UDF Examples " + + "(https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); @@ -275,8 +276,8 @@ public interface Options order = 6, optional = false, description = "Temporary directory for BigQuery loading process", - helpText = "Temporary directory for the BigQuery loading process.", - example = "gs://your-bucket/your-files/temp-dir") + helpText = "Temporary directory for BigQuery loading process.", + example = "gs://my-bucket/my-files/temp_dir") String getBigQueryLoadingTemporaryDirectory(); void setBigQueryLoadingTemporaryDirectory(String directory); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index 63192c4e92..fb90519d41 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -479,10 +479,9 @@ public interface TextToBigQueryStreamingOptions optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "BigQuery table for failed messages. Messages failed to reach the output table for different reasons " - + "(e.g., mismatched schema, malformed json) are written to this table. If it doesn't exist, it will" - + " be created during pipeline execution. If not specified, \"outputTableSpec_error_records\" is used instead.", - example = "your-project-id:your-dataset.your-table-name") + "Table for messages that failed to reach the output table. For example: " + + "my-project:dataset.my-unprocessed-table. If it doesn't exist, it is created during pipeline " + + "execution. If not specified, _error_records is used instead.") String getOutputDeadletterTable(); void setOutputDeadletterTable(String value); From c405ff5b6a18574db9613e82c07af5c44372cd2e Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Fri, 3 May 2024 19:03:12 +0530 Subject: [PATCH 2/9] Revert "Modified parameters for Stream_GCS_Text_to_BigQuery_Flex template" This reverts commit ad65fdd9530f6fee04e8326513f15c9263d7fe06. --- .../transforms/JavascriptTextTransformer.java | 9 ++++---- .../v2/templates/TextIOToBigQuery.java | 23 +++++++++---------- .../v2/templates/TextToBigQueryStreaming.java | 7 +++--- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java index 45c8008898..3e47ae19b5 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java @@ -78,9 +78,9 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to Javascript UDF source", helpText = - "\tThe Cloud Storage URI of the .js file that defines the JavaScript user-defined function (UDF) " - + "you want to use.", - example = "gs://my-bucket/my-udfs/my_file.js") + "The Cloud Storage path pattern for the JavaScript code containing your user-defined " + + "functions.", + example = "gs://your-bucket/your-function.js") String getJavascriptTextTransformGcsPath(); void setJavascriptTextTransformGcsPath(String javascriptTextTransformGcsPath); @@ -102,8 +102,7 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "JavaScript UDF auto-reload interval (minutes)", helpText = - "Define the interval that workers may check for JavaScript UDF changes to reload the files. Defaults " + - "to: 0.") + "Define the interval that workers may check for JavaScript UDF changes to reload the files.") @Default.Integer(0) Integer getJavascriptTextTransformReloadIntervalMinutes(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index 4e7b69c78f..aa1bb7a814 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -215,8 +215,8 @@ public interface Options optional = false, regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "The GCS location of the text you'd like to process", - helpText = "Cloud Storage location of the text you'd like to process.", - example = "gs://my-bucket/my-files/text.txt") + helpText = "The path to the Cloud Storage text to read.", + example = "gs://your-bucket/your-file.txt") String getInputFilePattern(); void setInputFilePattern(String value); @@ -225,8 +225,8 @@ public interface Options order = 2, optional = false, description = "JSON file with BigQuery Schema description", - helpText = "Cloud Storage location of your BigQuery schema file, described as a JSON.", - example = "gs://path/to/my/schema.json") + helpText = "The Cloud Storage path to the JSON file that defines your BigQuery schema.", + example = "gs://your-bucket/your-schema.json") String getJSONPath(); void setJSONPath(String value); @@ -237,8 +237,9 @@ public interface Options regexes = {".+:.+\\..+"}, description = "Output table to write to", helpText = - "The fully qualified BigQuery table.", - example = "my-project:dataset.table") + "The location of the BigQuery table in which to store your processed data. If you reuse" + + " an existing table, it will be overwritten.", + example = "your-project:your-dataset.your-table") String getOutputTable(); void setOutputTable(String value); @@ -262,10 +263,8 @@ public interface Options regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if your " + - "JavaScript function code is myTransform(inJson) { /*...do stuff...*/ }, then the function name " + - "is myTransform. For sample JavaScript UDFs, see UDF Examples " + - "(https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", + "The name of the function to call from your JavaScript file. Use only letters, digits," + + " and underscores.", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); @@ -276,8 +275,8 @@ public interface Options order = 6, optional = false, description = "Temporary directory for BigQuery loading process", - helpText = "Temporary directory for BigQuery loading process.", - example = "gs://my-bucket/my-files/temp_dir") + helpText = "Temporary directory for the BigQuery loading process.", + example = "gs://your-bucket/your-files/temp-dir") String getBigQueryLoadingTemporaryDirectory(); void setBigQueryLoadingTemporaryDirectory(String directory); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index fb90519d41..63192c4e92 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -479,9 +479,10 @@ public interface TextToBigQueryStreamingOptions optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "Table for messages that failed to reach the output table. For example: " + - "my-project:dataset.my-unprocessed-table. If it doesn't exist, it is created during pipeline " + - "execution. If not specified, _error_records is used instead.") + "BigQuery table for failed messages. Messages failed to reach the output table for different reasons " + + "(e.g., mismatched schema, malformed json) are written to this table. If it doesn't exist, it will" + + " be created during pipeline execution. If not specified, \"outputTableSpec_error_records\" is used instead.", + example = "your-project-id:your-dataset.your-table-name") String getOutputDeadletterTable(); void setOutputDeadletterTable(String value); From 48cc7ed1521c16c44dd1fcd0473793b97f10f9b3 Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Fri, 3 May 2024 21:45:00 +0530 Subject: [PATCH 3/9] Modified parameters for Stream_GCS_Text_to_BigQuery_Flex and Cloud_PubSub_to_MongoDB templates --- .../v2/templates/TextIOToBigQuery.java | 19 ++++++++++--------- .../v2/templates/PubSubToMongoDB.java | 9 +++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index aa1bb7a814..c5f588997c 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -215,7 +215,7 @@ public interface Options optional = false, regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "The GCS location of the text you'd like to process", - helpText = "The path to the Cloud Storage text to read.", + helpText = "Cloud Storage location of the text you'd like to process.", example = "gs://your-bucket/your-file.txt") String getInputFilePattern(); @@ -225,7 +225,7 @@ public interface Options order = 2, optional = false, description = "JSON file with BigQuery Schema description", - helpText = "The Cloud Storage path to the JSON file that defines your BigQuery schema.", + helpText = "Cloud Storage location of your BigQuery schema file, described as a JSON.", example = "gs://your-bucket/your-schema.json") String getJSONPath(); @@ -237,8 +237,7 @@ public interface Options regexes = {".+:.+\\..+"}, description = "Output table to write to", helpText = - "The location of the BigQuery table in which to store your processed data. If you reuse" - + " an existing table, it will be overwritten.", + "The fully qualified BigQuery table.", example = "your-project:your-dataset.your-table") String getOutputTable(); @@ -250,8 +249,8 @@ public interface Options regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "GCS path to javascript fn for transforming output", helpText = - "The Cloud Storage path pattern for the JavaScript code containing your user-defined" - + " functions.", + "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) you want " + + "to use.", example = "gs://your-bucket/your-transforms/*.js") String getJavascriptTextTransformGcsPath(); @@ -263,8 +262,10 @@ public interface Options regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the function to call from your JavaScript file. Use only letters, digits," - + " and underscores.", + "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if " + + "your JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then " + + "the function name is `myTransform`. For sample JavaScript UDFs, " + + "see UDF Examples(https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); @@ -275,7 +276,7 @@ public interface Options order = 6, optional = false, description = "Temporary directory for BigQuery loading process", - helpText = "Temporary directory for the BigQuery loading process.", + helpText = "Temporary directory for BigQuery loading process.", example = "gs://your-bucket/your-files/temp-dir") String getBigQueryLoadingTemporaryDirectory(); diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index 8b4e3e0561..e365000e4d 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -136,8 +136,7 @@ public interface Options order = 1, description = "Pub/Sub input subscription", helpText = - "Pub/Sub subscription to read the input from, in the format of" - + " 'projects/your-project-id/subscriptions/your-subscription-name'", + "Name of the Pub/Sub subscription.", example = "projects/your-project-id/subscriptions/your-subscription-name") @Validation.Required String getInputSubscription(); @@ -167,7 +166,7 @@ public interface Options @TemplateParameter.Text( order = 4, description = "MongoDB collection", - helpText = "Name of the collection inside MongoDB database to insert the documents.", + helpText = "Name of the collection inside MongoDB database.", example = "my-collection") @Validation.Required String getCollection(); @@ -178,9 +177,7 @@ public interface Options order = 5, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "BigQuery table for failed messages. Messages failed to reach the output table for different reasons " - + "(e.g., mismatched schema, malformed json) are written to this table. If it doesn't exist, it will" - + " be created during pipeline execution. If not specified, \"outputTableSpec_error_records\" is used instead.", + "BigQuery table that store messages due to failures (mismatched schema, malformed JSON etc).", example = "your-project-id:your-dataset.your-table-name") @Validation.Required String getDeadletterTable(); From 1f7133a96794c43fa18797a475fbc14ed2dbb41e Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Mon, 6 May 2024 14:44:08 +0530 Subject: [PATCH 4/9] Modified parameters for Stream_GCS_Text_to_BigQuery_Flex_template --- .../v2/transforms/JavascriptTextTransformer.java | 14 ++++++++++---- .../teleport/v2/templates/TextIOToBigQuery.java | 4 ++-- .../v2/templates/TextToBigQueryStreaming.java | 7 +++---- .../teleport/v2/templates/PubSubToMongoDB.java | 14 +++++++------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java index 3e47ae19b5..c39b95e7b5 100644 --- a/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java +++ b/v2/common/src/main/java/com/google/cloud/teleport/v2/transforms/JavascriptTextTransformer.java @@ -78,8 +78,8 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "Cloud Storage path to Javascript UDF source", helpText = - "The Cloud Storage path pattern for the JavaScript code containing your user-defined " - + "functions.", + "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) you want " + + "to use.", example = "gs://your-bucket/your-function.js") String getJavascriptTextTransformGcsPath(); @@ -91,7 +91,10 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the function to call from your JavaScript file. Use only letters, digits, and underscores.", + "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if your " + + "JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then the function " + + "name is `myTransform`. For sample JavaScript UDFs, see UDF " + + "Examples https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples", example = "'transform' or 'transform_udf1'") String getJavascriptTextTransformFunctionName(); @@ -102,7 +105,10 @@ public interface JavascriptTextTransformerOptions extends PipelineOptions { optional = true, description = "JavaScript UDF auto-reload interval (minutes)", helpText = - "Define the interval that workers may check for JavaScript UDF changes to reload the files.") + "Specifies how frequently to reload the UDF, in minutes. If the value is greater than 0, Dataflow " + + "periodically checks the UDF file in Cloud Storage, and reloads the UDF if the file is modified. " + + "This parameter allows you to update the UDF while the pipeline is running, without needing to " + + "restart the job. If the value is 0, UDF reloading is disabled. The default value is 0") @Default.Integer(0) Integer getJavascriptTextTransformReloadIntervalMinutes(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index c5f588997c..05e9f602b8 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -238,7 +238,7 @@ public interface Options description = "Output table to write to", helpText = "The fully qualified BigQuery table.", - example = "your-project:your-dataset.your-table") + example = ":.") String getOutputTable(); void setOutputTable(String value); @@ -265,7 +265,7 @@ public interface Options "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if " + "your JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then " + "the function name is `myTransform`. For sample JavaScript UDFs, " + - "see UDF Examples(https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", + "see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index 63192c4e92..56103e3c81 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -479,10 +479,9 @@ public interface TextToBigQueryStreamingOptions optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "BigQuery table for failed messages. Messages failed to reach the output table for different reasons " - + "(e.g., mismatched schema, malformed json) are written to this table. If it doesn't exist, it will" - + " be created during pipeline execution. If not specified, \"outputTableSpec_error_records\" is used instead.", - example = "your-project-id:your-dataset.your-table-name") + "Table for messages that failed to reach the output table. If it doesn't exist, it is created during " + + "pipeline execution. If not specified, `_error_records` is used instead.", + example = ":.") String getOutputDeadletterTable(); void setOutputDeadletterTable(String value); diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index e365000e4d..e4b44918e0 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -166,7 +166,7 @@ public interface Options @TemplateParameter.Text( order = 4, description = "MongoDB collection", - helpText = "Name of the collection inside MongoDB database.", + helpText = "Name of the collection in the MongoDB database.", example = "my-collection") @Validation.Required String getCollection(); @@ -188,7 +188,7 @@ public interface Options order = 6, optional = true, description = "Batch Size", - helpText = "Batch Size used for batch insertion of documents into MongoDB.") + helpText = "Batch size used for batch insertion of documents into MongoDB.") @Default.Long(1000) Long getBatchSize(); @@ -198,7 +198,7 @@ public interface Options order = 7, optional = true, description = "Batch Size in Bytes", - helpText = "Batch Size in bytes used for batch insertion of documents into MongoDB.") + helpText = "Batch size in bytes.") @Default.Long(5242880) Long getBatchSizeBytes(); @@ -218,7 +218,7 @@ public interface Options order = 9, optional = true, description = "SSL Enabled", - helpText = "Indicates whether connection to MongoDB is ssl enabled.") + helpText = "Boolean value indicating whether connection to MongoDB is SSL enabled.") @Default.Boolean(true) Boolean getSslEnabled(); @@ -228,7 +228,7 @@ public interface Options order = 10, optional = true, description = "Ignore SSL Certificate", - helpText = "Indicates whether SSL certificate should be ignored.") + helpText = "Boolean value indicating if SSL certificate should be ignored.") @Default.Boolean(true) Boolean getIgnoreSSLCertificate(); @@ -238,7 +238,7 @@ public interface Options order = 11, optional = true, description = "withOrdered", - helpText = "Enables ordered bulk insertions into MongoDB.") + helpText = "Boolean value enabling ordered bulk insertions into MongoDB.") @Default.Boolean(true) Boolean getWithOrdered(); @@ -248,7 +248,7 @@ public interface Options order = 12, optional = true, description = "withSSLInvalidHostNameAllowed", - helpText = "Indicates whether invalid host name is allowed for ssl connection.") + helpText = "Boolean value indicating if invalid hostname is allowed for SSL connection.") @Default.Boolean(true) Boolean getWithSSLInvalidHostNameAllowed(); From 2f051c351c8a79eff9db28e2ca24dbbb123b2a79 Mon Sep 17 00:00:00 2001 From: rajkgupt Date: Tue, 7 May 2024 06:21:41 +0530 Subject: [PATCH 5/9] Apply suggestions from code review changes suggested in descriptions Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../cloud/teleport/v2/templates/TextIOToBigQuery.java | 6 +++--- .../teleport/v2/templates/TextToBigQueryStreaming.java | 2 +- .../cloud/teleport/v2/templates/PubSubToMongoDB.java | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index 05e9f602b8..85dda5ffd9 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -249,7 +249,7 @@ public interface Options regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "GCS path to javascript fn for transforming output", helpText = - "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) you want " + + "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) " + "to use.", example = "gs://your-bucket/your-transforms/*.js") String getJavascriptTextTransformGcsPath(); @@ -262,10 +262,10 @@ public interface Options regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the JavaScript user-defined function (UDF) that you want to use. For example, if " + + "The name of the JavaScript user-defined function (UDF) =to use. For example, if " + "your JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then " + "the function name is `myTransform`. For sample JavaScript UDFs, " + - "see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples)", + "see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples).", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index 56103e3c81..b4b191165e 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -480,7 +480,7 @@ public interface TextToBigQueryStreamingOptions description = "The dead-letter table name to output failed messages to BigQuery", helpText = "Table for messages that failed to reach the output table. If it doesn't exist, it is created during " + - "pipeline execution. If not specified, `_error_records` is used instead.", + "pipeline execution. If not specified, `_error_records` is used.", example = ":.") String getOutputDeadletterTable(); diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index 937517ea47..27e8d2e38a 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -214,7 +214,7 @@ public interface Options order = 5, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "BigQuery table that store messages due to failures (mismatched schema, malformed JSON etc).", + "The BigQuery table that stores messages caused by failures such as mismatched schema, malformed JSON and so on.", example = "your-project-id:your-dataset.your-table-name") @Validation.Required String getDeadletterTable(); @@ -255,7 +255,7 @@ public interface Options order = 9, optional = true, description = "SSL Enabled", - helpText = "Boolean value indicating whether connection to MongoDB is SSL enabled.") + helpText = "Boolean value indicating whether the connection to MongoDB is SSL enabled.") @Default.Boolean(true) Boolean getSslEnabled(); @@ -265,7 +265,7 @@ public interface Options order = 10, optional = true, description = "Ignore SSL Certificate", - helpText = "Boolean value indicating if SSL certificate should be ignored.") + helpText = "Boolean value indicating whether to ignore the SSL certificate.") @Default.Boolean(true) Boolean getIgnoreSSLCertificate(); @@ -285,7 +285,7 @@ public interface Options order = 12, optional = true, description = "withSSLInvalidHostNameAllowed", - helpText = "Boolean value indicating if invalid hostname is allowed for SSL connection.") + helpText = "Boolean value indicating whether an invalid hostname is allowed for the SSL connection.") @Default.Boolean(true) Boolean getWithSSLInvalidHostNameAllowed(); From 4f4d381d53f489b969152f72de277df31dcb9fbf Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Tue, 7 May 2024 06:32:48 +0530 Subject: [PATCH 6/9] Changes for spotless apply --- .../teleport/v2/templates/TextIOToBigQuery.java | 15 +++++++-------- .../v2/templates/TextToBigQueryStreaming.java | 4 ++-- .../teleport/v2/templates/PubSubToMongoDB.java | 6 +++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index 85dda5ffd9..16f853c48c 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -236,8 +236,7 @@ public interface Options optional = false, regexes = {".+:.+\\..+"}, description = "Output table to write to", - helpText = - "The fully qualified BigQuery table.", + helpText = "The fully qualified BigQuery table.", example = ":.") String getOutputTable(); @@ -249,8 +248,8 @@ public interface Options regexes = {"^gs:\\/\\/[^\\n\\r]+$"}, description = "GCS path to javascript fn for transforming output", helpText = - "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) " + - "to use.", + "The Cloud Storage URI of the `.js` file that defines the JavaScript user-defined function (UDF) " + + "to use.", example = "gs://your-bucket/your-transforms/*.js") String getJavascriptTextTransformGcsPath(); @@ -262,10 +261,10 @@ public interface Options regexes = {"[a-zA-Z0-9_]+"}, description = "UDF Javascript Function Name", helpText = - "The name of the JavaScript user-defined function (UDF) =to use. For example, if " + - "your JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then " + - "the function name is `myTransform`. For sample JavaScript UDFs, " + - "see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples).", + "The name of the JavaScript user-defined function (UDF) =to use. For example, if " + + "your JavaScript function code is `myTransform(inJson) { /*...do stuff...*/ }`, then " + + "the function name is `myTransform`. For sample JavaScript UDFs, " + + "see UDF Examples (https://github.com/GoogleCloudPlatform/DataflowTemplates#udf-examples).", example = "transform_udf1") String getJavascriptTextTransformFunctionName(); diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index b4b191165e..df75e54a24 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -479,8 +479,8 @@ public interface TextToBigQueryStreamingOptions optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "Table for messages that failed to reach the output table. If it doesn't exist, it is created during " + - "pipeline execution. If not specified, `_error_records` is used.", + "Table for messages that failed to reach the output table. If it doesn't exist, it is created during " + + "pipeline execution. If not specified, `_error_records` is used.", example = ":.") String getOutputDeadletterTable(); diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index 27e8d2e38a..633e8c1603 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -172,8 +172,7 @@ public interface Options @TemplateParameter.PubsubSubscription( order = 1, description = "Pub/Sub input subscription", - helpText = - "Name of the Pub/Sub subscription.", + helpText = "Name of the Pub/Sub subscription.", example = "projects/your-project-id/subscriptions/your-subscription-name") @Validation.Required String getInputSubscription(); @@ -285,7 +284,8 @@ public interface Options order = 12, optional = true, description = "withSSLInvalidHostNameAllowed", - helpText = "Boolean value indicating whether an invalid hostname is allowed for the SSL connection.") + helpText = + "Boolean value indicating whether an invalid hostname is allowed for the SSL connection.") @Default.Boolean(true) Boolean getWithSSLInvalidHostNameAllowed(); From 2d4887fb1dd7e2971945f7b628cc05714d29c13f Mon Sep 17 00:00:00 2001 From: rajkgupt Date: Mon, 13 May 2024 10:42:56 +0530 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../cloud/teleport/v2/templates/TextToBigQueryStreaming.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java index b217d60b71..3fbb79d6a0 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextToBigQueryStreaming.java @@ -479,7 +479,7 @@ public interface TextToBigQueryStreamingOptions optional = true, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "Table for messages that failed to reach the output table. If it doesn't exist, it is created during " + "Table for messages that failed to reach the output table. If a table doesn't exist, it is created during " + "pipeline execution. If not specified, `_error_records` is used.", example = ":.") String getOutputDeadletterTable(); From f5ae6a9816ac2b97fe0ae5ac92563ff7100958b1 Mon Sep 17 00:00:00 2001 From: Rajkumar Gupta Date: Mon, 13 May 2024 10:58:08 +0530 Subject: [PATCH 8/9] Modified output table parameter --- .../google/cloud/teleport/v2/templates/TextIOToBigQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java index c544fdb20b..6a04f41050 100644 --- a/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java +++ b/v2/googlecloud-to-googlecloud/src/main/java/com/google/cloud/teleport/v2/templates/TextIOToBigQuery.java @@ -238,7 +238,7 @@ public interface Options regexes = {".+:.+\\..+"}, description = "Output table to write to", helpText = - "The BigQuery table name you want to create to store your processed data in. If you reuse an existing BigQuery table, the data is appended to the destination table.", + "The location of the BigQuery table to use to store the processed data. If you reuse an existing table, it is overwritten.", example = ":.") String getOutputTable(); From 72544b94beb3eb5439035edd9211a4ec87d63fa9 Mon Sep 17 00:00:00 2001 From: rajkgupt Date: Tue, 14 May 2024 12:57:31 +0530 Subject: [PATCH 9/9] Update v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java Co-authored-by: Rebecca Szper <98840847+rszper@users.noreply.github.com> --- .../com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java index 633e8c1603..ebe2545385 100644 --- a/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java +++ b/v2/pubsub-to-mongodb/src/main/java/com/google/cloud/teleport/v2/templates/PubSubToMongoDB.java @@ -213,7 +213,7 @@ public interface Options order = 5, description = "The dead-letter table name to output failed messages to BigQuery", helpText = - "The BigQuery table that stores messages caused by failures such as mismatched schema, malformed JSON and so on.", + "The BigQuery table that stores messages caused by failures, such as mismatched schema, malformed JSON, and so on.", example = "your-project-id:your-dataset.your-table-name") @Validation.Required String getDeadletterTable();