Merge pull request #8 from deseretdigital/feature-create-terraform-mo…

…dule-for-pub-sub-topics-sc-308777

Bump Documentation

README.md

@@ -1,4 +1,4 @@
-# Google PubSub Subscription
+# Google PubSub Subscription with Dead Letter
 
 This module create a Google PubSub Subscription as as well as a Topic/Subscription for Dead Letters. 
 
@@ -9,15 +9,14 @@ We found that in order to follow the documentation for the provider, we were con
 ### Basic Configuration:
 
 ```hcl
-module "ddm-pubsub-subscription" {
+module "pubsub_subscription_module" {
   source  = "deseretdigital/ddm-pubsub-subscription/google"
-  version = "1.0.0"
+  version = "~> 2.0.0"
   
   # Required
   pubsub_service_account = {GKE_PUBSUB_SA_EMAIL}
   subscription_name      = {YOUR_SUBSCRIPTION_NAME}
   topic_id               = {PARENT_TOPIC_ID}
-  topic_name             = {PARENT_TOPIC_NAME}
 
   # Optional
   labels = {
@@ -55,10 +54,9 @@ resource "google_pubsub_topic" "example" {
 
 module "pubsub_subscription_module" {
   source                 = "deseretdigital/ddm-pubsub-subscription/google"
-  version                = "~> 1.0.0"
+  version                = "~> 2.0.0"
   pubsub_service_account = "service-{NUMBERS}@gcp-sa-pubsub.iam.gserviceaccount.com"
   subscription_name      = "Example_SubscriptionName"
-  topic_name             = google_pubsub_topic.example.name
   topic_id               = google_pubsub_topic.example.id
   
   labels = {

modules/big-query/.gitignore

@@ -0,0 +1,34 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

modules/big-query/README.md

@@ -0,0 +1,94 @@
+# Google PubSub Subscription with BigQuery Output
+
+This module create a Google PubSub Subscription that will automagically write to a BigQuery table. 
+
+## Usage
+
+### Basic Configuration:
+
+```hcl
+module "pubsub_subscription_module" {
+  source  = "deseretdigital/ddm-pubsub-subscription/google//modules/big-query"
+  version = "~> 2.0.0"
+  
+  # Required
+  bigquery_table         = {TABLE_NAME}
+  pubsub_service_account = "service-{NUMBERS}@gcp-sa-pubsub.iam.gserviceaccount.com"
+  subscription_name      = {YOUR_SUBSCRIPTION_NAME}
+  topic_id               = {PARENT_TOPIC_ID}
+
+  # Optional
+  labels = {
+    env    = "prod"
+    region = {REGION}
+    # etc...
+  }
+
+  use_topic_schema = false
+}
+```
+
+#### Example Usage
+
+This example assumes you are not using the topic schema and instead are using the table schema. You'll need to know this before you go in which schema you will use as your source of truth.
+
+```hcl
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+      version = "~> 6.0"
+    }
+  }
+}
+
+provider "google" {
+  # Configuration options
+}
+
+resource "google_pubsub_topic" "example" {
+  name = "Example_TopicName"
+}
+
+resource "google_bigquery_dataset" "test" {
+  dataset_id = "example_dataset"
+}
+
+resource "google_bigquery_table" "test" {
+  deletion_protection = false
+  table_id            = "example_table"
+  dataset_id          = google_bigquery_dataset.test.dataset_id
+
+  schema = <<EOF
+[
+  {
+    "name": "data",
+    "type": "STRING",
+    "mode": "NULLABLE",
+    "description": "The data"
+  }
+]
+EOF
+}
+
+
+module "pubsub_subscription_module" {
+  source  = "deseretdigital/ddm-pubsub-subscription/google//modules/big-query"
+  version = "~> 2.0.0"
+  
+  # Required
+  bigquery_table         = "${google_bigquery_table.test.project}.${google_bigquery_table.test.dataset_id}.${google_bigquery_table.test.table_id}"
+  pubsub_service_account = "service-{NUMBERS}@gcp-sa-pubsub.iam.gserviceaccount.com"
+  subscription_name      = "Example_SubscriptionName"
+  topic_id               = google_pubsub_topic.example.id
+
+  # Optional
+  labels = {
+    env    = "prod"
+    region = {REGION}
+    # etc...
+  }
+
+  use_topic_schema = false # Since we're using the scheme of 'google_bigquery_table.test'
+}
+```

modules/big-query/data.tf

@@ -0,0 +1,2 @@
+data "google_project" "project" {
+}

modules/big-query/main.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 6.0"
+    }
+  }
+}

modules/big-query/output.tf

@@ -0,0 +1,9 @@
+output "subscription_id" {
+  value       = google_pubsub_subscription.subscription.id
+  description = "The ID of the created subscription."
+}
+
+output "subscription_name" {
+  value       = google_pubsub_subscription.subscription.name
+  description = "The name of the created subscription."
+}

modules/big-query/resources.tf

@@ -0,0 +1,24 @@
+resource "google_pubsub_subscription" "subscription" {
+  name   = var.subscription_name
+  topic  = var.topic_id
+  labels = var.labels
+
+  bigquery_config {
+    use_topic_schema = var.use_topic_schema
+    table            = var.bigquery_table
+  }
+
+  depends_on = [google_project_iam_member.viewer, google_project_iam_member.editor]
+}
+
+resource "google_project_iam_member" "viewer" {
+  project = data.google_project.project.project_id
+  role    = "roles/bigquery.metadataViewer"
+  member  = "serviceAccount:${var.pubsub_service_account}"
+}
+
+resource "google_project_iam_member" "editor" {
+  project = data.google_project.project.project_id
+  role    = "roles/bigquery.dataEditor"
+  member  = "serviceAccount:${var.pubsub_service_account}"
+}

modules/big-query/terraform-docs.md

@@ -0,0 +1,42 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 6.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | ~> 6.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [google_project_iam_member.editor](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
+| [google_project_iam_member.viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
+| [google_pubsub_subscription.subscription](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
+| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_bigquery_table"></a> [bigquery\_table](#input\_bigquery\_table) | The name of the table to which to write data, of the form {projectId}.{datasetId}.{tableId} | `string` | n/a | yes |
+| <a name="input_labels"></a> [labels](#input\_labels) | A set of key/value label pairs to assign to this Topic. | `map(string)` | `{}` | no |
+| <a name="input_pubsub_service_account"></a> [pubsub\_service\_account](#input\_pubsub\_service\_account) | The service account to be used by the Pub/Sub system. Looks like 'service-<project-number>@gcp-sa-pubsub.iam.gserviceaccount.com'. | `string` | n/a | yes |
+| <a name="input_subscription_name"></a> [subscription\_name](#input\_subscription\_name) | The name of the subscription. | `string` | n/a | yes |
+| <a name="input_topic_id"></a> [topic\_id](#input\_topic\_id) | A reference to a Topic resource, of the form projects/{project}/topics/{{name}} (as in the id property of a google\_pubsub\_topic), or just a topic name if the topic is in the same project as the subscription. | `string` | n/a | yes |
+| <a name="input_use_topic_schema"></a> [use\_topic\_schema](#input\_use\_topic\_schema) | When true, use the topic's schema as the columns to write to in BigQuery, if it exists. | `bool` | `false` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_subscription_id"></a> [subscription\_id](#output\_subscription\_id) | The ID of the created subscription. |
+| <a name="output_subscription_name"></a> [subscription\_name](#output\_subscription\_name) | The name of the created subscription. |

modules/big-query/variables.tf

@@ -0,0 +1,45 @@
+variable "bigquery_table" {
+  description = "The name of the table to which to write data, of the form {projectId}.{datasetId}.{tableId}"
+  type        = string
+
+  validation {
+    condition     = can(regex("^[^\\.]+\\.[^\\.]+\\.[^\\.]+$", var.bigquery_table))
+    error_message = "Value must be a valid BigQuery table name."
+  }
+}
+
+variable "labels" {
+  description = "A set of key/value label pairs to assign to this Topic."
+  type        = map(string)
+  default     = {}
+}
+
+variable "pubsub_service_account" {
+  description = "The service account to be used by the Pub/Sub system. Looks like 'service-<project-number>@gcp-sa-pubsub.iam.gserviceaccount.com'."
+  type        = string
+  validation {
+    condition     = can(regex("^service-\\d+@gcp-sa-pubsub\\.iam\\.gserviceaccount\\.com$", var.pubsub_service_account))
+    error_message = "value must be a valid service account email address."
+  }
+}
+
+variable "subscription_name" {
+  description = "The name of the subscription."
+  type        = string
+}
+
+variable "topic_id" {
+  description = " A reference to a Topic resource, of the form projects/{project}/topics/{{name}} (as in the id property of a google_pubsub_topic), or just a topic name if the topic is in the same project as the subscription."
+  type        = string
+
+  validation {
+    condition     = can(regex("projects/[^/]+/topics/[^/]+", var.topic_id))
+    error_message = "value must be a reference to a Topic resource, of the form projects/{project}/topics/{{name}}."
+  }
+}
+
+variable "use_topic_schema" {
+  description = "When true, use the topic's schema as the columns to write to in BigQuery, if it exists."
+  type        = bool
+  default     = false
+}

resources.tf

@@ -1,6 +1,7 @@
 resource "google_pubsub_subscription" "subscription" {
-  name  = var.subscription_name
-  topic = var.topic_id
+  name   = var.subscription_name
+  topic  = var.topic_id
+  labels = var.labels
 
   dead_letter_policy {
     dead_letter_topic     = google_pubsub_topic.dead_letter_subscription_topic.id
@@ -30,6 +31,7 @@ resource "google_pubsub_topic_iam_binding" "assign_pubsub_subscriber" {
 }
 
 resource "google_pubsub_subscription" "dead_letter_subscription" {
-  name  = "${var.subscription_name}_DeadLetter"
-  topic = google_pubsub_topic.dead_letter_subscription_topic.id
+  name   = "${var.subscription_name}_DeadLetter"
+  topic  = google_pubsub_topic.dead_letter_subscription_topic.id
+  labels = var.labels
 }

terraform-docs.md

@@ -36,11 +36,10 @@ No modules.
 | <a name="input_pubsub_service_account"></a> [pubsub\_service\_account](#input\_pubsub\_service\_account) | The service account to be used by the Pub/Sub system. Looks like 'service-<project-number>@gcp-sa-pubsub.iam.gserviceaccount.com'. | `string` | n/a | yes |
 | <a name="input_subscription_name"></a> [subscription\_name](#input\_subscription\_name) | The name of the subscription. | `string` | n/a | yes |
 | <a name="input_topic_id"></a> [topic\_id](#input\_topic\_id) | A reference to a Topic resource, of the form projects/{project}/topics/{{name}} (as in the id property of a google\_pubsub\_topic), or just a topic name if the topic is in the same project as the subscription. | `string` | n/a | yes |
-| <a name="input_topic_name"></a> [topic\_name](#input\_topic\_name) | The name of the topic. | `string` | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_subscription_id"></a> [subscription\_id](#output\_subscription\_id) | n/a |
-| <a name="output_subscription_name"></a> [subscription\_name](#output\_subscription\_name) | n/a |
+| <a name="output_subscription_id"></a> [subscription\_id](#output\_subscription\_id) | The ID of the created subscription. |
+| <a name="output_subscription_name"></a> [subscription\_name](#output\_subscription\_name) | The name of the created subscription. |

variables.tf

@@ -49,8 +49,3 @@ variable "topic_id" {
     error_message = "value must be a reference to a Topic resource, of the form projects/{project}/topics/{{name}}."
   }
 }
-
-variable "topic_name" {
-  description = "The name of the topic."
-  type        = string
-}