Merge pull request #13 from terraform-google-modules/splunk-example

Add Splunk example to log export module

examples/apply_all.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -e
-cd project-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars
-cd ../folder-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars
-cd ../org-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars
+cd project-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample
+cd ../folder-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample
+cd ../org-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample
+cd ../splunk-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample

examples/destroy_all.sh

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -e
-cd project-sink/ && terraform init && terraform destroy --var-file=../example.tfvars
-cd ../folder-sink/ && terraform init && terraform destroy --var-file=../example.tfvars
-cd ../org-sink/ && terraform init && terraform destroy --var-file=../example.tfvars
+cd project-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample
+cd ../folder-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample
+cd ../org-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample
+cd ../splunk-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample

examples/folder-sink/terraform.tfvars.sample

@@ -0,0 +1 @@
+../terraform.tfvars.sample

examples/org-sink/terraform.tfvars.sample

@@ -0,0 +1 @@
+../terraform.tfvars.sample

examples/project-sink/terraform.tfvars.sample

@@ -0,0 +1 @@
+../terraform.tfvars.sample

examples/readme.md

@@ -5,8 +5,8 @@ Examples to create logsinks on Google Cloud Platform using the `logsink` module.
 
 Variables
 ---------
-The Terraform variables for all the examples are defined in the `example.tfvars` in the [examples](./examples)
-directory. Replace the variables by your own before running the examples.
+The Terraform variables for all the examples are defined in the `terraform.tfvars.sample` in the [examples](./examples)
+directory. This file is symlinked to every example directory. Replace all the variables by your own before running the examples.
 
 Examples
 --------
@@ -18,19 +18,21 @@ Examples are structured as follows:
 
 * ***org-sink/*** creates 3 organization-level aggregated sinks (Pub/Sub, Cloud Storage, BigQuery)
 
+* ***splunk-sink/*** creates 1 project-level sink to Pub/Sub in order to be ingested to Splunk
+
 Each example can be run individually by going to each folder and running:
 
 ```
 terraform init
-terraform apply --var-file=../example.tfvars
+terraform apply --var-file=terraform.tfvars.sample
 ```
 
 Scripts
 -------
 
 Scripts have been written to automate running all the examples on an organization:
 
-* `./apply_all.sh` will run all the examples (thus creating 1 folder, 1 project and 9 logsinks).
+* `./apply_all.sh` will run all the examples (thus creating 1 folder, 1 project and 10 log sinks).
 
 * `./destroy_all.sh` will destroy all the resources previously created.
 

examples/splunk-sink/README.md

@@ -0,0 +1,62 @@
+# Splunk sink example
+
+The solution helps you set up a log-streaming pipeline from Stackdriver Logging to Splunk.
+
+The example is for a project-level sink, but it can be easily be adapted for aggregated log sinks.
+
+## Instructions
+
+1. Fill the required variables in the `terraform.tfvars.sample` file located in the `examples/` directory.
+
+2. Verify the IAM roles for your Terraform service account:
+    - `roles/logging.configWriter` on the project (to create the logsink)
+    - `roles/iam.admin` on the project (to grant write permissions for logsink service account)
+    - `roles/serviceusage.admin` on the destination project (to enable destination API)
+    - `roles/pubsub.admin` on the destination project (to create a pub/sub topic)
+    - `roles/serviceAccount.admin` on the destination project (to create a service account for the logsink subscriber)
+
+2. Run the Terraform automation:
+    ```
+    terraform init
+    terraform apply
+    ```
+
+    You should see similar outputs as the following:
+
+    ![screen shot 2019-01-25 at 1 21 14 pm](https://user-images.githubusercontent.com/9629314/51767833-23459980-20a4-11e9-831c-01a2943ee745.png)
+
+3. In the GCP console, under `IAM > Service Accounts`, find the Pub/Sub subscriber service account and create a set of JSON credentials:
+
+    ![screen shot 2019-01-25 at 1 24 27 pm](https://user-images.githubusercontent.com/9629314/51767992-8fc09880-20a4-11e9-8e69-aa8b3f6e360d.png)
+
+4. Go to your `Splunk` web console. On the left panel, click on the big `+` squared box:
+
+    ![screen shot 2019-01-25 at 1 28 10 pm](https://user-images.githubusercontent.com/9629314/51768142-170e0c00-20a5-11e9-9190-eac68a057e86.png)
+
+5. Search for the `Google Cloud Platform` add-on and install it.
+
+    ![screen shot 2019-01-25 at 1 30 00 pm](https://user-images.githubusercontent.com/9629314/51768246-65bba600-20a5-11e9-829f-2feae4f295dd.png)
+
+    *Note: you might need to restart your Splunk instance after the installation.*
+
+5. Click on the add-on tile and navigate to the `Configuration` tab. Under the `Google Credentials` menu item, click on `Add Credential`.
+
+    ![screen shot 2019-01-25 at 1 34 16 pm](https://user-images.githubusercontent.com/9629314/51768443-f72b1800-20a5-11e9-955c-4c3ae6952e7f.png)
+
+6. Copy the content of the downloaded JSON file to the popup window:
+
+    ![screen shot 2019-01-25 at 1 37 17 pm](https://user-images.githubusercontent.com/9629314/51768595-5c7f0900-20a6-11e9-9135-d28fa4fbff20.png)
+
+7. Switch to the `Inputs` tab, and click on `Create New Input` and select `Cloud Pub/Sub`.
+
+8. Fill the required values from the Terraform outputs, and click `Add`:
+
+    ![screen shot 2019-01-25 at 1 39 16 pm](https://user-images.githubusercontent.com/9629314/51768687-a1a33b00-20a6-11e9-9871-b4b6c97f29bb.png)
+
+    *Note: If you have lost the Terraform outputs, simply run `terraform output` in this directory to see them again.*
+
+9. Switch to the `Search` tab. You should see that Splunk ingested some events:
+
+    ![screen shot 2019-01-25 at 1 42 25 pm](https://user-images.githubusercontent.com/9629314/51768902-33ab4380-20a7-11e9-8f91-22d4eed777e7.png)
+
+10. **Congratulations !** Your Stackdriver-to-Splunk logging pipeline is up and running !

examples/splunk-sink/main.tf

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+provider "google" {
+  credentials = "${file(var.credentials_path)}"
+}
+
+module "splunk-sink" {
+  source  = "../../"
+  name    = "test-splunk-sink"
+  project = "${var.project_id}"
+
+  pubsub = {
+    name              = "splunk-sink"
+    project           = "${var.project_id}"
+    create_subscriber = true
+  }
+}

examples/splunk-sink/outputs.tf

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "pubsub_topic_name" {
+  description = "Pub/Sub topic name"
+  value       = "projects/${module.splunk-sink.destination["project"]}/topics/${module.splunk-sink.destination["name"]}"
+}
+
+output "pubsub_topic_project" {
+  description = "Pub/Sub topic project id"
+  value       = "${module.splunk-sink.destination["project"]}"
+}
+
+output "pubsub_subscription_name" {
+  description = "Pub/Sub topic subscription name"
+  value       = "${module.splunk-sink.pubsub_subscription}"
+}
+
+output "pubsub_subscriber" {
+  description = "Pub/Sub topic subscriber email"
+  value       = "${module.splunk-sink.pubsub_subscriber}"
+}

examples/splunk-sink/terraform.tfvars.sample

@@ -0,0 +1 @@
+../terraform.tfvars.sample

examples/splunk-sink/variables.tf

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "credentials_path" {
+  description = "Path to a Service Account credentials file with permissions documented in the readme"
+}
+
+variable "project_id" {
+  description = "The project name (for project-level sink)"
+}

examples/terraform.tfvars.sample

@@ -0,0 +1,11 @@
+credentials_path = "<PATH_TO_CREDENTIALS_JSON>"
+
+org_id = "<ORG_ID>"
+
+folder_id = "<FOLDER_ID>"
+
+project_id = "<PROJECT_ID>"
+
+destination_project_id = "<DESTINATION_PROJECT_ID>"
+
+gcs_bucket_name = "<GCS_BUCKET_NAME>"

main.tf

@@ -39,6 +39,7 @@ locals {
   # Additional options for specific destinations
   pubsub_create_subscriber = "${lookup(local.destination, "create_subscriber", false)}"
   pubsub_subscriber        = "${element(concat(google_service_account.pubsub_subscriber.*.email, list("")), 0)}"
+  pubsub_subscription      = "${element(concat(google_pubsub_subscription.pubsub_subscription.*.id, list("")), 0)}"
 
   # Role assigned to sink writer and sink level
   role  = "${local.role_map[local.destination_type]}"

outputs.tf

@@ -15,13 +15,21 @@
  */
 
 output "sink" {
-  value = "${local.sink_output}"
+  description = "Log sink data map"
+  value       = "${local.sink_output}"
 }
 
 output "destination" {
-  value = "${local.destination_output}"
+  description = "Destination data map"
+  value       = "${local.destination_output}"
 }
 
 output "pubsub_subscriber" {
-  value = "${local.pubsub_subscriber}"
+  description = "Pub/Sub subscriber email (if any)"
+  value       = "${local.pubsub_subscriber}"
+}
+
+output "pubsub_subscription" {
+  description = "Pub/Sub subscription id (if any)"
+  value       = "${local.pubsub_subscription}"
 }

variables.tf

@@ -54,16 +54,19 @@ variable "unique_writer_identity" {
 }
 
 variable "bigquery" {
-  type    = "map"
-  default = {}
+  description = "BigQuery dataset configuration"
+  type        = "map"
+  default     = {}
 }
 
 variable "storage" {
-  type    = "map"
-  default = {}
+  description = "Cloud Storage bucket configuration"
+  type        = "map"
+  default     = {}
 }
 
 variable "pubsub" {
-  type    = "map"
-  default = {}
+  description = "Pub/Sub topic configuration"
+  type        = "map"
+  default     = {}
 }