diff --git a/examples/apply_all.sh b/examples/apply_all.sh index 8c2195cb..d8c80676 100755 --- a/examples/apply_all.sh +++ b/examples/apply_all.sh @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. set -e -cd project-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars -cd ../folder-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars -cd ../org-sink/ && terraform init -upgrade && terraform apply --var-file=../example.tfvars +cd project-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample +cd ../folder-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample +cd ../org-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample +cd ../splunk-sink/ && terraform init && terraform apply -auto-approve -var-file=terraform.tfvars.sample diff --git a/examples/destroy_all.sh b/examples/destroy_all.sh index 4e4d49b8..9ccee072 100755 --- a/examples/destroy_all.sh +++ b/examples/destroy_all.sh @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. set -e -cd project-sink/ && terraform init && terraform destroy --var-file=../example.tfvars -cd ../folder-sink/ && terraform init && terraform destroy --var-file=../example.tfvars -cd ../org-sink/ && terraform init && terraform destroy --var-file=../example.tfvars +cd project-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample +cd ../folder-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample +cd ../org-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample +cd ../splunk-sink/ && terraform init && terraform destroy -auto-approve -var-file=terraform.tfvars.sample diff --git a/examples/example.tfvars b/examples/example.tfvars deleted file mode 100644 index 5f1419e5..00000000 --- a/examples/example.tfvars +++ /dev/null @@ -1,11 +0,0 @@ -credentials_path="" - -org_id="" - -folder_id="" - -project_id="" - -destination_project_id="" - -gcs_bucket_name=" diff --git a/examples/folder-sink/terraform.tfvars.sample b/examples/folder-sink/terraform.tfvars.sample new file mode 120000 index 00000000..f458842a --- /dev/null +++ b/examples/folder-sink/terraform.tfvars.sample @@ -0,0 +1 @@ +../terraform.tfvars.sample \ No newline at end of file diff --git a/examples/org-sink/terraform.tfvars.sample b/examples/org-sink/terraform.tfvars.sample new file mode 120000 index 00000000..f458842a --- /dev/null +++ b/examples/org-sink/terraform.tfvars.sample @@ -0,0 +1 @@ +../terraform.tfvars.sample \ No newline at end of file diff --git a/examples/project-sink/terraform.tfvars.sample b/examples/project-sink/terraform.tfvars.sample new file mode 120000 index 00000000..f458842a --- /dev/null +++ b/examples/project-sink/terraform.tfvars.sample @@ -0,0 +1 @@ +../terraform.tfvars.sample \ No newline at end of file diff --git a/examples/readme.md b/examples/readme.md index e22f9b48..e3fff0e4 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -5,8 +5,8 @@ Examples to create logsinks on Google Cloud Platform using the `logsink` module. Variables --------- -The Terraform variables for all the examples are defined in the `example.tfvars` in the [examples](./examples) -directory. Replace the variables by your own before running the examples. +The Terraform variables for all the examples are defined in the `terraform.tfvars.sample` in the [examples](./examples) +directory. This file is symlinked to every example directory. Replace all the variables by your own before running the examples. Examples -------- @@ -18,11 +18,13 @@ Examples are structured as follows: * ***org-sink/*** creates 3 organization-level aggregated sinks (Pub/Sub, Cloud Storage, BigQuery) +* ***splunk-sink/*** creates 1 project-level sink to Pub/Sub in order to be ingested to Splunk + Each example can be run individually by going to each folder and running: ``` terraform init -terraform apply --var-file=../example.tfvars +terraform apply --var-file=terraform.tfvars.sample ``` Scripts @@ -30,7 +32,7 @@ Scripts Scripts have been written to automate running all the examples on an organization: -* `./apply_all.sh` will run all the examples (thus creating 1 folder, 1 project and 9 logsinks). +* `./apply_all.sh` will run all the examples (thus creating 1 folder, 1 project and 10 log sinks). * `./destroy_all.sh` will destroy all the resources previously created. diff --git a/examples/splunk-sink/README.md b/examples/splunk-sink/README.md new file mode 100644 index 00000000..a6c4fa6f --- /dev/null +++ b/examples/splunk-sink/README.md @@ -0,0 +1,62 @@ +# Splunk sink example + +The solution helps you set up a log-streaming pipeline from Stackdriver Logging to Splunk. + +The example is for a project-level sink, but it can be easily be adapted for aggregated log sinks. + +## Instructions + +1. Fill the required variables in the `terraform.tfvars.sample` file located in the `examples/` directory. + +2. Verify the IAM roles for your Terraform service account: + - `roles/logging.configWriter` on the project (to create the logsink) + - `roles/iam.admin` on the project (to grant write permissions for logsink service account) + - `roles/serviceusage.admin` on the destination project (to enable destination API) + - `roles/pubsub.admin` on the destination project (to create a pub/sub topic) + - `roles/serviceAccount.admin` on the destination project (to create a service account for the logsink subscriber) + +2. Run the Terraform automation: + ``` + terraform init + terraform apply + ``` + + You should see similar outputs as the following: + + ![screen shot 2019-01-25 at 1 21 14 pm](https://user-images.githubusercontent.com/9629314/51767833-23459980-20a4-11e9-831c-01a2943ee745.png) + +3. In the GCP console, under `IAM > Service Accounts`, find the Pub/Sub subscriber service account and create a set of JSON credentials: + + ![screen shot 2019-01-25 at 1 24 27 pm](https://user-images.githubusercontent.com/9629314/51767992-8fc09880-20a4-11e9-8e69-aa8b3f6e360d.png) + +4. Go to your `Splunk` web console. On the left panel, click on the big `+` squared box: + + ![screen shot 2019-01-25 at 1 28 10 pm](https://user-images.githubusercontent.com/9629314/51768142-170e0c00-20a5-11e9-9190-eac68a057e86.png) + +5. Search for the `Google Cloud Platform` add-on and install it. + + ![screen shot 2019-01-25 at 1 30 00 pm](https://user-images.githubusercontent.com/9629314/51768246-65bba600-20a5-11e9-829f-2feae4f295dd.png) + + *Note: you might need to restart your Splunk instance after the installation.* + +5. Click on the add-on tile and navigate to the `Configuration` tab. Under the `Google Credentials` menu item, click on `Add Credential`. + + ![screen shot 2019-01-25 at 1 34 16 pm](https://user-images.githubusercontent.com/9629314/51768443-f72b1800-20a5-11e9-955c-4c3ae6952e7f.png) + +6. Copy the content of the downloaded JSON file to the popup window: + + ![screen shot 2019-01-25 at 1 37 17 pm](https://user-images.githubusercontent.com/9629314/51768595-5c7f0900-20a6-11e9-9135-d28fa4fbff20.png) + +7. Switch to the `Inputs` tab, and click on `Create New Input` and select `Cloud Pub/Sub`. + +8. Fill the required values from the Terraform outputs, and click `Add`: + + ![screen shot 2019-01-25 at 1 39 16 pm](https://user-images.githubusercontent.com/9629314/51768687-a1a33b00-20a6-11e9-9871-b4b6c97f29bb.png) + + *Note: If you have lost the Terraform outputs, simply run `terraform output` in this directory to see them again.* + +9. Switch to the `Search` tab. You should see that Splunk ingested some events: + + ![screen shot 2019-01-25 at 1 42 25 pm](https://user-images.githubusercontent.com/9629314/51768902-33ab4380-20a7-11e9-8f91-22d4eed777e7.png) + +10. **Congratulations !** Your Stackdriver-to-Splunk logging pipeline is up and running ! diff --git a/examples/splunk-sink/main.tf b/examples/splunk-sink/main.tf new file mode 100644 index 00000000..b5ba6ac8 --- /dev/null +++ b/examples/splunk-sink/main.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + credentials = "${file(var.credentials_path)}" +} + +module "splunk-sink" { + source = "../../" + name = "test-splunk-sink" + project = "${var.project_id}" + + pubsub = { + name = "splunk-sink" + project = "${var.project_id}" + create_subscriber = true + } +} diff --git a/examples/splunk-sink/outputs.tf b/examples/splunk-sink/outputs.tf new file mode 100644 index 00000000..d91245ce --- /dev/null +++ b/examples/splunk-sink/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pubsub_topic_name" { + description = "Pub/Sub topic name" + value = "projects/${module.splunk-sink.destination["project"]}/topics/${module.splunk-sink.destination["name"]}" +} + +output "pubsub_topic_project" { + description = "Pub/Sub topic project id" + value = "${module.splunk-sink.destination["project"]}" +} + +output "pubsub_subscription_name" { + description = "Pub/Sub topic subscription name" + value = "${module.splunk-sink.pubsub_subscription}" +} + +output "pubsub_subscriber" { + description = "Pub/Sub topic subscriber email" + value = "${module.splunk-sink.pubsub_subscriber}" +} diff --git a/examples/splunk-sink/terraform.tfvars.sample b/examples/splunk-sink/terraform.tfvars.sample new file mode 120000 index 00000000..f458842a --- /dev/null +++ b/examples/splunk-sink/terraform.tfvars.sample @@ -0,0 +1 @@ +../terraform.tfvars.sample \ No newline at end of file diff --git a/examples/splunk-sink/variables.tf b/examples/splunk-sink/variables.tf new file mode 100644 index 00000000..61398bf1 --- /dev/null +++ b/examples/splunk-sink/variables.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "credentials_path" { + description = "Path to a Service Account credentials file with permissions documented in the readme" +} + +variable "project_id" { + description = "The project name (for project-level sink)" +} diff --git a/examples/terraform.tfvars.sample b/examples/terraform.tfvars.sample new file mode 100644 index 00000000..8ff8011f --- /dev/null +++ b/examples/terraform.tfvars.sample @@ -0,0 +1,11 @@ +credentials_path = "" + +org_id = "" + +folder_id = "" + +project_id = "" + +destination_project_id = "" + +gcs_bucket_name = "" diff --git a/main.tf b/main.tf index aa24d363..126c7694 100644 --- a/main.tf +++ b/main.tf @@ -39,6 +39,7 @@ locals { # Additional options for specific destinations pubsub_create_subscriber = "${lookup(local.destination, "create_subscriber", false)}" pubsub_subscriber = "${element(concat(google_service_account.pubsub_subscriber.*.email, list("")), 0)}" + pubsub_subscription = "${element(concat(google_pubsub_subscription.pubsub_subscription.*.id, list("")), 0)}" # Role assigned to sink writer and sink level role = "${local.role_map[local.destination_type]}" diff --git a/outputs.tf b/outputs.tf index c58509b0..f2b011d1 100644 --- a/outputs.tf +++ b/outputs.tf @@ -15,13 +15,21 @@ */ output "sink" { - value = "${local.sink_output}" + description = "Log sink data map" + value = "${local.sink_output}" } output "destination" { - value = "${local.destination_output}" + description = "Destination data map" + value = "${local.destination_output}" } output "pubsub_subscriber" { - value = "${local.pubsub_subscriber}" + description = "Pub/Sub subscriber email (if any)" + value = "${local.pubsub_subscriber}" +} + +output "pubsub_subscription" { + description = "Pub/Sub subscription id (if any)" + value = "${local.pubsub_subscription}" } diff --git a/variables.tf b/variables.tf index 6c5e3774..0c0a40ca 100644 --- a/variables.tf +++ b/variables.tf @@ -54,16 +54,19 @@ variable "unique_writer_identity" { } variable "bigquery" { - type = "map" - default = {} + description = "BigQuery dataset configuration" + type = "map" + default = {} } variable "storage" { - type = "map" - default = {} + description = "Cloud Storage bucket configuration" + type = "map" + default = {} } variable "pubsub" { - type = "map" - default = {} + description = "Pub/Sub topic configuration" + type = "map" + default = {} }