diff --git a/.github/workflows/codehealth.yml b/.github/workflows/codehealth.yml index c200687..e5b3376 100644 --- a/.github/workflows/codehealth.yml +++ b/.github/workflows/codehealth.yml @@ -22,18 +22,20 @@ jobs: node-version: 22.4.1 check-latest: true - # Install eslint + - name: Use terraform + uses: hashicorp/setup-terraform@v3 + - name: Install packages working-directory: cloudrun-malware-scanner/ run: npm install - - name: ESlint + - name: Check ESlint working-directory: cloudrun-malware-scanner/ run: npm run eslint - name: Check Format working-directory: cloudrun-malware-scanner/ - run: npm run check-format -- --log-level warn + run: npm run check-format - name: Typescript checks working-directory: cloudrun-malware-scanner/ @@ -42,3 +44,15 @@ jobs: - name: NPM Audit working-directory: cloudrun-malware-scanner/ run: npm audit + + - name: terraform validate infra + working-directory: terraform/infra/ + run: | + terraform init -no-color -input=false + terraform validate -no-color + + - name: terraform validate service + working-directory: terraform/service/ + run: | + terraform init -no-color -input=false + terraform validate -no-color diff --git a/.gitignore b/.gitignore index 03cddf9..56bf837 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,12 @@ cvds pyenv config.json .vscode + +# Terraform +*.tfstate +*.tfstate.backup +*.tfstate.lock.info +*.tfplan +.terraform +**/.terraform/* +.terraform.tfstate.lock.info diff --git a/cloudrun-malware-scanner/.gcloudignore b/cloudrun-malware-scanner/.gcloudignore index da72999..598b49f 100644 --- a/cloudrun-malware-scanner/.gcloudignore +++ b/cloudrun-malware-scanner/.gcloudignore @@ -3,3 +3,4 @@ pyenv node_modules .gcloudignore .eslintrc.js +config.json.tmpl diff --git a/cloudrun-malware-scanner/.husky/pre-commit b/cloudrun-malware-scanner/.husky/pre-commit index f24bc74..7f1c14a 100644 --- a/cloudrun-malware-scanner/.husky/pre-commit +++ b/cloudrun-malware-scanner/.husky/pre-commit @@ -4,4 +4,5 @@ cd cloudrun-malware-scanner npm run eslint npm run check-format npm run typecheck +npm run terraform-validate npm audit diff --git a/cloudrun-malware-scanner/.prettierignore b/cloudrun-malware-scanner/.prettierignore index 784342a..4172e82 100644 --- a/cloudrun-malware-scanner/.prettierignore +++ b/cloudrun-malware-scanner/.prettierignore @@ -9,3 +9,4 @@ package-lock.json public CHANGELOG.md ../.release-please-manifest.json +../terraform/*/.terraform diff --git a/cloudrun-malware-scanner/cloudbuild.yaml b/cloudrun-malware-scanner/cloudbuild.yaml new file mode 100644 index 0000000..9d4868f --- /dev/null +++ b/cloudrun-malware-scanner/cloudbuild.yaml @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + - name: "gcr.io/cloud-builders/docker:20.10.24" + args: + [ + "build", + "--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/malware-scanner/malware-scanner:latest", + "-f", + "Dockerfile", + ".", + ] +images: + - "$LOCATION-docker.pkg.dev/$PROJECT_ID/malware-scanner/malware-scanner:latest" +options: + logging: "CLOUD_LOGGING_ONLY" diff --git a/cloudrun-malware-scanner/package.json b/cloudrun-malware-scanner/package.json index 622c697..37b55b3 100644 --- a/cloudrun-malware-scanner/package.json +++ b/cloudrun-malware-scanner/package.json @@ -4,9 +4,13 @@ "description": "Service to scan GCS documents for the malware and move the analyzed documents to appropriate buckets", "main": "index.js", "scripts": { - "check-format": "prettier --config .prettierrc.js --check ..", + "check-format": "npm run prettier-check && npm run terraform-fmt-check", "start": "node server.js", + "terraform-fmt": "terraform fmt ../terraform/*/*.tf ../terraform/*/*/*.tf", + "terraform-fmt-check": "terraform fmt -check ../terraform/*/*.tf ../terraform/*/*/*.tf", + "terraform-validate": "echo 'validating terraform/infra' && cd ../terraform/infra && terraform init -input=false && terraform validate && echo 'validating terraform/service' && cd ../service && terraform init -input=false && terraform validate", "prettier": "prettier --config .prettierrc.js --write ..", + "prettier-check": "prettier --config .prettierrc.js --check --log-level=warn ..", "start-proxy": "node gcs-proxy-server.js", "test": "echo \"Error: no test specified\" && exit 1", "eslint": "eslint *.js", diff --git a/release-please-config.json b/release-please-config.json index 977279a..42545eb 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -7,7 +7,11 @@ "bump-patch-for-minor-pre-major": false, "draft": false, "prerelease": false, - "include-component-in-tag": false + "include-component-in-tag": false, + "extra-files": [ + "terraform/infra/versions.tf", + "terraform/service/versions.tf" + ] } }, "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..fb6c2f5 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,227 @@ +# Terraform deployment + +This directory contains the terraform files required to deploy the +malware-scanner service on cloud run. + +The deployment is split into 4 stages: + +1. Set up the google cloud project environment and service configuration. +1. Use Terraform to set up the required service accounts and deploy required + infrastructure. +1. Launch cloud build to build the Docker image for the malware-scanner + service. +1. Use Terraform to deploy the malware-scanner service to cloud run, and + connect the service to the infrastructure created in stage 2. + +Follow the instructions below to use Terraform to deploy the malware scanner +service in a demo project. + +## Create a project and assign billing + +Using the Cloud Console, create a new Cloud project, and assign the billing +account. Take note of the Project ID of your new project. + +## Clone repo + +In Cloud shell, run the following to pull the malware-scanner source code from +GitHub. + +```bash +git clone https://github.com/GoogleCloudPlatform/docker-clamav-malware-scanner.git +cd docker-clamav-malware-scanner +``` + +## Initialize environment with service configuration + +Run the following commands in Cloud Shell to setup your environment and specify +the parameters of your service for the Terraform deployment. + +Replace `MY_PROJECT_ID` with the ID of your newly created Project. + +```bash + +PROJECT_ID=MY_PROJECT_ID +gcloud config set project $PROJECT_ID + +TF_VAR_project_id=$PROJECT_ID +TF_VAR_config_json=$(cat < **Error:** Error creating Trigger: googleapi: Error 400: Invalid resource +> state for "": Permission denied while using the Eventarc Service Agent. If you +> recently started to use Eventarc, it may take a few minutes before all +> necessary permissions are propagated to the Service Agent. Otherwise, verify +> that it has Eventarc Service Agent role. + +## Test the service + +The service can be tested by querying the version numbers from the cloud run +service, and by verifying the operation by uploading files to the unscanned GCS +bucket. + +### Get version info from cloud run + +You can query the malware-scanner service for the version information: + +```bash +MALWARE_SCANNER_URL="$(terraform output -raw cloud_run_uri)" +curl -H "Authorization: Bearer $(gcloud auth print-identity-token)" \ + "${MALWARE_SCANNER_URL}" +``` + +This command will output lines showing the versions of the malware scanner, +ClamAV and the current malware definitions version/datem for example: + +```text +gcs-malware-scanner version 3.0.0 +Using Clam AV version: ClamAV 1.0.5/27396/Thu Sep 12 08:46:40 2024 + +Service to scan GCS documents for the malware and move the analyzed documents to appropriate buckets +``` + +### Create and scan a clean file and an simulated infected file + +Run the following command to create 2 files in the unscanned bucket, a simple +`clean.txt` file and an `eicar-infected.txt` file containing a +[test string which simulates a virus](https://en.wikipedia.org/wiki/EICAR_test_file) + +```bash +echo -e 'HELLO WORLD!' \ + | gcloud storage cp - "gs://unscanned-${PROJECT_ID}/clean.txt" +echo -e 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' \ + | gcloud storage cp - "gs://unscanned-${PROJECT_ID}/eicar-infected.txt" +``` + +Check contents of the unscanned bucket + +```bash +gcloud storage ls gs://unscanned-${PROJECT_ID}/ +``` + +This should return no results as the files will have been moved by the +malware-scanner. If the files still exist, re-run the command after a couple of +seconds. + +Check contents of the clean files bucket + +```bash +gcloud storage ls gs://clean-${PROJECT_ID}/ +``` + +This should show that the clean.txt file has been moved to the clean bucket. + +Check contents of the quarantined bucket: + +```bash +gcloud storage ls gs://quarantined-${PROJECT_ID}/ +``` + +This should show that the eicar-infected.txt file has been moved to the +quarantined bucket. + +Show the log items with the scan status: + +```bash +gcloud logging read \ + 'resource.labels.service_name = "malware-scanner" AND "Scan status for"' \ + --limit=10 --format='value(jsonPayload.message)' +``` + +This will output log lines similar to: + +```text +Scan status for gs://unscanned-PROJECT_ID/clean.txt: CLEAN (13 bytes in 85 ms) +Scan status for gs://unscanned-PROJECT_ID/eicar-infected.txt: INFECTED stream: Eicar-Signature FOUND (69 bytes in 77 ms) +``` diff --git a/terraform/infra/.terraform.lock.hcl b/terraform/infra/.terraform.lock.hcl new file mode 100644 index 0000000..5c062c8 --- /dev/null +++ b/terraform/infra/.terraform.lock.hcl @@ -0,0 +1,41 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.3.0" + constraints = ">= 6.3.0" + hashes = [ + "h1:YWlCJbSyVtYpsGIozv8XrgcqY1nrlsChYOZZyFj17mQ=", + "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", + "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", + "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", + "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", + "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", + "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", + "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", + "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", + "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", + "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", + "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} diff --git a/terraform/infra/apis/main.tf b/terraform/infra/apis/main.tf new file mode 100644 index 0000000..66c4030 --- /dev/null +++ b/terraform/infra/apis/main.tf @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Enable APIs +# +# Requires that the following APIs are already enabled: +# cloudresourcemanager.googleapis.com +# serviceusage.googleapis.com +# +resource "google_project_service" "apis" { + for_each = toset([ + "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", + "cloudscheduler.googleapis.com", + "eventarc.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "pubsub.googleapis.com", + "run.googleapis.com", + ]) + service = each.key +} + diff --git a/terraform/infra/create_buckets/main.tf b/terraform/infra/create_buckets/main.tf new file mode 100644 index 0000000..cc92a17 --- /dev/null +++ b/terraform/infra/create_buckets/main.tf @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_storage_bucket" "scanner_buckets" { + for_each = var.bucket_names + name = each.key + location = var.bucket_location + uniform_bucket_level_access = var.uniform_bucket_level_access +} diff --git a/terraform/infra/create_buckets/variables.tf b/terraform/infra/create_buckets/variables.tf new file mode 100644 index 0000000..c7706bc --- /dev/null +++ b/terraform/infra/create_buckets/variables.tf @@ -0,0 +1,29 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "bucket_location" { + description = "Location to create Cloud Storage buckets" + type = string +} + +variable "uniform_bucket_level_access" { + description = "When creating cloud storage buckets, the parameter uniform_bucket_level_access is set to this value" + default = true + type = bool +} + +variable "bucket_names" { + description = "The set of bucket names to create" + type = set(string) +} diff --git a/terraform/infra/main.tf b/terraform/infra/main.tf new file mode 100644 index 0000000..e6660c1 --- /dev/null +++ b/terraform/infra/main.tf @@ -0,0 +1,153 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +provider "google" { + project = var.project_id + region = var.region +} + +data "google_project" "project" { + project_id = var.project_id +} + +locals { + repo_root = abspath("${path.module}/../..") + src_root = abspath("${local.repo_root}/cloudrun-malware-scanner") + + ## Read config and extract bucket names + config_json = jsondecode(var.config_json) + cvd_mirror_bucket = local.config_json.ClamCvdMirrorBucket + unscanned_bucket_names = local.config_json.buckets[*].unscanned + clean_bucket_names = local.config_json.buckets[*].clean + quarantined_bucket_names = local.config_json.buckets[*].quarantined + all_buckets = toset(concat(local.clean_bucket_names, local.unscanned_bucket_names, local.quarantined_bucket_names)) +} + +## Enable the APIs +module "apis" { + source = "./apis" + count = var.enable_apis ? 1 : 0 + depends_on = [data.google_project.project] +} + +## Create the service accounts for scanner and builder, and add roles +# +resource "google_service_account" "malware_scanner_sa" { + account_id = var.service_name + display_name = "Service Account for malware scanner cloud run service" + depends_on = [module.apis] +} + +resource "google_project_iam_member" "malware_scanner_iam" { + for_each = toset(["roles/monitoring.metricWriter", "roles/run.invoker", "roles/eventarc.eventReceiver"]) + project = data.google_project.project.project_id + role = each.value + member = "serviceAccount:${google_service_account.malware_scanner_sa.email}" +} + +resource "google_service_account" "build_service_account" { + account_id = "${var.service_name}-build" + display_name = "Service Account for malware scanner cloud run service" + depends_on = [module.apis] +} + +resource "google_project_iam_binding" "build_iam" { + for_each = toset(["roles/storage.objectViewer", "roles/logging.logWriter", "roles/artifactregistry.writer"]) + project = data.google_project.project.project_id + role = each.value + members = ["serviceAccount:${google_service_account.build_service_account.email}"] +} + +resource "google_artifact_registry_repository" "repo" { + location = var.region + repository_id = var.service_name + description = "Image registry for Malware Scanner" + format = "DOCKER" + depends_on = [module.apis] +} + +## Allow GCS to publish to pubsub +# +data "google_storage_project_service_account" "gcs_account" { + depends_on = [module.apis] +} +resource "google_project_iam_binding" "gcs_sa_pubsub_publish" { + project = data.google_project.project.project_id + role = "roles/pubsub.publisher" + members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"] +} + +## Create configured scanner buckets if requested. +# +module "create_buckets" { + source = "./create_buckets" + count = var.create_buckets ? 1 : 0 + bucket_location = var.bucket_location + uniform_bucket_level_access = var.uniform_bucket_level_access + bucket_names = local.all_buckets + depends_on = [module.apis] +} + +## Allow service account to admin the scanner buckets. +# +# They may not have been created by TF, so use a data resource +# to verify their existence. +# +data "google_storage_bucket" "scanner-buckets" { + for_each = local.all_buckets + name = each.value + depends_on = [module.create_buckets] +} +resource "google_storage_bucket_iam_binding" "buckets_sa_binding" { + for_each = local.all_buckets + bucket = data.google_storage_bucket.scanner-buckets[each.key].name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +## Create the CVD Mirror bucket and allow service account admin access. +# +resource "google_storage_bucket" "cvd_mirror_bucket" { + name = local.cvd_mirror_bucket + location = var.bucket_location + uniform_bucket_level_access = var.uniform_bucket_level_access + depends_on = [module.apis] +} +resource "google_storage_bucket_iam_binding" "cvd_mirror_bucket_sa_binding" { + bucket = google_storage_bucket.cvd_mirror_bucket.name + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.malware_scanner_sa.email}", + ] +} + +## Perform an update/initial load of mirror bucket. +# +resource "null_resource" "populate_cvd_mirror" { + provisioner "local-exec" { + command = join(" ; ", [ + "echo '\n\nPopulating CVD Mirror bucket ${google_storage_bucket.cvd_mirror_bucket.name}\n\n'", + "python3 -m venv pyenv", + ". pyenv/bin/activate", + "pip3 install crcmod cvdupdate", + "./updateCvdMirror.sh '${google_storage_bucket.cvd_mirror_bucket.name}'", + "echo '\n\nPopulating CVD Mirror bucket successful\n\n'", + ]) + interpreter = ["bash", "-x", "-e", "-c"] + working_dir = local.src_root + } +} diff --git a/terraform/infra/variables.tf b/terraform/infra/variables.tf new file mode 100644 index 0000000..a3cbf5f --- /dev/null +++ b/terraform/infra/variables.tf @@ -0,0 +1,56 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Google Cloud Project ID to use" + type = string +} + +variable "bucket_location" { + description = "Location to create Cloud Storage buckets" + type = string +} + +variable "region" { + description = "Region name for creation of resources" + type = string +} + +variable "service_name" { + default = "malware-scanner" + type = string +} + +variable "config_json" { + description = "String containing JSON encoded configuration to pass to the cloud run service as environment variable CONFIG_JSON" + type = string +} + +variable "enable_apis" { + description = "Automatically enable required APIs (requires that cloudresourcemanager.googleapis.com and serviceusage.googleapis.com are already enabled)" + default = true + type = bool +} + +variable "create_buckets" { + description = "Creates all the unscanned, clean, and quarantined buckets defined in the config. " + default = true + type = bool +} + +variable "uniform_bucket_level_access" { + description = "When creating cloud storage buckets, the parameter uniform_bucket_level_access is set to this value" + default = true + type = bool +} diff --git a/terraform/infra/versions.tf b/terraform/infra/versions.tf new file mode 100644 index 0000000..9452e13 --- /dev/null +++ b/terraform/infra/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.3.0" + } + } + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "cloud-solutions/gcs-malware-scanner-deploy-v3.1.0" # x-release-please-version + } +} diff --git a/terraform/service/.terraform.lock.hcl b/terraform/service/.terraform.lock.hcl new file mode 100644 index 0000000..578f220 --- /dev/null +++ b/terraform/service/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "6.3.0" + constraints = ">= 6.3.0" + hashes = [ + "h1:YWlCJbSyVtYpsGIozv8XrgcqY1nrlsChYOZZyFj17mQ=", + "zh:2d5099f33ceaaa74d6d6860b98287ffdc723a6fb3f0a48ae02e9ba11e387f8b0", + "zh:38d937bedaeb9429663eff661b3087e66d7b88b3516eb1dc895eed160283878d", + "zh:5a9a7da6e578fc8448defb87f07616f8745700bd0d17ed458909d2561c08353b", + "zh:60b047acf4e4b2140885608ac37914d10947505dd8afd2da7718af27ff0e7b64", + "zh:84d457c92224f59d6ac7d2cc43581c7fe7dd3500d47251798fed21c7fb816d64", + "zh:b12a32dac4816d5cae9aa2b491d7510160e5f157150e8f814bcc9414a68db1f0", + "zh:cd65223bf3523f3d217e4b2b8c795149be53cfc545055a34e1fbd94f252c52d9", + "zh:d55de710eab8d31da97bb6ff7e5c4fe8afee7bde84ba80ac778d74ced1b7cd74", + "zh:e817d9ec4a7ff3e096dfc1c2ea88eb6eb725ae8c827e88cb77af9905ed2e3aec", + "zh:ebaa5cdf7beb94dec01cc6a2df8b8086bedeaf40cf009ffd27b051ca3770ebc0", + "zh:f3b5177362fa3ce8835daadeea9dd10299d76db667208128d619eb0786348ba4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.3" + constraints = ">= 3.6.3" + hashes = [ + "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", + "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", + "zh:448f56199f3e99ff75d5c0afacae867ee795e4dfda6cb5f8e3b2a72ec3583dd8", + "zh:4b4c11ccfba7319e901df2dac836b1ae8f12185e37249e8d870ee10bb87a13fe", + "zh:4fa45c44c0de582c2edb8a2e054f55124520c16a39b2dfc0355929063b6395b1", + "zh:588508280501a06259e023b0695f6a18149a3816d259655c424d068982cbdd36", + "zh:737c4d99a87d2a4d1ac0a54a73d2cb62974ccb2edbd234f333abd079a32ebc9e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:a357ab512e5ebc6d1fda1382503109766e21bbfdfaa9ccda43d313c122069b30", + "zh:c51bfb15e7d52cc1a2eaec2a903ac2aff15d162c172b1b4c17675190e8147615", + "zh:e0951ee6fa9df90433728b96381fb867e3db98f66f735e0c3e24f8f16903f0ad", + "zh:e3cdcb4e73740621dabd82ee6a37d6cfce7fee2a03d8074df65086760f5cf556", + "zh:eff58323099f1bd9a0bec7cb04f717e7f1b2774c7d612bf7581797e1622613a0", + ] +} diff --git a/terraform/service/main.tf b/terraform/service/main.tf new file mode 100644 index 0000000..3c5e731 --- /dev/null +++ b/terraform/service/main.tf @@ -0,0 +1,156 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "google" { + project = var.project_id + region = var.region +} + +data "google_project" "project" { + project_id = var.project_id +} + +## Read config and extract bucket names +locals { + config_json = jsondecode(var.config_json) + unscanned_bucket_names = local.config_json.buckets[*].unscanned +} + +## Verify service account exists +# +data "google_service_account" "malware_scanner_sa" { + account_id = var.service_name + project = data.google_project.project.project_id +} + +## Lookup the hash of the latest image +## +data "google_artifact_registry_docker_image" "scanner-service-image" { + location = var.region + repository_id = var.service_name + image_name = "${var.service_name}:latest" + project = data.google_project.project.project_id +} + +## Deploy the Cloud Run Service +# +resource "google_cloud_run_v2_service" "malware_scanner" { + name = var.service_name + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + + template { + scaling { + max_instance_count = 5 + min_instance_count = 1 + } + service_account = data.google_service_account.malware_scanner_sa.email + containers { + image = data.google_artifact_registry_docker_image.scanner-service-image.self_link + resources { + limits = { + cpu = "1" + memory = "4Gi" + } + cpu_idle = false # CPU is still allocated outside of requests + startup_cpu_boost = true + } + env { + name = "CONFIG_JSON" + value = var.config_json + } + } + max_instance_request_concurrency = 20 + } + traffic { + type = "TRAFFIC_TARGET_ALLOCATION_TYPE_LATEST" + percent = 100 + } +} + +## Verify unscanned bucket(s) exist +# +data "google_storage_bucket" "unscanned-bucket" { + for_each = toset(local.unscanned_bucket_names) + name = each.value +} + +## Create EventArc Triggers on unscanned bucket(s) +# +resource "google_eventarc_trigger" "gcs-object-written" { + for_each = toset(local.unscanned_bucket_names) + name = "gcs-trigger-${each.value}" + location = var.bucket_location + matching_criteria { + attribute = "type" + value = "google.cloud.storage.object.v1.finalized" + } + matching_criteria { + attribute = "bucket" + value = data.google_storage_bucket.unscanned-bucket[each.value].name + } + destination { + cloud_run_service { + service = google_cloud_run_v2_service.malware_scanner.name + region = google_cloud_run_v2_service.malware_scanner.location + } + } + service_account = data.google_service_account.malware_scanner_sa.email +} + +## Update pubsub subscriptions to increase deadlines +# +resource "null_resource" "update-subscription-ack-deadline" { + for_each = toset(local.unscanned_bucket_names) + provisioner "local-exec" { + command = "gcloud pubsub subscriptions update \"${google_eventarc_trigger.gcs-object-written[each.key].transport[0].pubsub[0].subscription}\" --ack-deadline=120" + } +} + +## Deploy scheduled task to refresh the CVD Mirror + +# To avoid having too many clients use the same time slot, +# ClamAV requires that updates are scheduled at a random minute between 3 +# and 57 avoiding multiples of 10. +resource "random_integer" "cvd_mirror_update_schedule_minutes" { + min = 3 + max = 57 +} + +locals { + # Avoid multiples of 10 by subtracting 3. + cvd_mirror_update_schedule_minutes = ( + random_integer.cvd_mirror_update_schedule_minutes.result % 10 == 0 + ? random_integer.cvd_mirror_update_schedule_minutes.result - 3 + : random_integer.cvd_mirror_update_schedule_minutes.result + ) +} + +resource "google_cloud_scheduler_job" "cvd_mirror_update" { + name = "${var.service_name}-cvd-mirror-update" + schedule = "${local.cvd_mirror_update_schedule_minutes} */2 * * *" + attempt_deadline = "320s" + region = var.region + http_target { + http_method = "POST" + uri = google_cloud_run_v2_service.malware_scanner.uri + body = base64encode("{\"kind\":\"schedule#cvd_update\"}") + headers = { + "Content-Type" = "application/json" + } + oidc_token { + service_account_email = data.google_service_account.malware_scanner_sa.email + } + } +} diff --git a/terraform/service/outputs.tf b/terraform/service/outputs.tf new file mode 100644 index 0000000..42549c1 --- /dev/null +++ b/terraform/service/outputs.tf @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "cloud_run_uri" { + value = google_cloud_run_v2_service.malware_scanner.uri + description = "Cloud Run URI for malware-scanner service" +} diff --git a/terraform/service/variables.tf b/terraform/service/variables.tf new file mode 100644 index 0000000..5a11040 --- /dev/null +++ b/terraform/service/variables.tf @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + description = "Google Cloud Project ID to use" + type = string +} + +variable "bucket_location" { + description = "Location to create Cloud Storage buckets" + type = string +} + +variable "region" { + description = "Region name for creation of resources" + type = string +} + +variable "service_name" { + default = "malware-scanner" + type = string +} + +variable "config_json" { + description = "String containing JSON encoded configuration to pass to the cloud run service as environment variable CONFIG_JSON" + type = string +} diff --git a/terraform/service/versions.tf b/terraform/service/versions.tf new file mode 100644 index 0000000..ffbde6e --- /dev/null +++ b/terraform/service/versions.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 6.3.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.6.3" + } + } + required_version = ">= 1.5" + + provider_meta "google" { + module_name = "cloud-solutions/gcs-malware-scanner-deploy-v3.1.0" # x-release-please-version + } +}