Support new GitHub bot deployment (#15)

* initial commit

* update shared resources name patterns

* nodejs20.x + update default name pattern

* fmt

* enhance function name variable to api gateway

* improve secrets management

* example + readme

* fix example

* update readme + mvoe logic to locals

* github source code

* formatting

* update all sources codes

* update gitlab source code

* add changelog + cr changes

* update outputs & readme

* remove SPECTRAL_DSN

* rename to api_triggered_function_name

* improve role name

* add HOME to default variables

* make sure HOME is set

* lambda module take var from var

* formatting

* GH bot 2.0.4

* update date in change log

Author: ChengaDev

.gitignore

@@ -10,4 +10,5 @@ override.tf.json
 *_override.tf
 *_override.tf.json
 .terraformrc
-terraform.rc
+terraform.rc
+.DS_Store

CHANGELOG.md

@@ -0,0 +1,59 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+## [3.0.0] - 2024-06-25
+
+### Added
+
+- Support to GitHub bot 2.x deployment integration
+- Enable running multiple bot instances of the same type in a single region
+- Enable setting a custom pattern for all the resources created by the module
+- Enable setting a path to the lambda source code (Zip file)
+
+### Changed
+
+- Lambdas runtime upgraded to node20.x
+
+## [2.1.0] - 2023-08-16
+
+### Added
+
+- Support hardening & engines flag 
+
+## [2.0.0] - 2023-06-18
+
+### Changed
+
+- GitLab's integration infrastructure is now based on multiple lambda functions to make sure the response is being sent to GitLab in less than 10 seconds
+
+## [1.1.1] - 2023-05-31
+
+### Added
+
+- Option to pull the secrets required for the GitLab bot to accessed from AWS secrets manager
+
+## [1.1.0] - 2022-12-11
+
+### Changed
+
+- New versions of GitLab and TFC using new Spectral severities
+
+## [1.0.2] - 2022-10-23
+
+### Added
+
+- Support for Jira integration
+- Support for GitLab integration
+
+## [1.0.1] - 2022-10-05
+
+### Changed
+
+- Bots are now downloading the latest Spectral scanner version instead of accessing the scanner through a lambda layer
+
+## [1.0.0] - 2022-09-11
+
+### Added
+
+- Added support for Terraform cloud integration

README.md

@@ -6,42 +6,60 @@ Terraform configuration used to create the required AWS resources for integratin
 
 ## Requirements
 
-| Name      | Version |
-| ----------- | ----------- |
-| [terraform](https://www.terraform.io/downloads)      | >= 1.3.0    |
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.26.0 |
 
 ## Providers
 
-| Name      | Version |
-| ----------- | ----------- |
-| [aws](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)      | >= 4.37.0    |
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.26.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_api_gateway"></a> [api\_gateway](#module\_api\_gateway) | ./modules/api_gateway | n/a |
+| <a name="module_backend_lambda_function"></a> [backend\_lambda\_function](#module\_backend\_lambda\_function) | ./modules/lambda | n/a |
+| <a name="module_frontend_lambda_function"></a> [frontend\_lambda\_function](#module\_frontend\_lambda\_function) | ./modules/lambda | n/a |
+| <a name="module_lambda_function"></a> [lambda\_function](#module\_lambda\_function) | ./modules/lambda | n/a |
+| <a name="module_lambda_role"></a> [lambda\_role](#module\_lambda\_role) | ./modules/role | n/a |
+| <a name="module_secrets_manager"></a> [secrets\_manager](#module\_secrets\_manager) | ./modules/secrets_manager | n/a |
+
 
 ## Inputs
 
-| Name      | Description | Type | Default | Required |
-| ----------- | ----------- | ----------- | ----------- | ----------- |
-| `environment` | The target environment name for deployment | `string` | `prod` | No |
-| `integration_type` | Spectral integration type (A unique phrase describing the integration) - Available values: `terraform`, `jira` and `gitlab` | `string` | N/A | Yes |
-| [`env_vars`](#env_vars) | Extendable object contains all required environment variables required for the integration. | `map(string)` | N/A | No |
-| [`global_tags`](#global_tags) | Tags to be applied on every newly created resource. | `map(string)` | ```{ BusinessUnit = "Spectral" }``` | No |
-| [`tags`](#tags) | Tags to be applied on concrete resources  | `map(map(string))` | ```{ iam = { } lambda = { } api_gateway = { } }``` | No |
-| `lambda_enable_logs` | Specifies if Lambda should have CloudWatch a dedicated logs group. | `bool` | `false` | No |
-| `lambda_logs_retention_in_days` | Specifies the number of days you want to retain log events in the specified log group. | `number` | `30` | No |
-| `lambda_function_timeout` | Amount of time your Lambda Function has to run in seconds.  | `number` | 300 | No |
-| `lambda_function_memory_size` | Amount of memory in MB your Lambda Function can use at runtime. | `number` | 1024 | No |
-| `lambda_publish` | Whether to publish creation/change as new Lambda Function Version. | `bool` | `false` | No |
-| `store_secret_in_secrets_manager` | Whether to store secrets values on a vault (currently supporting AWS secrets manager on GitLab integration). | `bool` | `false` | No |
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_backend_lambda_source_code_path"></a> [backend\_lambda\_source\_code\_path](#input\_backend\_lambda\_source\_code\_path) | Path to the lambda source code zip file of the backend lambda | `string` | `null` | no |
+| <a name="input_env_vars"></a> [env\_vars](#input\_env\_vars) | Extendable object contains all required environment variables required for the integration. | `map(string)` | <pre>{<br>  "CHECK_POLICY": "Fail on errors only",<br>  "SPECTRAL_DSN": ""<br>}</pre> | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | The target environment name for deployment. | `string` | `"prod"` | no |
+| <a name="input_frontend_lambda_source_code_path"></a> [frontend\_lambda\_source\_code\_path](#input\_frontend\_lambda\_source\_code\_path) | Path to the lambda source code zip file of the frontend lambda | `string` | `null` | no |
+| <a name="input_gateway_api_integration_timeout_milliseconds"></a> [gateway\_api\_integration\_timeout\_milliseconds](#input\_gateway\_api\_integration\_timeout\_milliseconds) | Timeout for the API Gateway to wait for lambda response | `number` | `29000` | no |
+| <a name="input_global_tags"></a> [global\_tags](#input\_global\_tags) | A list of tags to apply on all newly created resources. | `map(string)` | <pre>{<br>  "BusinessUnit": "Spectral"<br>}</pre> | no |
+| <a name="input_integration_type"></a> [integration\_type](#input\_integration\_type) | Spectral integration type (A unique phrase describing the integration) - Available values: `github`, `terraform`, `jira` and `gitlab` | `string` | n/a | yes |
+| <a name="input_lambda_enable_logs"></a> [lambda\_enable\_logs](#input\_lambda\_enable\_logs) | Specifies if Lambda should have CloudWatch a dedicated logs group. | `bool` | `false` | no |
+| <a name="input_lambda_function_memory_size"></a> [lambda\_function\_memory\_size](#input\_lambda\_function\_memory\_size) | Amount of memory in MB your Lambda Function can use at runtime. Defaults to 1024. | `number` | `1024` | no |
+| <a name="input_lambda_function_timeout"></a> [lambda\_function\_timeout](#input\_lambda\_function\_timeout) | Amount of time your Lambda Function has to run in seconds. | `number` | `300` | no |
+| <a name="input_lambda_logs_retention_in_days"></a> [lambda\_logs\_retention\_in\_days](#input\_lambda\_logs\_retention\_in\_days) | Specifies the number of days you want to retain log events in the specified log group. | `number` | `30` | no |
+| <a name="input_lambda_publish"></a> [lambda\_publish](#input\_lambda\_publish) | Whether to publish creation/change as new Lambda Function Version. | `bool` | `false` | no |
+| <a name="input_lambda_source_code_path"></a> [lambda\_source\_code\_path](#input\_lambda\_source\_code\_path) | Path to the lambda source code zip file | `string` | `null` | no |
+| <a name="input_resource_name_common_part"></a> [resource\_name\_common\_part](#input\_resource\_name\_common\_part) | A common part for all resources created under the stack | `string` | `null` | no |
+| <a name="input_secrets_names"></a> [secrets\_names](#input\_secrets\_names) | Names of secrets to create | `list(string)` | `null` | no |
+| <a name="input_store_secret_in_secrets_manager"></a> [store\_secret\_in\_secrets\_manager](#input\_store\_secret\_in\_secrets\_manager) | Whether to store your secrets in secrets manager, default is false | `bool` | `false` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | A collection of tags grouped by key representing it's target resource. | `map(map(string))` | <pre>{<br>  "api_gateway": {},<br>  "iam": {},<br>  "lambda": {}<br>}</pre> | no |
 
 ### env_vars
 
-In some integrations, Spectral requires some extra environment variables besides the default ones.
-Those extra variables should be added to the `env_vars` map in addition to `SPECTRAL_DSN` which is mandatory.
-
-Please refer to our docs / source pages to view the extra environment variables needed for the integration.
+In some integrations, Spectral requires some environment variables besides the default ones.
+Those variables should be added to the `env_vars`.
 
-##### SPECTRAL_DSN (mandatory)
+Please refer to our [docs](https://guides.spectralops.io/docs/welcome-to-checkpoint-cloudguard-guides) / source pages to view the extra environment variables needed for the integration.
 
-Your SpectralOps identifier, retrieved from your SpectralOps account.
 
 ### global_tags
 
@@ -142,13 +160,20 @@ module "spectral_lambda_integration" {
 
 ### This module has the following outputs
 
-1. `rest_api_id` - The ID of the REST API.
-2. `rest_api_url` - The URL for accessing the lambda through the ApiGateway.
-3. `rest_api_arn` - Amazon Resource Name (ARN) identifying your Rest API.
-4. `rest_api_execution_arn` - The execution ARN part to be used in lambda_permission's source_arn, not concatenated to other allowed API resources.
-5. `rest_api_lambda_execution_arn` - The execution ARN part to be used in lambda_permission's source_arn, concatenated with allowed API resources (method & path).
-6. `lambda_function_arn` - Amazon Resource Name (ARN) identifying your Lambda Function.
-7. `lambda_function_name` - The name of the lambda function.
-8. `lambda_iam_role_arn` - Amazon Resource Name (ARN) specifying the role.
-9. `lambda_iam_role_name` - Name of the role.
-10. `secrets_arns` - Arns of created secrets in secrets manager.
+| Name | Description |
+|------|-------------|
+| <a name="output_lambda_function_arn"></a> [lambda\_function\_arn](#output\_lambda\_function\_arn) | Amazon Resource Name (ARN) identifying your Lambda Function |
+| <a name="output_lambda_function_name"></a> [lambda\_function\_name](#output\_lambda\_function\_name) | The name of the lambda function |
+| <a name="output_lambda_iam_role_arn"></a> [lambda\_iam\_role\_arn](#output\_lambda\_iam\_role\_arn) | Amazon Resource Name (ARN) specifying the role |
+| <a name="output_lambda_iam_role_name"></a> [lambda\_iam\_role\_name](#output\_lambda\_iam\_role\_name) | Name of the role |
+| <a name="output_rest_api_arn"></a> [rest\_api\_arn](#output\_rest\_api\_arn) | Amazon Resource Name (ARN) identifying your Rest API |
+| <a name="output_rest_api_execution_arn"></a> [rest\_api\_execution\_arn](#output\_rest\_api\_execution\_arn) | The execution ARN part to be used in lambda\_permission's source\_arn, not concatenated to other allowed API resources |
+| <a name="output_rest_api_id"></a> [rest\_api\_id](#output\_rest\_api\_id) | The ID of the REST API |
+| <a name="output_rest_api_lambda_execution_arn"></a> [rest\_api\_lambda\_execution\_arn](#output\_rest\_api\_lambda\_execution\_arn) | The execution ARN part to be used in lambda\_permission's source\_arn, concatenated with allowed API resources (method & path) |
+| <a name="output_rest_api_url"></a> [rest\_api\_url](#output\_rest\_api\_url) | The URL for accessing the lambda through the ApiGateway |
+| <a name="output_secrets_arns"></a> [secrets\_arns](#output\_secrets\_arns) | Arns of created secrets in secrets manager |
+
+## Support
+
+For GitHub deployment - only bot version 2.x is supported.
+The default GitHub bot version that this module deploys is 2.0.4, if you wish to use other versions please set local paths to the relevant ZIP files.

examples/basic-github-integration.tf

@@ -0,0 +1,31 @@
+module "spectral_lambda_integration" {
+  source = "github.com/SpectralOps/spectral-terraform-lambda-integration"
+
+  integration_type   = "github"
+  lambda_enable_logs = true
+
+  # Use this attributes to deploy specific version of the bot
+  frontend_lambda_source_code_path = "./source-code/github/github-frontend.zip"
+  backend_lambda_source_code_path  = "./source-code/github/github-backend.zip"
+
+  env_vars = {
+    # Required environment variables    
+    SPECTRAL_DSN          = "MySpectralDSN"
+    CHECK_POLICY          = "Fail on any issue" # (Fail on any issue / Fail on warnings and above / Fail on errors only / Always Pass)
+    GITHUB_APP_ID         = "MyGitHubAppId"
+    GITHUB_WEBHOOK_SECRET = "MyGitHubWebhookSecret"
+    GITHUB_PRIVATE_KEY    = "MyGitHubPrivateKey"
+    # Optional environment variables
+    SECRETS_VAULT                      = "aws_secrets_manager"
+    VAULT_KEY_SPECTRAL_DSN             = "Spectral_Dsn-..."
+    VAULT_KEY_GITHUB_WEBHOOK_SECRET    = "Spectral_GithubBot_WebhookSecret-..."
+    VAULT_KEY_GITHUB_PRIVATE_KEY       = "Spectral_GithubBot_PrivateKey-..."
+    GITHUB_SHOULD_POST_REVIEW_COMMENTS = false
+    GITHUB_SHOULD_SKIP_CHECK           = false
+    S3_BLACK_LIST_OBJECT_KEY           = "The S3 object key of your blacklist flie"
+    S3_BLACK_LIST_BUCKET_NAME          = "The S3 bucket name that holds your blacklist file"
+    SHOULD_SKIP_INGEST                 = false
+    STRICT_MODE                        = false
+    SPECTRAL_TAGS                      = "iac,base,audit"
+  }
+}

locals.tf

@@ -1,6 +1,17 @@
 locals {
-  resource_name_pattern       = "spectral-${var.integration_type}-integration-${var.environment}"
-  single_lambda_integration   = contains(["jira", "terraform"], var.integration_type) ? true : false
-  multiple_lambda_integration = contains(["gitlab"], var.integration_type) ? true : false
-  api_triggered_function_arn  = local.single_lambda_integration ? module.lambda_function[0].lambda_function_arn : module.frontend_lambda_function[0].lambda_function_arn
+  resource_name_pattern        = coalesce(var.resource_name_common_part, "spectral-${var.integration_type}-integration-${var.environment}-${random_string.random_resource_name_suffix.id}")
+  single_lambda_integration    = contains(["jira", "terraform"], var.integration_type) ? true : false
+  multiple_lambda_integration  = contains(["gitlab", "github"], var.integration_type) ? true : false
+  api_triggered_function_arn   = local.single_lambda_integration ? module.lambda_function[0].lambda_function_arn : module.frontend_lambda_function[0].lambda_function_arn
+  frontend_lambda_handler      = contains(["github"], var.integration_type) ? "index.handler" : "frontend.app"
+  backend_lambda_handler       = contains(["github"], var.integration_type) ? "index.handler" : "backend.app"
+  shared_default_secrets_names = ["Spectral_Dsn"]
+  default_secrets_names = {
+    "github" = coalesce(var.secrets_names, concat(local.shared_default_secrets_names, ["Spectral_GithubBot_PrivateKey", "Spectral_GithubBot_WebhookSecret"])),
+    "gitlab" = coalesce(var.secrets_names, concat(local.shared_default_secrets_names, ["Spectral_GitlabBot_GitlabToken", "Spectral_GitlabBot_WebhookSecret"]))
+  }
+  # Please do not change or replace the 'frontend' suffix since there a logic in the bot based in it
+  api_triggered_function_name = local.single_lambda_integration ? local.resource_name_pattern : "${local.resource_name_pattern}-frontend"
+  # Merge user env vars with env vars which are not based on user input
+  env_vars = merge(var.env_vars, { HOME = "/tmp" })
 }

modules/api_gateway/rest_api.tf

@@ -31,6 +31,7 @@ resource "aws_api_gateway_integration" "api_proxy_integration" {
   integration_http_method = "POST"
   type                    = "AWS_PROXY"
   uri                     = var.lambda_function_arn
+  timeout_milliseconds    = var.gateway_api_integration_timeout_milliseconds
 }
 
 resource "aws_api_gateway_method_response" "response_200" {
@@ -43,7 +44,7 @@ resource "aws_api_gateway_method_response" "response_200" {
 resource "aws_lambda_permission" "lambda_permission" {
   statement_id  = "AllowExecutionFromAPIGateway"
   action        = "lambda:InvokeFunction"
-  function_name = var.resource_name_pattern
+  function_name = var.function_name
   principal     = "apigateway.amazonaws.com"
   source_arn    = local.rest_api_execution_arn
 }

modules/api_gateway/variables.tf

@@ -33,5 +33,16 @@ variable "tags" {
 
 variable "integration_type" {
   type        = string
-  description = "Spectral integration type (A unique phrase describing the integration) - Available values: `terraform`."
+  description = "Spectral integration type (A unique phrase describing the integration) - Available values: `github`, `terraform`, `jira` and `gitlab`"
+}
+
+variable "function_name" {
+  type        = string
+  description = "The name of the function the API would trigger upon request"
+}
+
+variable "gateway_api_integration_timeout_milliseconds" {
+  description = "Timeout for the API Gateway to wait for lambda response"
+  type        = number
+  default     = 29000
 }

modules/lambda/lambda.tf

@@ -1,6 +1,6 @@
 locals {
-  runtime                     = "nodejs14.x"
-  lambda_source_code_zip_path = "${path.module}/source_code/${var.integration_type}/${var.lambda_source_code_filename}"
+  runtime                     = "nodejs20.x"
+  lambda_source_code_zip_path = coalesce(var.lambda_source_code_path, "${path.module}/source_code/${var.integration_type}/${var.lambda_source_code_filename}")
 }
 
 resource "aws_lambda_function" "spectral_scanner_lambda" {

modules/lambda/source_code/github/backend.zip

@@ -44,7 +44,7 @@ variable "tags" {
 
 variable "integration_type" {
   type        = string
-  description = "Spectral integration type (A unique phrase describing the integration) - Available values: `terraform`."
+  description = "Spectral integration type (A unique phrase describing the integration) - Available values: `github`, `terraform`, `jira` and `gitlab`"
 }
 
 variable "timeout" {
@@ -71,16 +71,16 @@ variable "secrets_arns" {
   default     = []
 }
 
-variable "store_secret_in_secrets_manager" {
-  description = "Whether to store your secrets in secrets manager, default is false"
-  type        = bool
-}
-
 variable "lambda_source_code_filename" {
   type        = string
   description = "The lambda source code filename"
 }
 
+variable "lambda_source_code_path" {
+  type        = string
+  description = "The lambda source code path"
+}
+
 variable "role_arn" {
   type        = string
   description = "The lambda source code filename"

modules/lambda/source_code/github/frontend.zip

@@ -13,7 +13,7 @@ data "aws_iam_policy_document" "assume_role_policy" {
 }
 
 resource "aws_iam_role" "lambda_execution_role" {
-  name               = var.resource_name_pattern
+  name               = var.role_name
   assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
 
   tags = merge(

modules/lambda/source_code/gitlab/backend.zip

@@ -27,9 +27,9 @@ variable "tags" {
   }
 }
 
-variable "resource_name_pattern" {
+variable "role_name" {
   type        = string
-  description = "A common resource name created by pattern."
+  description = "The name of the role"
 }
 
 variable "multiple_lambda_integration" {

modules/lambda/source_code/gitlab/frontend.zip

@@ -1,15 +1,8 @@
 locals {
-  secrets_arns = concat(
-    try(module.gitlab[0].secrets_arns, []),
-    [aws_secretsmanager_secret.spectral_dsn.arn]
-  )
+  secrets_arns = [for secret in aws_secretsmanager_secret.general_secret : secret.arn]
 }
 
-resource "aws_secretsmanager_secret" "spectral_dsn" {
-  name = "Spectral_Dsn"
-}
-
-module "gitlab" {
-  count  = var.integration_type == "gitlab" ? 1 : 0
-  source = "./gitlab"
+resource "aws_secretsmanager_secret" "general_secret" {
+  count = length(var.secrets_names)
+  name  = var.secrets_names[count.index]
 }