diff --git a/.github/workflows/trivy-cache.yml b/.github/workflows/trivy-cache.yml new file mode 100644 index 0000000000..b08d708fe9 --- /dev/null +++ b/.github/workflows/trivy-cache.yml @@ -0,0 +1,38 @@ +# In your scan workflow, set TRIVY_SKIP_DB_UPDATE=true and TRIVY_SKIP_JAVA_DB_UPDATE=true. +name: Update Trivy Cache + +on: + schedule: + - cron: '0 0 * * *' # Run daily at midnight UTC + workflow_dispatch: # Allow manual triggering + +jobs: + update-trivy-db: + runs-on: ubuntu-latest + steps: + - name: Setup oras + uses: oras-project/setup-oras@v1 + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Download and extract the vulnerability DB + run: | + mkdir -p $GITHUB_WORKSPACE/.cache/trivy/db + oras pull ghcr.io/aquasecurity/trivy-db:2 + tar -xzf db.tar.gz -C $GITHUB_WORKSPACE/.cache/trivy/db + rm db.tar.gz + + - name: Download and extract the Java DB + run: | + mkdir -p $GITHUB_WORKSPACE/.cache/trivy/java-db + oras pull ghcr.io/aquasecurity/trivy-java-db:1 + tar -xzf javadb.tar.gz -C $GITHUB_WORKSPACE/.cache/trivy/java-db + rm javadb.tar.gz + + - name: Cache DBs + uses: actions/cache/save@v4 + with: + path: ${{ github.workspace }}/.cache/trivy + key: cache-trivy-${{ steps.date.outputs.date }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index d9c9244264..f275f3c72d 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -20,6 +20,7 @@ jobs: security-events: write actions: read env: + repo_name: gsa-tts/fac DOCKER_NAME: fac WORKING_DIRECTORY: ./backend name: Trivy Scan FAC Web Container @@ -40,6 +41,11 @@ jobs: - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.28.0 + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db,ghcr.io/aquasecurity/trivy-db + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db,ghcr.io/aquasecurity/trivy-java-db + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true with: image-ref: '${{ env.DOCKER_NAME }}:${{ steps.date.outputs.date }}' scan-type: 'image' @@ -63,6 +69,8 @@ jobs: actions: read name: Trivy Scan Third Party Images runs-on: ubuntu-latest + env: + repo_name: gsa-tts/fac strategy: fail-fast: false matrix: @@ -75,6 +83,11 @@ jobs: - name: Run Trivy vulnerability scanner on Third Party Images uses: aquasecurity/trivy-action@0.28.0 + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db,ghcr.io/aquasecurity/trivy-db + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db,ghcr.io/aquasecurity/trivy-java-db + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true with: image-ref: '${{ matrix.image.name }}' scan-type: 'image' diff --git a/.gitignore b/.gitignore index 5330333dd8..b1fb5fbcf3 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,14 @@ terraform/**/*.tfstate* terraform/**/*.tfvars terraform/shared/modules/egress-proxy/proxy.zip terraform/shared/modules/egress-proxy/test/client.zip +terraform/shared/modules/app/app.zip +terraform/shared/modules/https-proxy/proxy.zip +terraform/shared/modules/https-proxy/.terraform.lock.hcl +terraform/shared/modules/stream-proxy/proxy.zip +terraform/shared/modules/https-proxy/.terraform.lock.hcl +terraform/shared/modules/sandbox-proxy/proxy.zip +terraform/shared/modules/sandbox-proxy/.terraform.lock.hcl + # XLSX ignores .~*# diff --git a/backend/.profile b/backend/.profile index 75952618ad..544a2ee1cc 100644 --- a/backend/.profile +++ b/backend/.profile @@ -1,9 +1,9 @@ #!/bin/bash # Source everything; everything is now a function. -# Remember: bash has no idea if a function exists, +# Remember: bash has no idea if a function exists, # so a typo in a function name will fail silently. Similarly, -# bash has horrible scoping, so use of `local` in functions is +# bash has horrible scoping, so use of `local` in functions is # critical for cleanliness in the startup script. source tools/util_startup.sh # This will choose the correct environment @@ -14,6 +14,7 @@ source tools/migrate_app_tables.sh source tools/api_standup.sh source tools/run_collectstatic.sh source tools/seed_cog_baseline.sh +source tools/materialized_views.sh ##### # SETUP THE CGOV ENVIRONMENT @@ -43,15 +44,18 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then ##### # COLLECT STATIC # Do Django things with static files. - run_collectstatic - gonogo "run_collectstatic" + # run_collectstatic + # gonogo "run_collectstatic" ##### # SEED COG/OVER TABLES # Setup tables for cog/over assignments seed_cog_baseline gonogo "seed_cog_baseline" + + # materialized_views + # gonogo "materialized_views" fi # Make psql usable by scripts, for debugging, etc. -alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' \ No newline at end of file +alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' diff --git a/backend/config/settings.py b/backend/config/settings.py index ce06cdf602..c05f8b3d00 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -227,8 +227,7 @@ # Environment specific configurations DEBUG = False - -if ENVIRONMENT not in ["DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: +if ENVIRONMENT not in ["SANDBOX", "DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: DATABASES = { "default": env.dj_db_url( "DATABASE_URL", default="postgres://postgres:password@0.0.0.0/backend" diff --git a/backend/tools/materialized_views.sh b/backend/tools/materialized_views.sh new file mode 100755 index 0000000000..be5fe18e01 --- /dev/null +++ b/backend/tools/materialized_views.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function materialized_views { + startup_log "RUN_MATERIALIZEDVIEWS" "BEGIN" + python manage.py materialized_views --create && + local result=$? + startup_log "RUN_MATERIALIZEDVIEWS" "END" + return $result +} diff --git a/bin/ops/create_service_account.sh b/bin/ops/create_service_account.sh old mode 100644 new mode 100755 index 71fc6b3bfa..779c08acf2 --- a/bin/ops/create_service_account.sh +++ b/bin/ops/create_service_account.sh @@ -70,4 +70,4 @@ cat << EOF cf_user = $username cf_password = $password -EOF \ No newline at end of file +EOF diff --git a/bin/ops/destroy_service_account.sh b/bin/ops/destroy_service_account.sh old mode 100644 new mode 100755 index 85618a8713..3ae3137c32 --- a/bin/ops/destroy_service_account.sh +++ b/bin/ops/destroy_service_account.sh @@ -50,4 +50,4 @@ cf target -o $org -s $space cf delete-service-key $service ${service}-key -f # destroy service -cf delete-service $service -f \ No newline at end of file +cf delete-service $service -f diff --git a/bin/ops/get_service_account.sh b/bin/ops/get_service_account.sh new file mode 100755 index 0000000000..8f34e6b198 --- /dev/null +++ b/bin/ops/get_service_account.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +org="gsa-tts-oros-fac" + +usage=" +$0: Create a Service User Account for a given space + +Usage: + $0 -h + $0 -s -u [-r ] [-o ] + +Options: +-h: show help and exit +-s : configure the space to act on. Required +-u : set the service user name. Required +-r : set the service user's role to either space-deployer or space-auditor. Default: space-deployer +-o : configure the organization to act on. Default: $org +" + +set -e +set -o pipefail + +space="" +service="" +role="space-deployer" + +while getopts ":hs:u:r:o:" opt; do + case "$opt" in + s) + space=${OPTARG} + ;; + u) + service=${OPTARG} + ;; + r) + role=${OPTARG} + ;; + o) + org=${OPTARG} + ;; + h) + echo "$usage" + exit 0 + ;; + esac +done + +if [[ $space = "" || $service = "" ]]; then + echo "$usage" + exit 1 +fi + +>&2 echo "Targeting org $org and space $space" +cf target -o $org -s $space > /dev/null 2>&1 + +# get service key +cf service-key $service ${service}-key > /dev/null 2>&1 + +# output service key to stdout in secrets.auto.tfvars format +creds=`cf service-key $service ${service}-key | tail -n 7` +username=`echo $creds | jq '.credentials.username'` +password=`echo $creds | jq '.credentials.password'` + +cat < Resources > WSL2 Integration > Ubuntu (enable) + +Once this happens, ensure that you fully reboot your computer. Once restarted, open Docker Desktop, and open your IDE of choice (VSCode for example.) When you select a new terminal window, you will see Ubuntu(WSL). Open that terminal and run +``` +sudo apt update +sudo apt full-upgrade -y +``` + +If it fails to upgrade packages, run the following commands (Ref: [Stack Overflow Page](https://stackoverflow.com/a/63578387)) +``` +netsh winsock reset +netsh int ip reset all +netsh winhttp reset proxy +ipconfig /flushdns +``` +And restart your computer once more. + +# Core Dependencies + +Next install cf-cli v8 on WSL2 Ubuntu or Linux: +```bash +# ...first add the Cloud Foundry Foundation public key and package repository to your system +wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - +echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list +# ...then, update your local package index, then finally install the cf CLI +sudo apt-get update +sudo apt-get install cf8-cli +``` + +Next install AWS CLI on WSL2 Ubuntu or Linux: +```bash +sudo snap install aws-cli --classic + +# aws doesnt work with an x509 error? This fixed it on my WSL2 Ubuntu +pip install --upgrade cryptography==36.0.2 +``` + +Next install terraform on WSL2 Ubuntu or Linux: +```bash +sudo snap install terraform --classic +``` + +Finally, install necessary requirements: +``` +sudo apt install jq +sudo apt install npm +sudo apt install zip +``` + +# Obtain access to the sandbox environment +Ping Alex on slack for access to the sandbox environment. +``` +cf set-space-role gsa-tts-oros-fac sandbox SpaceDeveloper +``` + +# Get a deployer account credentials +A deployer account will be provided. +```bash +cd terraform/sandbox +../../bin/ops/get_service_account.sh -o gsa-tts-oros-fac -s sandbox -u sandbox-deployer >> secrets.auto.tfvars +``` +This will update a `secrets.auto.tfvars` in the directory for use with terraform. + +# Pre-Configuration + +Since this is operating under the belief that you do not have a `shared/config/sandbox.tfvars` file, a helper script in `terraform/sandbox` has been provided. This will create a secrets file that we will edit before attempting a deployment. +```bash +cd terraform/sandbox/helper_scripts/ +./create_tfvars.sh +``` + +```t +branch_name = "" # There is no default branch. This value should be the name of your branch. (ex / [as/terraform-work]) +new_relic_license_key = "" +pgrst_jwt_secret = "" +sam_api_key = "" +login_client_id = "" +login_secret_key = "" +django_secret_login_key = "" +``` + +Next, navigate to `terraform/shared/config` and create a `backend.tfvars` file. This will be used for the s3 partial configuration to store the terraform state. This will alleviate storing the `terraform.tfstate` on any one local file system, and when you run `terraform init` for the first time, it will read from the state, and inform terraform what resources already exist. + +Ensure that you are in the sandox space with `cf t -s sandbox` and populate these values from the output of `cf service-key sandbox-terraform-state sandbox-terraform-state-key`. This is a manually created s3 bucket, outside of terraforms knowledge, for the explicit purpose of storing the `terraform.tfstate`. +```t +bucket = "" # bucket value (non friendly name) +key = "/" +region = "us-gov-west-1" +access_key = "" # access_key_id value +secret_key = "" # secret_access_key value + +``` + +Navigate to `terraform/shared/modules/sandbox-proxy`, and run the following two commands. You'll be prompted for values; hit enter to leave them blank. When you are done, you should have `proxy.zip` in your current folder. + +```bash +terraform init +``` +```bash +terraform plan +``` +We do not care about actually making changes with terraform, only generating a `proxy.zip` for the modules to reference. + +# First Deployment +Due to the incorporation of partial s3 configuration, the `terraform.tfstate` will be stored in the s3 bucket. Due to this, when you run `./init.sh` and `./plan.sh` for the first time, you may not find a "clean" environment. You can, should you choose, run `terraform/sandbox/helper_scripts/destroy.sh` after running `terraform/sandbox/helper_scripts/init.sh`. This script will clean out both the `fac-private-s3` and `fac-public-s3` buckets, and then tear down the environment completely. It is completely safe to do this should you choose, and want to work in an immutable environment where you are certain every resource has been created from scratch. The catch to this, is a longer deployment time, as well as loss of data. This module was designed to have zero data at any given time, and should not be considered safe for storage regarding submissions, database entries, pdfs, excels, or staticfiles in `fac-private-s3`, `fac-public-s3`, `fac-db` or `fac-snapshot-db`. If you want to have reusable data, it is highly encouraged to have a repeatable way to load data into the database. + +If resources are already existing, and you wish to view changes on your branch, see below. + +Make sure you are in the sandbox environment. +```bash +cf t -s sandbox +``` + +Navigate to `terraform/sandbox/helper_scripts` and then run the `./init.sh` script. +This assumes you have a `sandbox.tfvars` and `backend.tfvars` in `terraform/shared/config/` from previous steps. + +Next, run `./plan.sh` script. You should see it creating ~20 resources in a clean environment, or updating a few. +Finally, run `./apply.sh` script and wait. + +# What is missing/omitted from Sandbox +The following resources were intentionally left out from the sandbox environment. Part of that is the lack of necessity for such things, and an attempt to have a more lightweight environment, that only runs the bare minimum to bring up the system. +- Logshipper Module +- smtp-proxy Module +- fac-file-scanner Module +- backups s3 bucket +- Connection to newrelic (though this might just be a bug/misconfiguration, we don't really care if this env reports to new relic) +- Full implementation of postgrest and API + - This is subject to a data.gov key, and still being considered. Functionally, everything works as expected, but we cannot use the api without a valid `sam_api_key` and configuration. + - WIP? TBD? Ask Matt. + +# Discoveries: +- It was discovered that the compiled css assets in the public s3 must be in the `backend/static/` folder when collectstatic is being run. Due to this, when it is run via github actions.. the `/static/compiled/` folder exists on the local file system, since the github runner does these steps, and handles keeping them in the local file system. To mitigate this for the terraform, we handle this in the `prepare_app.sh` script. +- When registering the application with login.gov, perform the following: +```bash +# Generate a public and private key +openssl req -nodes -x509 -days 365 -newkey rsa:2048 -keyout private.pem -out public.crt + +# Upload the public.crt to login.gov application page + +# Convert the private key to base64 +cat private.pem | base64 -w 0 > django_key.txt + +# The value in the django_key.txt will be your DJANGO_SECRET_LOGIN_KEY for shared/modules/config/sandbox.tfvars +``` + +# Investigate Further: +- It appears, that after getting the successful generation of the `compiled` staticfiles, we are running into an issue where the boot sequence collectstatic is taking > 3 minutes to process. I would imagine that under normal sequences, there are no updates, so it simply skips over everything. But, if for some reason it is actually processing and uploading them to the s3 bucket every time, this operation is causing us to trigger the "3 minute timeout" on cloud.gov. Since an operation is "hanging" and the health check fails due to the 3 minutes, it gets caught in a crash loop. + +# Documentation Section + +## Give the deployer account permissions in the ${ENV}-egress space + +*This has already been done. This section is for documentation only.* + +You must have a role of SpaceManager to assign the deployer service account. Ask Alex, Bret or Matt to do this for you (but the assumption is it will be done for you. The space will only have this service account and will not need to be done more than once, unless the account gets deleted from the sandbox space) +```bash +cf space-users gsa-tts-oros-fac sandbox +cf set-space-role gsa-tts-oros-fac sandbox-egress SpaceDeveloper +``` + +## ASGS + +*This has already been done. This section is for documentation only.* + +Since we do not rely on the github meta workflow (yet) to handle ASGS, we must explictly ensure they are there. This operation will only need to be done once. + +Security Groups: +```bash +cf bind-security-group trusted_local_networks gsa-tts-oros-fac --lifecycle running --space sandbox +cf bind-security-group trusted_local_networks_egress gsa-tts-oros-fac --lifecycle running --space sandbox + +cf bind-security-group public_networks_egress gsa-tts-oros-fac --lifecycle running --space sandbox-egress +``` diff --git a/terraform/sandbox/helper_scripts/apply.sh b/terraform/sandbox/helper_scripts/apply.sh new file mode 100755 index 0000000000..f2dce6e02c --- /dev/null +++ b/terraform/sandbox/helper_scripts/apply.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e +dir=$(pwd) +cd .. +terraform apply \ + sandbox.tfplan +cd "$dir" diff --git a/terraform/sandbox/helper_scripts/create_tfvars.sh b/terraform/sandbox/helper_scripts/create_tfvars.sh new file mode 100755 index 0000000000..0fb9be7a98 --- /dev/null +++ b/terraform/sandbox/helper_scripts/create_tfvars.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +DIRECTORY="../../shared/config/" +FILE="../../shared/config/sandbox.tfvars" +if [ ! -d "$DIRECTORY" ]; then + echo "$DIRECTORY does not exist. Creating.." + mkdir $DIRECTORY + else + echo "$DIRECTORY exists." + if [ -f "$FILE" ]; then + echo $FILE "exists, aborting. Update $FILE with the environments secrets." + exit 1 + else + echo $FILE "does not exist. Creating..." + fi +fi +read -p "Enter the name of your working branch: " branch + +cat > $FILE << EOM +# Generated with ./create_tfvars.sh +# You are responsible for populating the secrets. Please reference the drive doc for the sandbox secrets! + +branch_name = "$branch" +new_relic_license_key = "" +pgrst_jwt_secret = "" +sam_api_key = "" +login_client_id = "" +login_secret_key = "" +django_secret_login_key = "" +EOM diff --git a/terraform/sandbox/helper_scripts/destroy.sh b/terraform/sandbox/helper_scripts/destroy.sh new file mode 100755 index 0000000000..30bc9280b9 --- /dev/null +++ b/terraform/sandbox/helper_scripts/destroy.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +# Force the user of the script into the sanbox environment. +# This way, we don't accidentally operate on something else. +cf t -s sandbox + +dir=$(pwd) +cd .. +DestroyPublicS3() { + PUBLIC_S3=fac-public-s3; + PUBLIC_KEY_NAME=fac-public-s3-key; + cf create-service-key "${PUBLIC_S3}" "${PUBLIC_KEY_NAME}"; + echo "Sleeping for CF API" + sleep 10 + S3_CREDENTIALS=$(cf service-key "${PUBLIC_S3}" "${PUBLIC_KEY_NAME}" | tail -n +2); + export AWS_ACCESS_KEY_ID="$(echo "$S3_CREDENTIALS" | jq -r .credentials.access_key_id)"; + export AWS_SECRET_ACCESS_KEY="$(echo "$S3_CREDENTIALS" | jq -r .credentials.secret_access_key)"; + export BUCKET="$(echo "$S3_CREDENTIALS" | jq -r .credentials.bucket)"; + export AWS_DEFAULT_REGION="$(echo "$S3_CREDENTIALS" | jq -r .credentials.region)"; + aws s3 rm s3://$BUCKET/ --recursive + cf delete-service-key -f "${PUBLIC_S3}" "${PUBLIC_KEY_NAME}"; + echo "Sleeping for CF API" + sleep 10 +} + +DestroyPrivateS3() { + PRIVATE_S3=fac-private-s3; + PRIVATE_KEY_NAME=fac-private-s3-key; + cf create-service-key "${PRIVATE_S3}" "${PRIVATE_KEY_NAME}"; + echo "Sleeping for CF API" + sleep 10 + S3_CREDENTIALS=$(cf service-key "${PRIVATE_S3}" "${PRIVATE_KEY_NAME}" | tail -n +2); + export AWS_ACCESS_KEY_ID="$(echo "$S3_CREDENTIALS" | jq -r .credentials.access_key_id)"; + export AWS_SECRET_ACCESS_KEY="$(echo "$S3_CREDENTIALS" | jq -r .credentials.secret_access_key)"; + export BUCKET="$(echo "$S3_CREDENTIALS" | jq -r .credentials.bucket)"; + export AWS_DEFAULT_REGION="$(echo "$S3_CREDENTIALS" | jq -r .credentials.region)"; + aws s3 rm s3://$BUCKET/ --recursive + cf delete-service-key -f "${PRIVATE_S3}" "${PRIVATE_KEY_NAME}"; + echo "Sleeping for CF API" + sleep 10 +} + +TerraformDestroy() { + terraform plan \ + -var-file="../shared/config/sandbox.tfvars" \ + -out sandbox-destroy.tfplan \ + -destroy + + terraform apply sandbox-destroy.tfplan +} + +echo "Deleting contents of fac-public-s3" +DestroyPublicS3 +echo "Deleting contents of fac-private-s3" +DestroyPrivateS3 +echo "Performing Terraform Destroy" +TerraformDestroy + +cd "$dir" diff --git a/terraform/sandbox/helper_scripts/init.sh b/terraform/sandbox/helper_scripts/init.sh new file mode 100755 index 0000000000..7fe490e9e5 --- /dev/null +++ b/terraform/sandbox/helper_scripts/init.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# The content of this file is managed by Terraform. If you modify it, it may +# be reverted the next time Terraform runs. If you want to make changes, do it +# in ../meta/bootstrap-env/templates. + +set -e + +# Force the user of the script into the sanbox environment. +# This way, we don't accidentally operate on something else. +cf t -s sandbox + +dir=$(pwd) +cd .. +basename=$(basename "$(pwd)") +terraform init \ + --backend-config=../shared/config/backend.tfvars \ + --backend-config=key=terraform.tfstate.$basename + +cd "$dir" diff --git a/terraform/sandbox/helper_scripts/plan.sh b/terraform/sandbox/helper_scripts/plan.sh new file mode 100755 index 0000000000..9ff880b3e8 --- /dev/null +++ b/terraform/sandbox/helper_scripts/plan.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +dir=$(pwd) +cd .. +terraform plan \ + -var-file="../shared/config/sandbox.tfvars" \ + -out sandbox.tfplan + +cd "$dir" diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf new file mode 100644 index 0000000000..cb0688050d --- /dev/null +++ b/terraform/sandbox/providers.tf @@ -0,0 +1,25 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~>0.53.1" + } + } + + backend "s3" { + # The rest of the backend parameters must be supplied when you initialize: + # terraform init --backend-config=../shared/config/backend.tfvars \ + # --backend-config=key=terraform.tfstate.$(basename $(pwd)) + # + # For more info, see: + # https://developer.hashicorp.com/terraform/language/settings/backends/configuration#partial-configuration + encrypt = "true" + } +} + +provider "cloudfoundry" { + api_url = "https://api.fr.cloud.gov" + user = var.cf_user + password = var.cf_password +} diff --git a/terraform/sandbox/sandbox.tf b/terraform/sandbox/sandbox.tf new file mode 100644 index 0000000000..498c2756eb --- /dev/null +++ b/terraform/sandbox/sandbox.tf @@ -0,0 +1,19 @@ +module "sandbox" { + source = "../shared/modules/sandbox" + cf_space_name = "sandbox" + pgrst_jwt_secret = var.pgrst_jwt_secret + new_relic_license_key = var.new_relic_license_key + django_secret_login_key = var.django_secret_login_key + sam_api_key = var.sam_api_key + login_client_id = var.login_client_id + login_secret_key = var.login_secret_key + branch_name = var.branch_name + + database_plan = "medium-gp-psql" + https_proxy_instances = 1 + json_params = jsonencode( + { + "storage" : 50, + } + ) +} diff --git a/terraform/sandbox/variables.tf b/terraform/sandbox/variables.tf new file mode 100644 index 0000000000..2af0d9d445 --- /dev/null +++ b/terraform/sandbox/variables.tf @@ -0,0 +1,45 @@ +variable "cf_org_name" { + type = string + description = "name of the organization to configure" + default = "gsa-tts-oros-fac" +} + +variable "cf_user" { + type = string + description = "cloud.gov deployer account user" +} + +variable "cf_password" { + type = string + description = "secret; cloud.gov deployer account password" + sensitive = true +} + +variable "pgrst_jwt_secret" { + type = string + description = "the JWT signing secret for validating JWT tokens from api.data.gov" +} + +variable "new_relic_license_key" { + type = string + description = "the license key to use when setting up the New Relic agent" +} + +variable "sam_api_key" { + type = string +} +variable "django_secret_login_key" { + type = string +} +variable "login_client_id" { + type = string +} +variable "login_secret_key" { + type = string +} + +variable "branch_name" { + type = string + description = "the heads value for the branch you wish to deploy (default would be main)" + # We don't specify a default here because we want to specify a branch to deploy +} diff --git a/terraform/shared/modules/app/app.tf b/terraform/shared/modules/app/app.tf new file mode 100644 index 0000000000..49762d95da --- /dev/null +++ b/terraform/shared/modules/app/app.tf @@ -0,0 +1,120 @@ +data "cloudfoundry_domain" "public" { + name = "app.cloud.gov" +} + +data "cloudfoundry_space" "app_space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +resource "cloudfoundry_route" "app_route" { + space = data.cloudfoundry_space.app_space.id + domain = data.cloudfoundry_domain.public.id + hostname = "fac-${replace(var.cf_space_name, ".", "-")}" + # Yields something like: fac-sandbox.app.cloud.gov +} + +data "external" "app_zip" { + program = ["/bin/sh", "prepare_app.sh"] + working_dir = path.module + query = { + gitref = var.gitref + } +} + +resource "cloudfoundry_user_provided_service" "clam" { + name = "clamav_ups" + space = data.cloudfoundry_space.app_space.id + credentials = { + "AV_SCAN_URL" : local.scan_url + } +} + + +resource "cloudfoundry_user_provided_service" "credentials" { + name = "fac-key-service" + space = data.cloudfoundry_space.app_space.id + credentials_json = </dev/null || mktemp -d -t 'mytmpdir') +cd "$tmpdir" + +# Grab a copy of the zip file for the specified ref +curl -s -L "https://github.com/gsa-tts/fac/archive/${GITREF}.zip" --output local.zip +branch=$(echo "$GITREF" | cut -f3 -d"/") + +# Zip up just the FAC-main/ subdirectory for pushing +# Before zip stage, run [ npm ci --production | npm run build ] in /backend/ to get the compiled assets for the site in /static/compiled/ +unzip -q -u local.zip \*"FAC-$branch/backend/*"\* +cd "${tmpdir}/FAC-$branch/backend/" && +npm ci --production --silent && +npm run build > '/dev/null' 2>&1 && +zip -r -o -X "${popdir}/app.zip" ./ > /dev/null +# zip -q -j -r ${popdir}/app.zip fac-*/backend + +# Tell Terraform where to find it +cat << EOF +{ "path": "app.zip" } +EOF diff --git a/terraform/shared/modules/app/providers.tf b/terraform/shared/modules/app/providers.tf new file mode 100644 index 0000000000..d65b80a160 --- /dev/null +++ b/terraform/shared/modules/app/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~>0.53.1" + } + } +} diff --git a/terraform/shared/modules/app/variables.tf b/terraform/shared/modules/app/variables.tf new file mode 100644 index 0000000000..8acfbb3da7 --- /dev/null +++ b/terraform/shared/modules/app/variables.tf @@ -0,0 +1,93 @@ +variable "name" { + type = string + description = "name of the fac application" +} + +variable "cf_org_name" { + type = string + description = "cloud.gov organization name" +} + +variable "cf_space_name" { + type = string + description = "cloud.gov space name for app (eg firstname.lastname)" +} + +variable "gitref" { + type = string + description = "gitref for the specific version of app that you want to use" + default = "refs/heads/main" + # You can also specify a specific commit, eg "7487f882903b9e834a5133a883a88b16fb8b16c9" +} + +variable "app_memory" { + type = number + description = "Memory in MB to allocate to app app instance" + default = 1046 +} + +variable "app_instances" { + type = number + description = "the number of instances of the app to run (default: 1)" + default = 1 +} + +variable "disk_quota" { + type = number + description = "disk in MB to allocate to cg-logshipper app instance" + default = 512 +} + +variable "private_s3_id" { + type = string + description = "the full string of the private s3 resource id" +} + +variable "public_s3_id" { + type = string + description = "the full string of the public s3 resource id" +} + +variable "db_id" { + type = string + description = "the full string of the core db resource id" +} + +variable "backup_db_id" { + type = string + description = "the full string of the backup db resource id" +} +# # Can't be created before the app exists +variable "https_proxy" { + type = string + description = "the full string of the https proxy for use with the app" +} + +variable "https_proxy_creds_id" { + type = string + description = "the id of the credentials for the proxy to bind to the app" +} + +variable "new_relic_creds_id" { + type = string + description = "the id of the credentials for newrelic to bind to the app" +} + +variable "recursive_delete" { + type = bool + description = "when true, deletes service bindings attached to the resource (not recommended for production)" + default = false +} + +variable "sam_api_key" { + type = string +} +variable "django_secret_login_key" { + type = string +} +variable "login_client_id" { + type = string +} +variable "login_secret_key" { + type = string +} diff --git a/terraform/shared/modules/cors/sandbox-cors.json b/terraform/shared/modules/cors/sandbox-cors.json new file mode 100644 index 0000000000..245cbcd393 --- /dev/null +++ b/terraform/shared/modules/cors/sandbox-cors.json @@ -0,0 +1,19 @@ +{ + "CORSRules": [ + { + "AllowedHeaders": [ + "Authorization" + ], + "AllowedMethods": [ + "HEAD", + "GET" + ], + "AllowedOrigins": [ + "https://fac-sandbox.app.cloud.gov" + ], + "ExposeHeaders": [ + "ETag" + ] + } + ] + } diff --git a/terraform/shared/modules/https-proxy/https-proxy.tf b/terraform/shared/modules/https-proxy/https-proxy.tf index 5e7f42f715..7145d35fc1 100644 --- a/terraform/shared/modules/https-proxy/https-proxy.tf +++ b/terraform/shared/modules/https-proxy/https-proxy.tf @@ -110,6 +110,7 @@ locals { protocol = "https" port = 61443 app_id = cloudfoundry_app.egress_app.id + creds_id = cloudfoundry_user_provided_service.credentials.id } resource "cloudfoundry_user_provided_service" "credentials" { diff --git a/terraform/shared/modules/https-proxy/outputs.tf b/terraform/shared/modules/https-proxy/outputs.tf index c3463112f6..90a5366d75 100644 --- a/terraform/shared/modules/https-proxy/outputs.tf +++ b/terraform/shared/modules/https-proxy/outputs.tf @@ -25,3 +25,7 @@ output "app_id" { output "port" { value = local.port } + +output "creds_id" { + value = local.creds_id +} diff --git a/terraform/shared/modules/https-proxy/prepare-proxy.sh b/terraform/shared/modules/https-proxy/prepare-proxy.sh index 67f9d01446..d90256a423 100644 --- a/terraform/shared/modules/https-proxy/prepare-proxy.sh +++ b/terraform/shared/modules/https-proxy/prepare-proxy.sh @@ -13,7 +13,7 @@ tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') cd "$tmpdir" # Grab a copy of the zip file for the specified ref -curl -s -L https://github.com/GSA-TTS/cg-egress-proxy/archive/${GITREF}.zip --output local.zip +curl -s -L https://github.com/GSA-TTS/cg-egress-proxy/archive/${GITREF}.zip --output local.zip # Zip up just the proxy/ subdirectory for pushing unzip -q -u local.zip \*/proxy/\* diff --git a/terraform/shared/modules/sandbox-proxy/README.md b/terraform/shared/modules/sandbox-proxy/README.md new file mode 100644 index 0000000000..f9aacc3c20 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/README.md @@ -0,0 +1,66 @@ +# Egress proxy Terraform module + +## Usage + +``` +module "https-proxy" { + source = "" + + name = "egress" + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + client_space = local.cf_space_name + + allowlist = { + client1 = ["*.sam.gov:443", "*.login.gov:443"], + client2 = ["gsa.gov:443"], + } + denylist = { + client1 = ["bad.sam.gov:443", "verybad.login.gov:443"], + client2 = ["sobad.gsa.gov:443"] + } +} +``` + +Credentials and route for the proxy are stored in the `egress-creds` service in the client space. + +> **Note** +> +> It's up to you to bind the `egress-creds` service to the clients, read the credentials from `VCAP_SERVICES`, and configure your app appropriately to use the proxy! + +## Deployment architecture + +```mermaid + C4Context + title blue items are managed by the module + Boundary(system, "system boundary") { + Boundary(trusted_local_egress, "egress-controlled space", "trusted-local-egress ASG") { + System(credentials, "Proxy Credentials", "UPSI") + System_Ext(client1, "Client1", "a client") + } + + Boundary(public_egress, "egress-permitted space", "public-egress ASG") { + System(https_proxy, "web egress proxy", "proxy for HTTP/S connections") + } + } + + Boundary(external_boundary, "external boundary") { + System_Ext(external_service, "external service", "service that the application relies on") + } + + Rel(credentials, client1, "delivers credentials", "VCAP_SERVICES") + Rel(client1, https_proxy, "makes request", "HTTP/S") + Rel(https_proxy, external_service, "proxies request", "HTTP/S") +``` + + +1. Creates an egress proxy in the designated space +2. Adds network-policies so that clients can reach the proxy +3. Creates a user-provided service instance in the client space with credentials + +## TODO + +* Once it's possible, [create the UPSI in the egress space and share it to the client space](https://github.com/cloudfoundry-community/terraform-provider-cloudfoundry/issues/481) +* Support multiple client spaces (maybe a map of allowlist and denylist entries per space?) +* Pay attention to the port number; right now it's ignored (the proxy has a fixed set of ports in the config file) +* Pay attention to the client ID; right now allow/deny are a union across all client apps diff --git a/terraform/shared/modules/sandbox-proxy/acl.tftpl b/terraform/shared/modules/sandbox-proxy/acl.tftpl new file mode 100644 index 0000000000..c283a64ba9 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/acl.tftpl @@ -0,0 +1,5 @@ +%{ for app, dests in list ~} +%{ for dest in dests ~} +${ split(":", dest)[0] } +%{ endfor ~} +%{ endfor ~} \ No newline at end of file diff --git a/terraform/shared/modules/sandbox-proxy/https-proxy.tf b/terraform/shared/modules/sandbox-proxy/https-proxy.tf new file mode 100644 index 0000000000..0fa920ded5 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/https-proxy.tf @@ -0,0 +1,127 @@ +locals { + + # Make a clean list of the client apps for iteration purposes + clients = toset(keys(merge(var.allowlist, var.denylist))) + + # Generate Caddy-compatible allow and deny ACLs, one target per line. + # + # For now, there's just one consolidated allowlist and denylist, no matter + # what apps they were specified for. Future improvments could improve this, + # but it would mean also changing the proxy to be both more complex (in terms + # of how the Caddyfile is constructed) and more discriminating (in terms of + # recognizing client apps based on GUIDs supplied by Envoy in request headers, + # as well as the destination ports). However, adding these improvements won't + # require modifying the module's interface, since we're already collecting + # that refined information. + allowacl = templatefile("${path.module}/acl.tftpl", { list = var.allowlist }) + denyacl = templatefile("${path.module}/acl.tftpl", { list = var.denylist }) +} + +### +### Set up the authenticated egress application in the target space on apps.internal +### + +data "cloudfoundry_domain" "internal" { + name = "apps.internal" +} + +resource "cloudfoundry_route" "egress_route" { + space = data.cloudfoundry_space.egress_space.id + domain = data.cloudfoundry_domain.internal.id + hostname = "${var.cf_org_name}-${replace(var.cf_space_name, ".", "-")}-${var.name}" + # Yields something like: orgname-spacename-name.apps.internal +} + +resource "random_uuid" "username" {} +resource "random_password" "password" { + length = 16 + special = false +} + +data "cloudfoundry_space" "egress_space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +# This zips up just the depoyable files from the specified gitref in the +# cg-egress-proxy repository +data "external" "proxyzip" { + program = ["/bin/sh", "prepare-proxy.sh"] + working_dir = path.module + query = { + gitref = var.gitref + } +} + +resource "cloudfoundry_app" "egress_app" { + name = var.name + space = data.cloudfoundry_space.egress_space.id + path = "${path.module}/${data.external.proxyzip.result.path}" + source_code_hash = filesha256("${path.module}/${data.external.proxyzip.result.path}") + buildpack = "binary_buildpack" + command = "./caddy run --config Caddyfile" + memory = var.egress_memory + instances = var.instances + strategy = "rolling" + + routes { + route = cloudfoundry_route.egress_route.id + } + environment = { + PROXY_ALLOW : local.allowacl + PROXY_DENY : local.denyacl + PROXY_USERNAME : random_uuid.username.result + PROXY_PASSWORD : random_password.password.result + } +} + +### +### Set up network policies so that the clients can reach the proxy +### + +data "cloudfoundry_space" "client_space" { + org_name = var.cf_org_name + name = var.client_space +} + +# data "cloudfoundry_app" "clients" { +# for_each = local.clients +# name_or_id = each.key +# space = data.cloudfoundry_space.client_space.id +# } + +# resource "cloudfoundry_network_policy" "client_routing" { +# for_each = local.clients +# policy { +# source_app = data.cloudfoundry_app.clients[each.key].id +# destination_app = cloudfoundry_app.egress_app.id +# port = "61443" +# } +# } + +### +### Create a credential service for bound clients to use when make requests of the proxy +### +locals { + https_proxy = "https://${random_uuid.username.result}:${random_password.password.result}@${cloudfoundry_route.egress_route.endpoint}:61443" + domain = cloudfoundry_route.egress_route.endpoint + username = random_uuid.username.result + password = random_password.password.result + protocol = "https" + port = 61443 + app_id = cloudfoundry_app.egress_app.id + creds_id = cloudfoundry_user_provided_service.credentials.id +} + +resource "cloudfoundry_user_provided_service" "credentials" { + name = "${var.name}-creds" + space = data.cloudfoundry_space.client_space.id + credentials = { + "uri" = local.https_proxy + "domain" = local.domain + "username" = local.username + "password" = local.password + "protocol" = local.protocol + "port" = local.port + } +} diff --git a/terraform/shared/modules/sandbox-proxy/outputs.tf b/terraform/shared/modules/sandbox-proxy/outputs.tf new file mode 100644 index 0000000000..90a5366d75 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/outputs.tf @@ -0,0 +1,31 @@ +output "https_proxy" { + value = local.https_proxy +} + +output "domain" { + value = local.domain +} + +output "username" { + value = local.username +} + +output "password" { + value = local.password +} + +output "protocol" { + value = local.protocol +} + +output "app_id" { + value = local.app_id +} + +output "port" { + value = local.port +} + +output "creds_id" { + value = local.creds_id +} diff --git a/terraform/shared/modules/sandbox-proxy/prepare-proxy.sh b/terraform/shared/modules/sandbox-proxy/prepare-proxy.sh new file mode 100755 index 0000000000..a6db1d5c66 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/prepare-proxy.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Exit if any step fails +set -e + +eval "$(jq -r '@sh "GITREF=\(.gitref)"')" + +popdir=$(pwd) + +# Portable construct so this will work everywhere +# https://unix.stackexchange.com/a/84980 +tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir') +cd "$tmpdir" + +# Grab a copy of the zip file for the specified ref +curl -s -L https://github.com/GSA-TTS/cg-egress-proxy/archive/${GITREF}.zip --output local.zip + +# Zip up just the proxy/ subdirectory for pushing +unzip -q -u local.zip \*/proxy/\* +cd "${tmpdir}/cg-egress-proxy-main/proxy/" && zip -r -o -X "${popdir}/proxy.zip" ./ > /dev/null +#zip -q -j -r ${popdir}/proxy.zip cg-egress-proxy-*/proxy + +# Tell Terraform where to find it +cat << EOF +{ "path": "proxy.zip" } +EOF + diff --git a/terraform/shared/modules/sandbox-proxy/providers.tf b/terraform/shared/modules/sandbox-proxy/providers.tf new file mode 100644 index 0000000000..b811743998 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = ">=0.15" + } + } +} diff --git a/terraform/shared/modules/sandbox-proxy/test/.profile b/terraform/shared/modules/sandbox-proxy/test/.profile new file mode 100644 index 0000000000..bc7129f278 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/.profile @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Configure vars for https-proxy +export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +# Set the https_proxy env var based on egress-creds +function vcap_get_service () { + local path name + name="$1" + path="$2" + echo $VCAP_SERVICES | jq --raw-output --arg service_name "$name" ".[][] | select(.name == \$service_name) | $path" +} +export https_proxy=$(vcap_get_service https-egress-creds .credentials.uri) + +# Should fail via deny-all +echo www.yahoo.com: $(curl -s -o /dev/null -I -L -w "%{http_connect}" https://www.yahoo.com) + +# Should succeed via allow +echo www.google.com: $(curl -s -o /dev/null -I -L -w "%{http_connect}" https://www.google.com) + +# Should fail via deny +echo mail.google.com: $(curl -s -o /dev/null -I -L -w "%{http_connect}" https://mail.google.com) + diff --git a/terraform/shared/modules/sandbox-proxy/test/.terraform.lock.hcl b/terraform/shared/modules/sandbox-proxy/test/.terraform.lock.hcl new file mode 100644 index 0000000000..c84e30c081 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/.terraform.lock.hcl @@ -0,0 +1,63 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/cloudfoundry-community/cloudfoundry" { + version = "0.50.7" + constraints = ">= 0.15.0, ~> 0.50.2" + hashes = [ + "h1:dKYm6HNHrDEEb6faRE7xUoZEjD6fYsw2uG+avbdGMAI=", + "zh:0679ed5d688e2fbe21c8ffff285189e93c005535ebd00aecc27139aa7cde379a", + "zh:14a98148b838ea690bab3baeeb7eaecbd46b611c0147b09b07a4e08e0384c5c8", + "zh:1a3aae7823a1a84590dc9479c15e95e026066c99111bf88f16c618a4ff750a70", + "zh:1d04452eac30fea79d05d2232b60deeedc55552f88d1d03827d61cc1771feea0", + "zh:530cec74148a33f75d502cb7579af711c542aa36f9bcb011e368a08f00f578b0", + "zh:563566b53ccec3e21b880552df2d442cdf56d67c15609a2bdc1faf2e2f34f240", + "zh:5d8024966b3b16c0404cbbce37aa828a467ef453888c8de4c383a9ccac8b111b", + "zh:6a65fd3ee84a4db767cba694f239c3b2fe69c6c065441b2acac7c4b35ad2b840", + "zh:77b94f6bbd8d596071610ee72b040333cd25552c96bd4549707452ca052d0bdd", + "zh:78b813fd523c885a696dfbe4f23d49eb67b1231dc148d55e99508e8897c0920b", + "zh:97431d2f85f2883333fa895eb33483a2427fa0dcb4cfef0ae3a1218a81101659", + "zh:9a378e0ff3e6499f2bc510060909a7f8e2a488d9a98d273c540dbcbc190a3904", + "zh:a8e87b634d0f8465884fedbff32dcd5481874b77e3688fd5c1cb790ea3fbf46d", + "zh:ca549e49147e9e0a26607bc2c4641b29f09b6133f69e897c4958188925e588ea", + "zh:fdd35b648ca2a979133e1d42c8791f22bd3ec4dc55533768c4407f8a039ed09d", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.1" + hashes = [ + "h1:bROCw6g5D/3fFnWeJ01L4IrdnJl1ILU8DGDgXCtYzaY=", + "zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54", + "zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee", + "zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd", + "zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2", + "zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df", + "zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e", + "zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326", + "zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05", + "zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8", + "zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + hashes = [ + "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", + ] +} diff --git a/terraform/shared/modules/sandbox-proxy/test/README.md b/terraform/shared/modules/sandbox-proxy/test/README.md new file mode 100644 index 0000000000..4cd6ebe443 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/README.md @@ -0,0 +1,7 @@ +# Test the https-proxy module +1. Authenticate with cloud.gov (`cf login -a api.fr.cloud.gov --sso`) +2. Copy `terraform.tfvars-template` to `terraform.tfvars` and edit to taste +3. Run `terraform init` to configure Terraform with the necessary providers +3. Run `terraform apply` to deploy a test fixture app and the proxy +4. Verify success +5. Run `terraform destroy` to tear everything down \ No newline at end of file diff --git a/terraform/shared/modules/sandbox-proxy/test/https-proxy-test.tf b/terraform/shared/modules/sandbox-proxy/test/https-proxy-test.tf new file mode 100644 index 0000000000..7dc84558fa --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/https-proxy-test.tf @@ -0,0 +1,88 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~>0.50.2" + } + } +} + +variable "cf_org_name" { + type = string +} +variable "cf_space_name" { + type = string +} +variable "user" { + type = string +} +variable "password" { + type = string + sensitive = true +} + +provider "cloudfoundry" { + api_url = "https://api.fr.cloud.gov" + password = var.password + user = var.user +} + +data "cloudfoundry_space" "client_space" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +# A test client app. This app doesn't actually do anything; it's static. We just +# need it to exist so that there's something for the module to look for in its +# clients list, and for which it can set up a network-policy. +data "external" "clientzip" { + program = ["/bin/sh", "${path.module}/prepare-client.sh"] +} +resource "cloudfoundry_app" "test" { + name = "test" + space = data.cloudfoundry_space.client_space.id + path = data.external.clientzip.result.path + source_code_hash = filesha256(data.external.clientzip.result.path) + buildpack = data.external.clientzip.result.buildpack +} + +# Exercise the module under test +module "https-proxy" { + source = "./.." + + name = "https-egress" + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + client_space = var.cf_space_name + + allowlist = { + test = ["*.google.com:443", "*.login.gov:443", "gsa.gov:443"], + } + denylist = { + test = ["mail.google.com:443", "verybad.login.gov:443"], + } + + # Since there's no other way for Terraform to infer the dependency, this + # ensures the test app exists before we try to configure the proxy to handle + # it. + depends_on = [ + cloudfoundry_app.test + ] +} + +data "external" "validate" { + depends_on = [ + module.https-proxy + ] + program = ["/bin/sh", "${path.module}/validate.sh"] + query = { + APPNAME = "https-egress" + ORGNAME = var.cf_org_name + SPACENAME = var.cf_space_name + } +} + +output "test-result" { + value = data.external.validate.result.status +} diff --git a/terraform/shared/modules/sandbox-proxy/test/index.html b/terraform/shared/modules/sandbox-proxy/test/index.html new file mode 100644 index 0000000000..ddd9e890a7 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/index.html @@ -0,0 +1,9 @@ + + + + Hello world + + +

I'm just a static file. However, you can look at the space where this is deployed to see if the https-proxy is properly configured.

+ + \ No newline at end of file diff --git a/terraform/shared/modules/sandbox-proxy/test/prepare-client.sh b/terraform/shared/modules/sandbox-proxy/test/prepare-client.sh new file mode 100644 index 0000000000..1570b6d8e9 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/prepare-client.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Exit if any step fails +set -e + +# Zip up a staticfile client app +zip -q -j -r ./client.zip index.html .profile + +# Tell Terraform where to find it +cat << EOF +{ + "path": "client.zip", + "buildpack": "staticfile_buildpack" +} +EOF + diff --git a/terraform/shared/modules/sandbox-proxy/test/terraform.tfvars-template b/terraform/shared/modules/sandbox-proxy/test/terraform.tfvars-template new file mode 100644 index 0000000000..971753e966 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/terraform.tfvars-template @@ -0,0 +1,11 @@ +# Copy this file to terraform.tfvars, then edit to taste. Don't commit your copy +# to version control! + +# Provide the credentials of a deployer user +# Instructions for generating them are here: +# https://cloud.gov/docs/services/cloud-gov-service-account/#how-to-create-an-instance +user = "a-username" +password = "a-password" + +cf_org_name = "org-for-proxy" +cf_space_name = "space-for-proxy" \ No newline at end of file diff --git a/terraform/shared/modules/sandbox-proxy/test/validate.sh b/terraform/shared/modules/sandbox-proxy/test/validate.sh new file mode 100644 index 0000000000..896d2427a7 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/test/validate.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -e + +# A quick script to validate that the deployed config is what we expect + +# If we exit for any reason before this is explicitly changed, then something +# went wrong. +result="FAILED" + +# Tell Terraform what happened when we're done +finish() { +cat << EOF +{ + "status": "${result}" +} +EOF +} +trap finish EXIT + +# Get our parameters from the JSON input into env vars +eval "$(jq -r '@sh "APPNAME=\(.APPNAME) ORGNAME=\(.ORGNAME) SPACENAME=\(.SPACENAME)"')" + +# Target the place where we think stuff is +cf t -o "$ORGNAME" -s "$SPACENAME" > /dev/null 2>&1 + +# Check that the egress app exists +cf app "${APPNAME}" > /dev/null 2>&1 + +# Check that the egress app has the appropriate env vars set +# TODO: Improve this to ensure the var content is as expected. +cf env "${APPNAME}" | grep -q PROXY_ALLOW +cf env "${APPNAME}" | grep -q PROXY_DENY +cf env "${APPNAME}" | grep -q PROXY_USERNAME +cf env "${APPNAME}" | grep -q PROXY_PASSWORD + +# Check that a network policy with appropriate values exists +# TODO: Improve this to use just one robust regexp (or use "cf curl" and "jq"). +cf network-policies | grep test | grep "${APPNAME}" | grep 61443 | grep "$ORGNAME" | grep -q "$SPACENAME" + +# Check that a user-provided service with the egress credentials exists +cf service "${APPNAME}"-creds > /dev/null 2>&1 + +# Bind the creds to the test app and ensure it knows about them +cf bind-service test "${APPNAME}"-creds > /dev/null 2>&1 +cf restart test > /dev/null 2>&1 + +# Test that curl requests through the proxy are succeeding/failing as expected +output="$(cf ssh test -t -c "/home/vcap/app/.profile")" +expected_output="www.yahoo.com: 403 +www.google.com: 200 +mail.google.com: 403" + +[ "$output" = "$expected_output" ] && result="PASSED" + diff --git a/terraform/shared/modules/sandbox-proxy/variables.tf b/terraform/shared/modules/sandbox-proxy/variables.tf new file mode 100644 index 0000000000..58b118f057 --- /dev/null +++ b/terraform/shared/modules/sandbox-proxy/variables.tf @@ -0,0 +1,58 @@ +variable "cf_org_name" { + type = string + description = "cloud.gov organization name" +} + +variable "cf_space_name" { + type = string + description = "cloud.gov space name for egress (eg staging-egress or prod-egress)" +} + +variable "client_space" { + type = string + description = "cloud.gov space name for client apps (eg staging or prod)" +} + +variable "name" { + type = string + description = "name of the egress proxy application" +} + +variable "egress_memory" { + type = number + description = "Memory in MB to allocate to egress proxy app" + default = 64 +} + +variable "gitref" { + type = string + description = "gitref for the specific version of cg-egress-proxy that you want to use" + default = "refs/heads/main" + # You can also specify a specific commit, eg "7487f882903b9e834a5133a883a88b16fb8b16c9" +} + +variable "allowlist" { + description = "Allowed egress for apps (applied first). A map where keys are app names, and the values are sets of acl strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "*.example.com:443", "example2.com:443" ] + } +} + +variable "denylist" { + description = "Denied egress for apps (applied second). A map where keys are app names, and the values are sets of host:port strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "bad.example.com:443" ] + } +} + +variable "instances" { + type = number + description = "the number of instances of the HTTPS proxy application to run (default: 2)" + default = 2 +} diff --git a/terraform/shared/modules/sandbox/acl.tftpl b/terraform/shared/modules/sandbox/acl.tftpl new file mode 100644 index 0000000000..a109cb1dee --- /dev/null +++ b/terraform/shared/modules/sandbox/acl.tftpl @@ -0,0 +1,5 @@ +%{ for app, dests in list ~} +%{ for dest in dests ~} +${ split(":", dest)[0] } +%{ endfor ~} +%{ endfor ~} diff --git a/terraform/shared/modules/sandbox/app.tf b/terraform/shared/modules/sandbox/app.tf new file mode 100644 index 0000000000..a3ceacb3b5 --- /dev/null +++ b/terraform/shared/modules/sandbox/app.tf @@ -0,0 +1,34 @@ +locals { + app_name = "gsa-fac" +} + +module "fac-app" { + source = "../app" + name = local.app_name + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + https_proxy = module.https-proxy.https_proxy + https_proxy_creds_id = module.https-proxy.creds_id + new_relic_creds_id = cloudfoundry_user_provided_service.credentials.id + private_s3_id = module.s3-private.bucket_id + public_s3_id = module.s3-public.bucket_id + db_id = module.database.instance_id + backup_db_id = module.snapshot-database.instance_id + app_instances = 1 + app_memory = 4096 + disk_quota = 3072 + gitref = "refs/heads/${var.branch_name}" + django_secret_login_key = var.django_secret_login_key + sam_api_key = var.sam_api_key + login_client_id = var.login_client_id + login_secret_key = var.login_secret_key +} + +resource "cloudfoundry_network_policy" "app-network-policy" { + policy { + source_app = module.fac-app.app_id + destination_app = module.https-proxy.app_id + port = "61443" + protocol = "tcp" + } +} diff --git a/terraform/shared/modules/sandbox/clamav.tf b/terraform/shared/modules/sandbox/clamav.tf new file mode 100644 index 0000000000..ba633b947b --- /dev/null +++ b/terraform/shared/modules/sandbox/clamav.tf @@ -0,0 +1,38 @@ +locals { + clam_name = "fac-av-${var.cf_space_name}" +} + +data "docker_registry_image" "clamav" { + name = "ghcr.io/gsa-tts/fac/clamav:latest" +} + +module "clamav" { + source = "github.com/gsa-tts/terraform-cloudgov//clamav?ref=v1.1.0" + + # This generates eg "fac-av-staging.apps.internal", avoiding collisions with routes for other projects and spaces + name = local.clam_name + app_name_or_id = "gsa-fac" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + clamav_image = "ghcr.io/gsa-tts/fac/clamav@${data.docker_registry_image.clamav.sha256_digest}" + max_file_size = "30M" + instances = var.clamav_instances + clamav_memory = var.clamav_memory + + proxy_server = module.https-proxy.domain + proxy_port = module.https-proxy.port + proxy_username = module.https-proxy.username + proxy_password = module.https-proxy.password + # depends_on = [module.fac-app.app_id] + # depends_on = [ module.https-proxy.https_proxy ] +} + +# resource "cloudfoundry_network_policy" "clamav-network-policy" { +# policy { +# source_app = module.clamav.app_id +# destination_app = module.https-proxy.app_id +# port = "61443" +# protocol = "tcp" +# } +# } diff --git a/terraform/shared/modules/sandbox/env.tf b/terraform/shared/modules/sandbox/env.tf new file mode 100644 index 0000000000..ff3ee748be --- /dev/null +++ b/terraform/shared/modules/sandbox/env.tf @@ -0,0 +1,55 @@ +module "database" { + source = "github.com/gsa-tts/terraform-cloudgov//database?ref=v1.1.0" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-db" + tags = ["rds"] + rds_plan_name = var.database_plan + json_params = var.json_params +} + +module "snapshot-database" { + source = "github.com/gsa-tts/terraform-cloudgov//database?ref=v1.1.0" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-snapshot-db" + tags = ["rds"] + rds_plan_name = var.database_plan + json_params = var.json_params +} + +module "s3-public" { + source = "github.com/gsa-tts/terraform-cloudgov//s3?ref=v1.1.0" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-public-s3" + s3_plan_name = "basic-public" + tags = ["s3"] +} + +module "s3-private" { + source = "github.com/gsa-tts/terraform-cloudgov//s3?ref=v1.1.0" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-private-s3" + s3_plan_name = "basic" + tags = ["s3"] +} + +# Stuff used for apps in this space +data "cloudfoundry_space" "apps" { + org_name = var.cf_org_name + name = var.cf_space_name +} + +data "cloudfoundry_domain" "public" { + name = "app.cloud.gov" +} + +data "cloudfoundry_domain" "private" { + name = "apps.internal" +} diff --git a/terraform/shared/modules/sandbox/https-proxy.tf b/terraform/shared/modules/sandbox/https-proxy.tf new file mode 100644 index 0000000000..c70ede7523 --- /dev/null +++ b/terraform/shared/modules/sandbox/https-proxy.tf @@ -0,0 +1,55 @@ +module "https-proxy" { + source = "../sandbox-proxy" + name = "https-proxy" + cf_org_name = var.cf_org_name # gsa-tts-oros-fac + cf_space_name = "${var.cf_space_name}-egress" # eg prod-egress + client_space = var.cf_space_name # eg prod + instances = var.https_proxy_instances + + # head of the "main" branch as of this commit + gitref = "7487f882903b9e834a5133a883a88b16fb8b16c9" + + allowlist = { + gsa-fac = [ + # SAM.gov API (https://open.gsa.gov/api/entity-api/) + "api.sam.gov:443", + + # New Relic telemetry (https://docs.newrelic.com/docs/new-relic-solutions/get-started/networks/#data-ingest) + # + # Note: New Relic says that APM agent data ingests via `collector*.newrelic.com`, but we can't specify that + # to the Caddy forwardproxy (https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#access-control) + # because it only allows subdomain wildcards in `*.` as a prefix. So this wildcard is a little broader than + # we would prefer, but realistically the tightest domain mask we can specify given our current solution. + # We put in an upstream issue about this: https://github.com/caddyserver/forwardproxy/issues/102 + "*.newrelic.com:443", + + # It was determined that NR wants to proxy a connection to a proxy. This does need the NEW_RELIC_PROXY_HOST="$https_proxy" set, + # in conjunction with this. We believe this to be a potential bug with the new relic agent, + # and will be reporting this to New Relic in the hopes of being able to remove the proxy from our allow list. + # This is thanks to Ryan Ahearn at 18F for pointing us in this direction + # https://gsa-tts.slack.com/archives/C09CR1Q9Z/p1699394487090859 + "${var.cf_org_name}-${var.cf_space_name}-egress-https-proxy.apps.internal", + + # Login.gov sandbox + "idp.int.identitysandbox.gov:443", + + # Login.gov + "secure.login.gov:443", + + # Git + "*.github.com:443", + # The following needs to be added to the allowlist so that when we curl the s3-tar-tool to perform backups, + # the curl command can follow the redirect. + "objects.githubusercontent.com:443", + + # The following needs to be added to the allowlist so that we can get aws cli onto the instance to perform backups. + "awscli.amazonaws.com:443" + ], + # The parens here make Terraform understand that the key below is a reference + # Solution from https://stackoverflow.com/a/57401750 + (local.clam_name) = ["database.clamav.net:443"], + } + denylist = {} + # depends_on = [ module.fac-app.app_id, module.clamav.app_id ] + # depends_on = [ module.clamav.app_id ] +} diff --git a/terraform/shared/modules/sandbox/newrelic.tf b/terraform/shared/modules/sandbox/newrelic.tf new file mode 100644 index 0000000000..e7438934bb --- /dev/null +++ b/terraform/shared/modules/sandbox/newrelic.tf @@ -0,0 +1,16 @@ +resource "cloudfoundry_user_provided_service" "credentials" { + name = "newrelic-creds" + space = data.cloudfoundry_space.apps.id + credentials = { + "NEW_RELIC_LICENSE_KEY" = var.new_relic_license_key + "NEW_RELIC_LOGS_ENDPOINT" = "https://gov-log-api.newrelic.com/log/v1" + } + tags = ["newrelic-creds"] +} + +# module "newrelic" { +# source = "../newrelic" +# cf_space_name = var.cf_space_name +# new_relic_account_id = var.new_relic_account_id +# new_relic_api_key = var.new_relic_api_key +# } diff --git a/terraform/shared/modules/sandbox/postgrest.tf b/terraform/shared/modules/sandbox/postgrest.tf new file mode 100644 index 0000000000..0d0c4d7b69 --- /dev/null +++ b/terraform/shared/modules/sandbox/postgrest.tf @@ -0,0 +1,40 @@ +locals { + postgrest_name = "postgrest" +} + +resource "cloudfoundry_route" "postgrest" { + space = data.cloudfoundry_space.apps.id + domain = data.cloudfoundry_domain.public.id + hostname = "fac-${var.cf_space_name}-${local.postgrest_name}" +} + +resource "cloudfoundry_service_key" "postgrest" { + name = "postgrest" + service_instance = module.database.instance_id +} + +data "docker_registry_image" "postgrest" { + name = "ghcr.io/gsa-tts/fac/postgrest:latest" +} + +resource "cloudfoundry_app" "postgrest" { + name = local.postgrest_name + space = data.cloudfoundry_space.apps.id + docker_image = "ghcr.io/gsa-tts/fac/postgrest@${data.docker_registry_image.postgrest.sha256_digest}" + timeout = 180 + memory = 1024 + disk_quota = 256 + instances = var.postgrest_instances + strategy = "rolling" + routes { + route = cloudfoundry_route.postgrest.id + } + + environment = { + PGRST_DB_URI : cloudfoundry_service_key.postgrest.credentials.uri + PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0" + PGRST_DB_ANON_ROLE : "anon" + PGRST_JWT_SECRET : var.pgrst_jwt_secret + PGRST_DB_MAX_ROWS : 20000 + } +} diff --git a/terraform/shared/modules/sandbox/providers.tf b/terraform/shared/modules/sandbox/providers.tf new file mode 100644 index 0000000000..c60d0b3311 --- /dev/null +++ b/terraform/shared/modules/sandbox/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~>0.53.1" + } + + docker = { + source = "kreuzwerker/docker" + version = "~>3.0.2" + } + } +} diff --git a/terraform/shared/modules/sandbox/routes.tf b/terraform/shared/modules/sandbox/routes.tf new file mode 100644 index 0000000000..d701d32b67 --- /dev/null +++ b/terraform/shared/modules/sandbox/routes.tf @@ -0,0 +1,27 @@ +locals { + # Make a clean list of the client apps for iteration purposes + clients = toset(keys(merge(var.allowlist, var.denylist))) + client_space = var.cf_space_name +} + +data "cloudfoundry_space" "client_space" { + org_name = var.cf_org_name + name = local.client_space +} + +data "cloudfoundry_app" "clients" { + for_each = local.clients + name_or_id = each.key + space = data.cloudfoundry_space.client_space.id + depends_on = [module.fac-app, module.clamav, cloudfoundry_app.postgrest, module.https-proxy] +} + +resource "cloudfoundry_network_policy" "client_routing" { + for_each = local.clients + policy { + source_app = data.cloudfoundry_app.clients[each.key].id + destination_app = module.https-proxy.https_proxy + port = "61443" + } + depends_on = [module.fac-app, module.clamav, cloudfoundry_app.postgrest, module.https-proxy] +} diff --git a/terraform/shared/modules/sandbox/smtp-proxy.tf b/terraform/shared/modules/sandbox/smtp-proxy.tf new file mode 100644 index 0000000000..bdfb8260b1 --- /dev/null +++ b/terraform/shared/modules/sandbox/smtp-proxy.tf @@ -0,0 +1,12 @@ +# module "smtp-proxy" { +# source = "../stream-proxy" +# name = "smtp-proxy" +# cf_org_name = var.cf_org_name +# cf_space_name = "${var.cf_space_name}-egress" +# client_space = var.cf_space_name +# instances = var.smtp_proxy_instances + +# upstream = "smtp-relay.gmail.com:587" +# clients = ["gsa-fac"] +# depends_on = [ module.fac-app ] +# } diff --git a/terraform/shared/modules/sandbox/variables.tf b/terraform/shared/modules/sandbox/variables.tf new file mode 100644 index 0000000000..de4592334c --- /dev/null +++ b/terraform/shared/modules/sandbox/variables.tf @@ -0,0 +1,135 @@ +# These variables expose what is open for customization in an environment. Where +# there are defaults, they are the production defaults. +# +# Example usage: +# +# For production: +# module "production" { +# source = "../shared/modules/base" +# cf_space_name = "production" +# # No further customization needed +# } +# +# For dev: +# module "dev" { +# cf_space_name = "dev" +# database_plan = "micro-psql" +# recursive_delete = true +# } + + +variable "cf_org_name" { + type = string + description = "name of the organization to configure" + default = "gsa-tts-oros-fac" +} + +variable "cf_space_name" { + type = string + description = "name of the space to configure" + # No default... The calling module knows which env is for which space and we + # shouldn't assume it! +} + +variable "database_plan" { + type = string + description = "name of the cloud.gov RDS service plan name to create" + # See https://cloud.gov/docs/services/relational-database/#plans + default = "medium-gp-psql-redundant" +} + +variable "postgrest_instances" { + type = number + description = "the number of instances of the postgrest application to run (default: 2)" + default = 2 +} + +variable "swagger_instances" { + type = number + description = "the number of instances of the swagger application to run (default: 2)" + default = 2 +} + +variable "https_proxy_instances" { + type = number + description = "the number of instances of the HTTPS proxy application to run (default: 2)" + default = 2 +} + +variable "smtp_proxy_instances" { + type = number + description = "the number of instances of the SMTP proxy application to run (default: 2)" + default = 2 +} + +variable "clamav_instances" { + type = number + description = "the number of instances of the clamav application to run (default: 1)" + default = 1 +} + +variable "clamav_fs_instances" { + type = number + description = "the number of instances of the clamav application to run (default: 1)" + default = 1 +} + +variable "clamav_memory" { + type = number + description = "memory in MB to allocate to clamav app" + default = 2048 +} + +variable "pgrst_jwt_secret" { + type = string + description = "the JWT signing secret for validating JWT tokens from api.data.gov" +} + +variable "json_params" { + type = string + description = "Optional parameters used for service instance (-c)" +} + +variable "new_relic_license_key" { + type = string + description = "the license key to use when setting up the New Relic agent" +} + +variable "branch_name" { + type = string + description = "the heads value for the branch you wish to deploy (default would be main)" + # We don't specify a default here because we want to specify a branch to deploy +} + +variable "sam_api_key" { + type = string +} +variable "django_secret_login_key" { + type = string +} +variable "login_client_id" { + type = string +} +variable "login_secret_key" { + type = string +} + +variable "allowlist" { + description = "Allowed egress for apps (applied first). A map where keys are app names, and the values are sets of acl strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "*.example.com:443", "example2.com:443" ] + } +} + +variable "denylist" { + description = "Denied egress for apps (applied second). A map where keys are app names, and the values are sets of host:port strings." + # See the upstream documentation for possible acl strings: + # https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#caddyfile-syntax-server-configuration + type = map(set(string)) + default = { + # appname = [ "bad.example.com:443" ] + } +} diff --git a/terraform/shared/modules/stream-proxy/.terraform.lock.hcl b/terraform/shared/modules/stream-proxy/.terraform.lock.hcl new file mode 100644 index 0000000000..aca6c81142 --- /dev/null +++ b/terraform/shared/modules/stream-proxy/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/cloudfoundry-community/cloudfoundry" { + version = "0.53.1" + constraints = ">= 0.15.0" + hashes = [ + "h1:U6Cx0ZA1N8PlKPYOw2uxqWvSvU6zeEwSdmP6RzZ3eXw=", + "zh:017a55cdbd444ccf8fe45a3c7cdbc08ddf4f0f13550fcd457c31df9b2cfdb767", + "zh:100e9bd10868547193134082427abebad9db6359f6139a882192232e8e6911e3", + "zh:34467f6504e8527bd3e18e372d5386a43f2bffd88abf54bb72d51f04ab3e4e23", + "zh:3a278f5f71e39d29c7db999e2a34e8135b79cee4f36510b0f2c2dfec47997cf1", + "zh:3be1fbe17382c91561b1985d372606d802513d94bae6368e1bafd8dd49494737", + "zh:3f12bd7a629d547c706c380d9499ff39eab7b8824a14662aa446f230304bdd3a", + "zh:404acaa9ad7f95e83baf2332be54c065c21053bf304e80ac41ae49719462b184", + "zh:5ac5f6159d1e0c989e739cf16aa8dede6cee3562a6262bf9f2c6b53f4da866fe", + "zh:7a440ee173e69fa153ea4baea47adfca34d7171ffc83e7a1c0ec319d28998cbc", + "zh:87e2200bf66443671e249108d1cfa4aa13a31b9fdf445cec88364db8ea6be623", + "zh:b1b20b2b751df7765225cee5b01290b06e245e50faa8053495c2ef5ebe316998", + "zh:c8ddda9cf7dff40d762ea4dc22941c993ae8e9b2388c8d421f43254a56c98482", + "zh:d6ce83f0077a9f6262ffa1f7d777e2b72feac7ea7c8735aa39a5f86b4f3f7084", + "zh:d74126b9189ab4ca137ca634eaa25c571491bdd2456ccd0f3276a6d49163e412", + "zh:db5d415346e03eac0c5e025f9c10afdebfff35487e8a8383b3c4cd867c422fe2", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.3.4" + hashes = [ + "h1:XWkRZOLKMjci9/JAtE8X8fWOt7A4u+9mgXSUjc4Wuyo=", + "zh:037fd82cd86227359bc010672cd174235e2d337601d4686f526d0f53c87447cb", + "zh:0ea1db63d6173d01f2fa8eb8989f0809a55135a0d8d424b08ba5dabad73095fa", + "zh:17a4d0a306566f2e45778fbac48744b6fd9c958aaa359e79f144c6358cb93af0", + "zh:298e5408ab17fd2e90d2cd6d406c6d02344fe610de5b7dae943a58b958e76691", + "zh:38ecfd29ee0785fd93164812dcbe0664ebbe5417473f3b2658087ca5a0286ecb", + "zh:59f6a6f31acf66f4ea3667a555a70eba5d406c6e6d93c2c641b81d63261eeace", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:ad0279dfd09d713db0c18469f585e58d04748ca72d9ada83883492e0dd13bd58", + "zh:c69f66fd21f5e2c8ecf7ca68d9091c40f19ad913aef21e3ce23836e91b8cbb5f", + "zh:d4a56f8c48aa86fc8e0c233d56850f5783f322d6336f3bf1916e293246b6b5d4", + "zh:f2b394ebd4af33f343835517e80fc876f79361f4688220833bc3c77655dd2202", + "zh:f31982f29f12834e5d21e010856eddd19d59cd8f449adf470655bfd19354377e", + ] +} diff --git a/terraform/shared/modules/stream-proxy/prepare-proxy.sh b/terraform/shared/modules/stream-proxy/prepare-proxy.sh index 4740a08621..fa42f91aca 100644 --- a/terraform/shared/modules/stream-proxy/prepare-proxy.sh +++ b/terraform/shared/modules/stream-proxy/prepare-proxy.sh @@ -8,4 +8,3 @@ zip -q -j -r proxy.zip app cat << EOF { "path": "proxy.zip" } EOF - diff --git a/terraform/shared/modules/stream-proxy/stream-proxy.tf b/terraform/shared/modules/stream-proxy/stream-proxy.tf index 46fe433ea0..4409704d90 100644 --- a/terraform/shared/modules/stream-proxy/stream-proxy.tf +++ b/terraform/shared/modules/stream-proxy/stream-proxy.tf @@ -59,7 +59,7 @@ resource "cloudfoundry_app" "egress_app" { } } -### +### ### Set up network policies so that the clients can reach the proxy ### @@ -83,7 +83,7 @@ resource "cloudfoundry_network_policy" "client_routing" { } } -### +### ### Create a credential service for bound clients to use when make requests of the proxy ### diff --git a/terraform/staging/staging.tf-example b/terraform/staging/staging.tf-example index 6717bc4fa7..17363eff8b 100644 --- a/terraform/staging/staging.tf-example +++ b/terraform/staging/staging.tf-example @@ -1,3 +1,12 @@ +# The content of this file is managed by Terraform. If you modify it, it may +# be reverted the next time Terraform runs. If you want to make changes, do it +# in ../meta/bootstrap-env/templates. + +# Add resources to this module describing what you want in the corresponding +# space in cloud.gov. You should probably just reference a shared module as in the +# example below to keep consistency across spaces, but you can also vary the +# content for each environment as needed. + module "staging" { source = "../shared/modules/env" cf_space_name = "staging"