Merge pull request #4018 from GSA-TTS/main

GSA-TTS · Jun 25, 2024 · f82ad7e · f82ad7e
2 parents 5d88c5b + e00d9db
commit f82ad7e
Show file tree

Hide file tree

Showing 33 changed files with 656 additions and 622 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,7 +1,7 @@
-# This file is sensitive since rules below may be used for restricting who can 
-# make access control changes.
-/.github/CODEOWNERS @GSA-TTS/FAC-admins
-
-
-# Changes to the following Terraform directory will impact access control in cloud.gov spaces. Any PR involving these files should get a review from someone in FAC-admins.
-/terraform/meta/ @GSA-TTS/FAC-admins
+# This file is sensitive since rules below may be used for restricting who can
+# make access control changes.
+/.github/CODEOWNERS @GSA-TTS/FAC-admins
+/.github/workflows/fac-restore-util.yml @GSA-TTS/FAC-admins @asteel-gsa
+
+# Changes to the following Terraform directory will impact access control in cloud.gov spaces. Any PR involving these files should get a review from someone in FAC-admins.
+/terraform/meta/ @GSA-TTS/FAC-admins
diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml
@@ -1,148 +1,128 @@
----
-name: Deploy application to cloud.gov
-on:
-  workflow_call:
-    inputs:
-      environment:
-        required: true
-        type: string
-
-jobs:
-  push-with-creds:
-    name: Deploy to cloud.gov with updated credentials
-    runs-on: ubuntu-latest
-    environment: ${{ inputs.environment }}
-    env:
-      space: ${{ inputs.environment }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Restore npm cache
-        uses: actions/cache@v4
-        id: cache-npm
-        with:
-          path: ~/.npm
-          key: fac-build-npm-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            fac-build-npm-
-            fac-build-
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Restore pip cache
-        uses: actions/cache@v4
-        id: cache-pip
-        with:
-          path: |
-            ~/.cache/pip
-            /opt/hostedtoolcache/Python/
-          key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }}
-          restore-keys: |
-            fac-build-pip-${{ hashFiles('**/requirements.txt') }}-
-            fac-build-pip-
-            fac-build-
-
-      - name: Install npm dependencies
-        working-directory: ./backend
-        run: npm ci --production
-
-      - name: Compile JS/CSS assets
-        working-directory: ./backend
-        run: npm run build
-
-      - name: Update service keys
-        uses: cloud-gov/cg-cli-tools@main
-        env:
-          SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
-          DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
-          LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }}
-          SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"'
-
-      - name: Bind backup s3 bucket to prod app
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf bind-service gsa-fac backups
-
-      - name: Backup the database (Prod Only)
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 2G -m 2G --name pg_backup --command "./backup_database.sh ${{ env.space }}"
-
-      - name: Deploy Preview to cloud.gov
-        if: ${{ inputs.environment == 'preview' }}
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_manifest: backend/manifests/manifest-preview.yml
-          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
-          command: bin/ops/deploy_preview.sh
-
-      - name: Deploy fac to cloud.gov
-        if: ${{ inputs.environment  != 'preview' }}
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          cf_manifest: backend/manifests/manifest-fac.yml
-          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
-          command: bin/ops/deploy.sh
-
-      - name: Unbind backup s3 bucket from prod app
-        if: startsWith(github.ref, 'refs/tags/v1.')
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf unbind-service gsa-fac backups
-
-      - name: Load historical data
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
-
-      # This has to happen after an application deployment because the manifest (currently) is responsible
-      # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done
-      # based on the suspicion that fluentbit cannot register the incoming logs when it is initially
-      # created, resulting in a 502. Restarting the application after everything is configured results
-      # in a 201, or, the expected status when transmitting logs.
-      - name: Restart the logshipper application
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf restart logshipper
-
+---
+name: Deploy application to cloud.gov
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+
+jobs:
+  push-with-creds:
+    name: Deploy to cloud.gov with updated credentials
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Restore npm cache
+        uses: actions/cache@v4
+        id: cache-npm
+        with:
+          path: ~/.npm
+          key: fac-build-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            fac-build-npm-
+            fac-build-
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Restore pip cache
+        uses: actions/cache@v4
+        id: cache-pip
+        with:
+          path: |
+            ~/.cache/pip
+            /opt/hostedtoolcache/Python/
+          key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }}
+          restore-keys: |
+            fac-build-pip-${{ hashFiles('**/requirements.txt') }}-
+            fac-build-pip-
+            fac-build-
+
+      - name: Install npm dependencies
+        working-directory: ./backend
+        run: npm ci --production
+
+      - name: Compile JS/CSS assets
+        working-directory: ./backend
+        run: npm run build
+
+      - name: Update service keys
+        uses: cloud-gov/cg-cli-tools@main
+        env:
+          SAM_API_KEY: ${{ secrets.SAM_API_KEY }}
+          DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }}
+          LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"'
+
+      - name: Backup the database
+        # if: startsWith(github.ref, 'refs/tags/v1.')
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.3 deploy_backup"
+
+      - name: Deploy Preview to cloud.gov
+        if: ${{ inputs.environment == 'preview' }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_manifest: backend/manifests/manifest-preview.yml
+          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
+          command: bin/ops/deploy_preview.sh
+
+      - name: Deploy fac to cloud.gov
+        if: ${{ inputs.environment  != 'preview' }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          cf_manifest: backend/manifests/manifest-fac.yml
+          cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
+          command: bin/ops/deploy.sh
+
+      - name: Load historical data
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
+
+      # This has to happen after an application deployment because the manifest (currently) is responsible
+      # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done
+      # based on the suspicion that fluentbit cannot register the incoming logs when it is initially
+      # created, resulting in a 502. Restarting the application after everything is configured results
+      # in a 201, or, the expected status when transmitting logs.
+      - name: Restart the logshipper application
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf restart logshipper
+
diff --git a/.github/workflows/fac-backup-scheduler.yml b/.github/workflows/fac-backup-scheduler.yml
@@ -0,0 +1,24 @@
+---
+name: Backup Environments
+on:
+  schedule:
+    # Invoke every 2 hours
+    - cron: '0 */2 * * *'
+  workflow_dispatch: null
+
+jobs:
+  backup-environment:
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - name: dev
+          - name: staging
+          - name: production
+    uses: ./.github/workflows/fac-backup-util-scheduled.yml
+    secrets: inherit
+    with:
+      environment: ${{ matrix.environment.name }}
+      util_version: "v0.1.3"
+      backup_operation: "scheduled_backup"
+
diff --git a/.github/workflows/fac-backup-util-scheduled.yml b/.github/workflows/fac-backup-util-scheduled.yml
@@ -0,0 +1,36 @@
+---
+name: Backup the database with fac-backup-utility
+### Common Commands:
+# ./fac-backup-util.sh v0.1.3 scheduled_backup
+# ./fac-backup-util.sh v0.1.3 daily_backup
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: string
+jobs:
+  fac-backup-scheduled:
+    name: Perform Database Backup
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Backup FAC Database and Sync Media
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name backup_util_scheduled --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"
+
diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml
@@ -0,0 +1,43 @@
+---
+name: Backup the database with fac-backup-utility
+### Common Commands:
+# ./fac-backup-util.sh v0.1.3 initial_backup
+# ./fac-backup-util.sh v0.1.3 deploy_backup
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+          - 'dev'
+          - 'preview'
+          - 'staging'
+          - 'production'
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: choice
+        options:
+          - 'initial_backup'
+          - 'deploy_backup'
+jobs:
+  fac-backup:
+    name: Perform Database Backup
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Backup FAC Database and Sync Media
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup_util --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"