Merge pull request #3 from kinfinity/fix/pipeline-builds

Fix Pipeline Builds

.github/workflows/buildandpush-app.yaml

@@ -38,6 +38,12 @@ jobs:
 
     images:
         name: ImageBuild
+        permissions:
+          contents: read
+          issues: read
+          checks: write
+          pull-requests: write
+          id-token: 'write'
         needs: build_apps
         runs-on: ubuntu-latest
         environment: Dev
@@ -49,9 +55,8 @@ jobs:
             - name: Authenticate with Google Cloud
               uses: 'google-github-actions/auth@v2'
               with:
-                project_id: ${{ secrets.PROJECT_ID }}
                 workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITYPROVIDER }}
-                credentials_json:  ${{ secrets.GCP_CREDENTIALS }}
+                service_account: ${{ secrets.SERVICE_ACCOUNT }}
 
             - name: Build and Push Image
               run: |

.github/workflows/infra-release.yaml

@@ -8,6 +8,12 @@ on:
 
 jobs:
   terraform-release:
+    permissions:
+      contents: read
+      issues: read
+      checks: write
+      pull-requests: write
+      id-token: 'write'
     name: "Terraform Release"
     runs-on: ubuntu-latest
     environment: Dev
@@ -27,12 +33,12 @@ jobs:
           fi
 
           # Check if the extracted directory exists before deleting
-          if [ -d "terraform" ]; then
-            sudo rm -rf terraform
+          if [ -d "terraform_"$tf_version"_linux_amd64" ]; then
+            sudo rm -rf terraform_"$tf_version"_linux_amd64
           fi
 
-          sudo unzip -o terraform_"$tf_version"_linux_amd64.zip
-          sudo mv terraform /usr/local/bin/
+          sudo unzip -o terraform_"$tf_version"_linux_amd64.zip -d terraform_install
+          sudo mv terraform_install/terraform /usr/local/bin/
 
       - name: Verify Terraform version
         run: terraform --version
@@ -41,18 +47,18 @@ jobs:
       - name: Authenticate with Google Cloud
         uses: 'google-github-actions/auth@v2'
         with:
-          project_id: ${{ secrets.PROJECT_ID }}
           workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITYPROVIDER }}
-          credentials_json:  ${{ secrets.GCP_CREDENTIALS }}
+          service_account: ${{ secrets.SERVICE_ACCOUNT }}
 
       - name: Terraform Plan
         if: ${{ github.event_name == 'pull_request' }}
         run: |
           chmod +x ci/scripts/*.sh
-          python ci/scripts/execute-terraform.py  --config ci/configs/pipeline-config.json --command plan --env dev
+          ci/scripts/generate_tfvars.sh terraform/environments/dev ${{ secrets.PROJECT_ID }} dev
+          python3 ci/scripts/execute-terraform.py  --config ci/configs/pipeline-config.json --command plan --env dev
 
       - name: Terraform Apply
         if: ${{ github.event_name != 'pull_request' }}
         run: |
           chmod +x ci/scripts/*.sh
-          python ci/scripts/execute-terraform.py  --config ci/configs/pipeline-config.json --command apply --env dev
+          python3 ci/scripts/execute-terraform.py  --config ci/configs/pipeline-config.json --command apply --env dev

.github/workflows/k8s-apps-release.yaml

@@ -12,6 +12,12 @@ env:
 jobs:
     deploy:
         name: DeployApps
+        permissions:
+          contents: read
+          issues: read
+          checks: write
+          pull-requests: write
+          id-token: write
         runs-on: ubuntu-latest
         environment: Dev
         steps:
@@ -22,16 +28,22 @@ jobs:
             - name: Authenticate with Google Cloud
               uses: 'google-github-actions/auth@v2'
               with:
-                project_id: ${{ secrets.PROJECT_ID }}
                 workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITYPROVIDER }}
-                credentials_json:  ${{ secrets.GCP_CREDENTIALS }}
+                service_account: ${{ secrets.SERVICE_ACCOUNT }}
 
             - name: Configure kubectl
-              uses: google-github-actions/setup-gcloud@v0
+              uses: google-github-actions/setup-gcloud@v2
               with:
                 project_id: ${{ secrets.PROJECT_ID }}
-                service_account_key: ${{ secrets.GKE_SA_KEY }}
 
+            - name: Cluster Connection
+              run: |
+                gcloud components install gke-gcloud-auth-plugin
+                export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+                gcloud container clusters get-credentials cohere-main-gke --region=us-central1
+                kubectl config get-contexts
+                kubectl config current-context
+
             - name: Deploy App w Database
               run: |
                 kubectl apply -f manifests/cohere-app.yaml

ci/scripts/cmd-terraform-generic.sh

@@ -27,7 +27,7 @@ terraform workspace list
 terraform workspace select $WORKSPACE
 
 echo "configured workspace: ${ENV_NAME}/${WORKSPACE}"
-
+ls
 #  Run the provided command (e.g., plan or apply) with .tfvars
 if [[ $TERRAFORM_CMD == *"plan"* ]]; then
     terraform $TERRAFORM_CMD -var-file=variables.tfvars

ci/scripts/generate_tfvars.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -eu
+
+# TODO: should be able to parse the json into terraform map
+USAGE="USAGE:
+${0} <terraform-env-directory> <project-id> <env-name>"
+
+if [[ $# -ne 3 ]]; then
+    echo "${USAGE}" >&2
+    exit 1
+fi
+
+# Get absolute path of terraform dir
+TERRAFORM_ENV_DIR="$(cd "${1}"; pwd -P)"
+
+pushd ${TERRAFORM_ENV_DIR} > /dev/null
+
+cat > variables.tfvars << EOF
+    main_region                 =   "us-central1"
+    gke_name                    =   "cohere-main-gke"
+    gke_node_count              =   2
+    project_id                  =   "${2}"
+    env_name                    =   "${3}"
+    network                     =   "default"
+    subnetwork                  =   "default"
+EOF

terraform/environments/dev/provider.tf

@@ -1,5 +1,5 @@
 provider "google" {
   project = var.project_id
   region  = var.main_region
-  credentials = "${file(var.credentials_path)}"
+  # credentials = "${file(var.credentials_path)}"
 }

terraform/environments/dev/variables.tf

@@ -6,4 +6,4 @@ variable "project_id"{}
 variable "env_name" {}
 variable "network"{}
 variable "subnetwork" {}
-variable "credentials_path" {}
+# variable "credentials_path" {}