From ba3f0cd1b8ac24a49060e88032b58d9c77ce6d14 Mon Sep 17 00:00:00 2001 From: Benjamin Fogel Date: Tue, 21 Feb 2023 21:56:07 -0600 Subject: [PATCH] Move Agones deployments into a single deployment pipeline (#58) * Move Agones deployments into a single deployment pipeline * Add Cloud Deploy promote instructions for Agones * Deploy agones-system namespace with Cloud Deploy * Include gcloud command for promoting Cloud Deploy pipeline --- .gitignore | 2 + README.md | 9 +++ infrastructure/allocation-endpoint.tf | 13 ++-- .../deploy/agones/install/cloudbuild.yaml | 67 +------------------ .../files/agones/ae-lb-ip-patch.yaml.tpl | 3 + .../agones}/agones-system.yaml | 0 infrastructure/files/agones/skaffold.yaml.tpl | 10 ++- infrastructure/pipelines.tf | 18 ++--- infrastructure/variables.tf | 8 +++ 9 files changed, 53 insertions(+), 77 deletions(-) rename infrastructure/{deploy/agones/install => files/agones}/agones-system.yaml (100%) diff --git a/.gitignore b/.gitignore index 6129e93..c56df6c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ patch-agones-allocator-*.yaml # Ignore Terraform generated skaffold files skaffold-global-game-*.yaml +infrastructure/deploy/agones/install/skaffold.yaml # Ignore Terraform generated kustomization files **/global-game-*/kustomization.yaml +**/global-game-*/agones-system.yaml diff --git a/README.md b/README.md index 67944a1..049ed18 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,15 @@ cd $GAME_DEMO_HOME/infrastructure/deploy/agones/install gcloud builds submit --config=cloudbuild.yaml --substitutions=_RELEASE_NAME=rel-1 ``` +Navigate to the [agones-deploy-pipeline](https://console.cloud.google.com/deploy/delivery-pipelines/us-central1/agones-deploy-pipeline) delivery pipeline to review the rollout status. Cloudbuild will create a Cloud Deploy release which automatically deploys Agones to the first game server cluster. Agones can be deployed to subsequent clusters by clicking on the `promote` button within the Pipeline visualization or by running the following gcloud command: + +```shell +# Replace RELEASE_NAME with the unique build name +$ gcloud deploy releases promote --release=RELEASE_NAME --delivery-pipeline=agones-deploy-pipeline --region=us-central1` +``` + +Continue the promotion until Agones has been deployed to all clusters. + You can monitor the status of the deployment through the Cloud Logging URL returned by the `gcloud builds` command as well as the Kubernetes Engine/Worloads panel in the GCP Console. Once the Worloads have been marked as OK, you can proceed to apply the Allocation Endpoint Patch. #### Allocation Endpoint Patch diff --git a/infrastructure/allocation-endpoint.tf b/infrastructure/allocation-endpoint.tf index 07269ee..b5d8823 100644 --- a/infrastructure/allocation-endpoint.tf +++ b/infrastructure/allocation-endpoint.tf @@ -193,13 +193,11 @@ resource "google_compute_address" "allocation-endpoint" { # Make Skaffold file for Cloud Deploy into each GKE Cluster resource "local_file" "agones-skaffold-file" { - for_each = var.game_gke_clusters - content = templatefile( "${path.module}/files/agones/skaffold.yaml.tpl", { - cluster_name = each.key + gke_clusters = var.game_gke_clusters }) - filename = "${path.module}/deploy/agones/install/skaffold-${each.key}.yaml" + filename = "${path.module}/deploy/agones/install/skaffold.yaml" } # Make cluster specific helm value for LB IP @@ -213,6 +211,13 @@ resource "local_file" "agones-ae-lb-file" { filename = "${path.module}/deploy/agones/install/${each.key}/kustomization.yaml" } +# Create agones-system ns manifest as resource referenced by kustomization.yaml +resource "local_file" "agones-ns-file" { + for_each = var.game_gke_clusters + + content = file("${path.module}/files/agones/agones-system.yaml") + filename = "${path.module}/deploy/agones/install/${each.key}/agones-system.yaml" +} # Make Kubernetes manifest files to patch the Agones deployment for Allocation Endpoint resource "local_file" "patch-agones-manifest" { diff --git a/infrastructure/deploy/agones/install/cloudbuild.yaml b/infrastructure/deploy/agones/install/cloudbuild.yaml index 2eed4b8..b2d37ea 100644 --- a/infrastructure/deploy/agones/install/cloudbuild.yaml +++ b/infrastructure/deploy/agones/install/cloudbuild.yaml @@ -14,75 +14,14 @@ serviceAccount: projects/${PROJECT_ID}/serviceAccounts/cloudbuild-cicd@${PROJECT_ID}.iam.gserviceaccount.com steps: - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=agones-system.yaml - - --location=asia-east1 - - --cluster=global-game-asia-east1-01 - - --output=output/agones-deploy-asia-east1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=agones-system.yaml - - --location=europe-west1 - - --cluster=global-game-eu-west1-01 - - --output=output/agones-deploy-europe-west1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=agones-system.yaml - - --location=us-central1 - - --cluster=global-game-us-central1-01 - - --output=output/agones-deploy-us-central1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=agones-system.yaml - - --location=us-central1 - - --cluster=global-game-us-central1-02 - - --output=output/agones-deploy-us-central1-02 - - - name: gcr.io/google.com/cloudsdktool/cloud-sdk - entrypoint: gcloud - args: - [ - "deploy", "releases", "create", "${_RELEASE_NAME}", - "--delivery-pipeline", "agones-deploy-pipeline-global-game-asia-east1-01", - "--skaffold-file", "skaffold-global-game-asia-east1-01.yaml", - "--region", "asia-east1" - ] - - - name: gcr.io/google.com/cloudsdktool/cloud-sdk - entrypoint: gcloud - args: - [ - "deploy", "releases", "create", "${_RELEASE_NAME}", - "--delivery-pipeline", "agones-deploy-pipeline-global-game-eu-west1-01", - "--skaffold-file", "skaffold-global-game-eu-west1-01.yaml", - "--region", "europe-west1" - ] - - - name: gcr.io/google.com/cloudsdktool/cloud-sdk - entrypoint: gcloud - args: - [ - "deploy", "releases", "create", "${_RELEASE_NAME}", - "--delivery-pipeline", "agones-deploy-pipeline-global-game-us-central1-01", - "--skaffold-file", "skaffold-global-game-us-central1-01.yaml", - "--region", "us-central1" - ] - - name: gcr.io/google.com/cloudsdktool/cloud-sdk entrypoint: gcloud args: [ "deploy", "releases", "create", "${_RELEASE_NAME}", - "--delivery-pipeline", "agones-deploy-pipeline-global-game-us-central1-02", - "--skaffold-file", "skaffold-global-game-us-central1-02.yaml", + "--delivery-pipeline", "agones-deploy-pipeline", + "--skaffold-file", "skaffold.yaml", + "--skaffold-version", "1.39", "--region", "us-central1" ] diff --git a/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl b/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl index 9ee996a..84ed6aa 100644 --- a/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl +++ b/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl @@ -13,3 +13,6 @@ helmCharts: http: enabled: false loadBalancerIP: "${lb_ip}" + +resources: +- agones-system.yaml \ No newline at end of file diff --git a/infrastructure/deploy/agones/install/agones-system.yaml b/infrastructure/files/agones/agones-system.yaml similarity index 100% rename from infrastructure/deploy/agones/install/agones-system.yaml rename to infrastructure/files/agones/agones-system.yaml diff --git a/infrastructure/files/agones/skaffold.yaml.tpl b/infrastructure/files/agones/skaffold.yaml.tpl index c736b47..3516678 100644 --- a/infrastructure/files/agones/skaffold.yaml.tpl +++ b/infrastructure/files/agones/skaffold.yaml.tpl @@ -3,7 +3,15 @@ kind: Config deploy: kustomize: paths: - - "./${cluster_name}" + - "./path_to_cluster_specific_skaffold" buildArgs: ["--enable-helm"] flags: apply: ['--server-side'] # Avoid the "Too long: must have at most 262144 bytes" problem +profiles: +%{ for cluster_name, values in gke_clusters ~} +- name: ${cluster_name} + patches: + - op: replace + path: /deploy/kustomize/paths/0 + value: "./${cluster_name}" +%{ endfor ~} \ No newline at end of file diff --git a/infrastructure/pipelines.tf b/infrastructure/pipelines.tf index 054ce61..6fdeb55 100644 --- a/infrastructure/pipelines.tf +++ b/infrastructure/pipelines.tf @@ -74,7 +74,7 @@ resource "google_clouddeploy_delivery_pipeline" "spanner" { resource "google_clouddeploy_target" "agones" { for_each = var.game_gke_clusters - location = each.value.region + location = var.clouddeploy_config.location name = "agones-deploy-target-${each.key}" annotations = { @@ -102,10 +102,8 @@ resource "google_clouddeploy_target" "agones" { } resource "google_clouddeploy_delivery_pipeline" "agones" { - for_each = var.game_gke_clusters - - location = each.value.region - name = "agones-deploy-pipeline-${each.key}" + location = var.clouddeploy_config.location + name = "agones-deploy-pipeline" annotations = { my_first_annotation = "agones-annotation-1" @@ -113,7 +111,7 @@ resource "google_clouddeploy_delivery_pipeline" "agones" { my_second_annotation = "agones-annotation-2" } - description = "Global Game: Agones Deploy Pipeline - ${each.key}" + description = "Global Game: Agones Deploy Pipeline" labels = { my_first_label = "global-game-demo" @@ -124,8 +122,12 @@ resource "google_clouddeploy_delivery_pipeline" "agones" { project = var.project serial_pipeline { - stages { - target_id = google_clouddeploy_target.agones[each.key].target_id + dynamic "stages" { + for_each = var.game_gke_clusters + content { + target_id = google_clouddeploy_target.agones[stages.key].target_id + profiles = [stages.key] + } } } } diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index 78a0740..bc0b0ea 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -44,6 +44,14 @@ variable "game_gke_clusters" { description = "GKE gameclusters & associated values" } +### Cloud Deploy Variables ### + +variable "clouddeploy_config" { + type = object({ + location = string + }) +} + ### Spanner Variables ### variable "spanner_config" {