From fd2e7e711d96a859ee77bf3d3bac0fe45d92cf40 Mon Sep 17 00:00:00 2001 From: Andrew Marcum <123010092+abmarcum@users.noreply.github.com> Date: Thu, 2 Mar 2023 18:31:05 -0600 Subject: [PATCH] Added GKE Autopilot Cluster support (#91) * Added GKE Autopilot group * Increased Cloud Build timeout for Autopilot clusters * Updated Agones modules to 1.30.0 release * Updated README, cleaned up Endpoint deploy, and added Helm values required for Autopilot * Applied suggested changes & fixed open match errors --------- Co-authored-by: Mark Mandel --- README.md | 2 +- infrastructure/agones-gke.tf | 35 ++++++++-- infrastructure/allocation-endpoint.tf | 29 ++++---- .../files/agones/ae-lb-ip-patch.yaml.tpl | 3 +- infrastructure/open-match.tf | 10 +-- infrastructure/pipelines.tf | 12 ++-- infrastructure/terraform.tfvars.sample | 24 +++---- infrastructure/variables.tf | 9 ++- platform/agones/{install => }/cloudbuild.yaml | 1 + .../agones/endpoint-patch/cloudbuild.yaml | 66 ------------------- platform/agones/{install => }/skaffold.yaml | 0 11 files changed, 74 insertions(+), 117 deletions(-) rename platform/agones/{install => }/cloudbuild.yaml (98%) delete mode 100644 platform/agones/endpoint-patch/cloudbuild.yaml rename platform/agones/{install => }/skaffold.yaml (100%) diff --git a/README.md b/README.md index 3ace1f3..41d3ba2 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ The Agones deployment is in two steps: The Initial Install and the Allocation En Replace the` _RELEASE_NAME` substitution with a unique build name. Cloudbuild will deploy Agones using Cloud Deploy. ```shell -cd $GAME_DEMO_HOME/platform/agones/install +cd $GAME_DEMO_HOME/platform/agones/ gcloud builds submit --config=cloudbuild.yaml --substitutions=_RELEASE_NAME=rel-1 ``` diff --git a/infrastructure/agones-gke.tf b/infrastructure/agones-gke.tf index 655d8bf..bc337b9 100644 --- a/infrastructure/agones-gke.tf +++ b/infrastructure/agones-gke.tf @@ -17,15 +17,15 @@ ##------------------------------## data "google_container_engine_versions" "regions" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) location = each.value.region } -module "agones_gke_clusters" { - for_each = var.game_gke_clusters +module "agones_gke_standard_clusters" { + for_each = var.game_gke_standard_clusters - source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=main" + source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=v1.30.0" cluster = { name = each.key @@ -46,13 +46,34 @@ module "agones_gke_clusters" { depends_on = [google_compute_subnetwork.subnet, google_project_service.project] } -data "google_container_cluster" "game-demo-agones-gke" { - for_each = var.game_gke_clusters +module "agones_gke_autopilot_clusters" { + for_each = var.game_gke_autopilot_clusters + + source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke-autopilot/?ref=v1.30.0" + + cluster = { + name = each.key + location = each.value.region + project = var.project + + # Install Current GKE default version + kubernetesVersion = data.google_container_engine_versions.regions[each.key].default_cluster_version + + network = google_compute_network.vpc.id + subnetwork = "global-game-${each.value.region}-subnet" + } + udpFirewall = false + + depends_on = [google_compute_subnetwork.subnet, google_project_service.project] +} + +data "google_container_cluster" "game-demo-agones" { + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) name = each.key location = each.value.region - depends_on = [module.agones_gke_clusters] + depends_on = [module.agones_gke_standard_clusters, module.agones_gke_autopilot_clusters] } resource "google_compute_firewall" "agones-gameservers" { diff --git a/infrastructure/allocation-endpoint.tf b/infrastructure/allocation-endpoint.tf index 512ef03..8ccba74 100644 --- a/infrastructure/allocation-endpoint.tf +++ b/infrastructure/allocation-endpoint.tf @@ -23,7 +23,7 @@ resource "random_string" "endpoint_random_string" { } resource "google_endpoints_service" "endpoints_service" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) service_name = "${each.key}-${random_string.endpoint_random_string.result}.endpoints.${var.project}.cloud.goog" grpc_config = templatefile( "${path.module}/files/agones/api_config.yaml.tpl", { @@ -35,7 +35,7 @@ resource "google_endpoints_service" "endpoints_service" { } resource "google_endpoints_service_iam_binding" "endpoints_service_binding" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) service_name = google_endpoints_service.endpoints_service[each.key].service_name role = "roles/servicemanagement.serviceController" @@ -53,7 +53,7 @@ resource "google_service_account_iam_binding" "workload-identity-binding" { "serviceAccount:${var.project}.svc.id.goog[${var.allocation_endpoint.agones_namespace}/agones-allocator]", ] - depends_on = [module.agones_gke_clusters] + depends_on = [module.agones_gke_standard_clusters, module.agones_gke_autopilot_clusters] } resource "google_service_account" "ae_sa" { @@ -66,7 +66,7 @@ resource "google_service_account_key" "ae_sa_key" { } resource "google_cloud_run_service_iam_binding" "binding" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) service = google_cloud_run_service.aep_cloud_run[each.key].name project = google_cloud_run_service.aep_cloud_run[each.key].project @@ -79,9 +79,8 @@ resource "google_cloud_run_service_iam_binding" "binding" { ] } - resource "google_cloud_run_service" "aep_cloud_run" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) project = var.project name = "allocation-endpoint-proxy-${each.key}" @@ -97,7 +96,7 @@ resource "google_cloud_run_service" "aep_cloud_run" { name = "CLUSTERS_INFO" value = templatefile( "${path.module}/files/agones/clusters_info.tpl", { - name = data.google_container_cluster.game-demo-agones-gke[each.key].name + name = data.google_container_cluster.game-demo-agones[each.key].name ip = google_compute_address.allocation-endpoint[each.key].address weight = var.allocation_endpoint.weight namespace = var.allocation_endpoint.agones_namespace @@ -196,7 +195,7 @@ resource "google_secret_manager_secret_iam_member" "secret-access" { } resource "google_project_service" "allocator-service" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) service = google_endpoints_service.endpoints_service[each.key].id disable_dependent_services = true @@ -204,7 +203,7 @@ resource "google_project_service" "allocator-service" { resource "google_compute_address" "allocation-endpoint" { project = var.project - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) region = each.value.region provider = google-beta @@ -219,14 +218,14 @@ resource "google_compute_address" "allocation-endpoint" { resource "local_file" "agones-skaffold-file" { content = templatefile( "${path.module}/files/agones/skaffold.yaml.tpl", { - gke_clusters = var.game_gke_clusters + gke_clusters = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) }) - filename = "${path.module}/${var.platform_directory}/agones/install/skaffold.yaml" + filename = "${path.module}/${var.platform_directory}/agones/skaffold.yaml" } # Make cluster specific helm value for LB IP resource "local_file" "agones-ae-lb-file" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) content = templatefile( "${path.module}/files/agones/ae-lb-ip-patch.yaml.tpl", { @@ -235,13 +234,13 @@ resource "local_file" "agones-ae-lb-file" { sa_email = google_service_account.ae_sa.email location = each.value.region }) - filename = "${path.module}/${var.platform_directory}/agones/install/${each.key}/kustomization.yaml" + filename = "${path.module}/${var.platform_directory}/agones/${each.key}/kustomization.yaml" } # Create agones-system ns manifest as resource referenced by kustomization.yaml resource "local_file" "agones-ns-file" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) content = file("${path.module}/files/agones/agones-system.yaml") - filename = "${path.module}/${var.platform_directory}/agones/install/${each.key}/agones-system.yaml" + filename = "${path.module}/${var.platform_directory}/agones/${each.key}/agones-system.yaml" } diff --git a/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl b/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl index 0b3498a..2df8ba4 100644 --- a/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl +++ b/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl @@ -1,13 +1,14 @@ helmCharts: - name: agones repo: https://agones.dev/chart/stable - version: 1.29.0 + version: 1.30.0 releaseName: agones namespace: agones-system valuesInline: agones: crds: cleanupOnDelete: false + featureGates: "SplitControllerAndExtensions=true" allocator: disableMTLS: true disableTLS: true diff --git a/infrastructure/open-match.tf b/infrastructure/open-match.tf index 3018d20..53157af 100644 --- a/infrastructure/open-match.tf +++ b/infrastructure/open-match.tf @@ -13,10 +13,6 @@ // limitations under the License. -data "google_compute_network" "vpc" { - name = var.vpc_name -} - resource "google_redis_instance" "open-match" { name = "global-game-open-match" tier = "STANDARD_HA" @@ -26,7 +22,7 @@ resource "google_redis_instance" "open-match" { location_id = "${var.services_gke_config.location}-a" alternative_location_id = "${var.services_gke_config.location}-f" - authorized_network = data.google_compute_network.vpc.id + authorized_network = google_compute_network.vpc.id transit_encryption_mode = "DISABLED" connect_mode = "PRIVATE_SERVICE_ACCESS" @@ -45,13 +41,13 @@ resource "google_compute_global_address" "private_service_range" { purpose = "VPC_PEERING" address_type = "INTERNAL" prefix_length = 16 - network = data.google_compute_network.vpc.id + network = google_compute_network.vpc.id depends_on = [google_project_service.project] } resource "google_service_networking_connection" "private_service_connection" { - network = data.google_compute_network.vpc.id + network = google_compute_network.vpc.id service = "servicenetworking.googleapis.com" reserved_peering_ranges = [google_compute_global_address.private_service_range.name] diff --git a/infrastructure/pipelines.tf b/infrastructure/pipelines.tf index 9864905..b6fa5a4 100644 --- a/infrastructure/pipelines.tf +++ b/infrastructure/pipelines.tf @@ -54,8 +54,8 @@ resource "google_clouddeploy_delivery_pipeline" "services_pipeline" { ##### Agones Pipelines ##### -resource "google_clouddeploy_target" "agones" { - for_each = var.game_gke_clusters +resource "google_clouddeploy_target" "agones-gke" { + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) location = var.clouddeploy_config.location name = "${each.value.short_name}-agones-deploy" @@ -70,7 +70,7 @@ resource "google_clouddeploy_target" "agones" { description = "Global Game: Agones Deploy Target - ${each.key}" gke { - cluster = data.google_container_cluster.game-demo-agones-gke[each.key].id + cluster = data.google_container_cluster.game-demo-agones[each.key].id } labels = { @@ -83,7 +83,7 @@ resource "google_clouddeploy_target" "agones" { depends_on = [google_project_service.project] } -resource "google_clouddeploy_delivery_pipeline" "agones" { +resource "google_clouddeploy_delivery_pipeline" "agones-gke" { location = var.clouddeploy_config.location name = "agones-deploy-pipeline" @@ -103,9 +103,9 @@ resource "google_clouddeploy_delivery_pipeline" "agones" { serial_pipeline { dynamic "stages" { - for_each = var.game_gke_clusters + for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters) content { - target_id = google_clouddeploy_target.agones[stages.key].target_id + target_id = google_clouddeploy_target.agones-gke[stages.key].target_id profiles = [stages.key] } } diff --git a/infrastructure/terraform.tfvars.sample b/infrastructure/terraform.tfvars.sample index 1d459b9..39dcdcb 100644 --- a/infrastructure/terraform.tfvars.sample +++ b/infrastructure/terraform.tfvars.sample @@ -42,36 +42,36 @@ vpc_regions = { # Game GKE Cluster Config Values ### NOTE: If you change the GKE Clusters, please make sure to change `cloudbuild.yaml` in ### `platform/agones/install` as they are not dynamically created. -game_gke_clusters = { +game_gke_standard_clusters = { "global-game-us-central1-01" : { "short_name" : "us-central1-01", "region" : "us-central1", "machine_type" : "e2-standard-4" }, - "global-game-us-central1-02" : { - "short_name" : "us-central1-02", - "region" : "us-central1", - "machine_type" : "e2-standard-4" - }, "global-game-eu-west1-01" : { "short_name" : "europe-west1-01", "region" : "europe-west1", "machine_type" : "e2-standard-4" }, - "global-game-eu-west1-02" : { - "short_name" : "europe-west1-02", - "region" : "europe-west1", - "machine_type" : "e2-standard-4" - }, "global-game-asia-east1-01" : { "short_name" : "asia-east1-01", "region" : "asia-east1", "machine_type" : "e2-standard-4" + } +} + +game_gke_autopilot_clusters = { + "global-game-us-central1-02" : { + "short_name" : "us-central1-02", + "region" : "us-central1", + }, + "global-game-eu-west1-02" : { + "short_name" : "europe-west1-02", + "region" : "europe-west1", }, "global-game-asia-east1-02" : { "short_name" : "asia-east1-02", "region" : "asia-east1", - "machine_type" : "e2-standard-4" } } diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index 8793264..bf51852 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -44,9 +44,14 @@ variable "vpc_regions" { ### Agones Variables ### -variable "game_gke_clusters" { +variable "game_gke_standard_clusters" { type = map(any) - description = "GKE gameclusters & associated values" + description = "GKE Standard Game Clusters & Associated values" +} + +variable "game_gke_autopilot_clusters" { + type = map(any) + description = "GKE Autopilot Game Clusters & Associated values" } ### Cloud Deploy Variables ### diff --git a/platform/agones/install/cloudbuild.yaml b/platform/agones/cloudbuild.yaml similarity index 98% rename from platform/agones/install/cloudbuild.yaml rename to platform/agones/cloudbuild.yaml index b2d37ea..0758ff2 100644 --- a/platform/agones/install/cloudbuild.yaml +++ b/platform/agones/cloudbuild.yaml @@ -24,6 +24,7 @@ steps: "--skaffold-version", "1.39", "--region", "us-central1" ] + timeout: 1800s substitutions: _RELEASE_NAME: rel-0001 diff --git a/platform/agones/endpoint-patch/cloudbuild.yaml b/platform/agones/endpoint-patch/cloudbuild.yaml deleted file mode 100644 index a8cf6ca..0000000 --- a/platform/agones/endpoint-patch/cloudbuild.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2023 Google LLC All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -serviceAccount: projects/${PROJECT_ID}/serviceAccounts/cloudbuild-cicd@${PROJECT_ID}.iam.gserviceaccount.com -steps: - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-asia-east1-01.yaml - - --location=asia-east1 - - --cluster=global-game-asia-east1-01 - - --output=output/agones-patch-asia-east1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-asia-east1-02.yaml - - --location=asia-east1 - - --cluster=global-game-asia-east1-02 - - --output=output/agones-patch-asia-east1-02 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-eu-west1-01.yaml - - --location=europe-west1 - - --cluster=global-game-eu-west1-01 - - --output=output/agones-patch-europe-west1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-eu-west1-02.yaml - - --location=europe-west1 - - --cluster=global-game-eu-west1-02 - - --output=output/agones-patch-europe-west1-02 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-us-central1-01.yaml - - --location=us-central1 - - --cluster=global-game-us-central1-01 - - --output=output/agones-patch-us-central1-01 - - - name: "gcr.io/cloud-builders/gke-deploy" - args: - - run - - --filename=patch-agones-allocator-global-game-us-central1-02.yaml - - --location=us-central1 - - --cluster=global-game-us-central1-02 - - --output=output/agones-patch-us-central1-02 - -options: - logging: CLOUD_LOGGING_ONLY diff --git a/platform/agones/install/skaffold.yaml b/platform/agones/skaffold.yaml similarity index 100% rename from platform/agones/install/skaffold.yaml rename to platform/agones/skaffold.yaml