Added GKE Autopilot Cluster support (#91)

* Added GKE Autopilot group * Increased Cloud Build timeout for Autopilot clusters * Updated Agones modules to 1.30.0 release * Updated README, cleaned up Endpoint deploy, and added Helm values required for Autopilot * Applied suggested changes & fixed open match errors --------- Co-authored-by: Mark Mandel <[email protected]>
googleforgames · Mar 3, 2023 · fd2e7e7 · fd2e7e7
1 parent 1824c76
commit fd2e7e7
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 117 deletions.
diff --git a/README.md b/README.md
@@ -85,7 +85,7 @@ The Agones deployment is in two steps: The Initial Install and the Allocation En
 Replace the` _RELEASE_NAME` substitution with a unique build name. Cloudbuild will deploy Agones using Cloud Deploy.
 
 ```shell
-cd $GAME_DEMO_HOME/platform/agones/install
+cd $GAME_DEMO_HOME/platform/agones/
 gcloud builds submit --config=cloudbuild.yaml --substitutions=_RELEASE_NAME=rel-1
 ```
 

diff --git a/infrastructure/agones-gke.tf b/infrastructure/agones-gke.tf
@@ -17,15 +17,15 @@
 ##------------------------------##
 
 data "google_container_engine_versions" "regions" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   location = each.value.region
 }
 
-module "agones_gke_clusters" {
-  for_each = var.game_gke_clusters
+module "agones_gke_standard_clusters" {
+  for_each = var.game_gke_standard_clusters
 
-  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=main"
+  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=v1.30.0"
 
   cluster = {
     name             = each.key
@@ -46,13 +46,34 @@ module "agones_gke_clusters" {
   depends_on = [google_compute_subnetwork.subnet, google_project_service.project]
 }
 
-data "google_container_cluster" "game-demo-agones-gke" {
-  for_each = var.game_gke_clusters
+module "agones_gke_autopilot_clusters" {
+  for_each = var.game_gke_autopilot_clusters
+
+  source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke-autopilot/?ref=v1.30.0"
+
+  cluster = {
+    name     = each.key
+    location = each.value.region
+    project  = var.project
+
+    # Install Current GKE default version
+    kubernetesVersion = data.google_container_engine_versions.regions[each.key].default_cluster_version
+
+    network    = google_compute_network.vpc.id
+    subnetwork = "global-game-${each.value.region}-subnet"
+  }
+  udpFirewall = false
+
+  depends_on = [google_compute_subnetwork.subnet, google_project_service.project]
+}
+
+data "google_container_cluster" "game-demo-agones" {
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   name     = each.key
   location = each.value.region
 
-  depends_on = [module.agones_gke_clusters]
+  depends_on = [module.agones_gke_standard_clusters, module.agones_gke_autopilot_clusters]
 }
 
 resource "google_compute_firewall" "agones-gameservers" {

diff --git a/infrastructure/allocation-endpoint.tf b/infrastructure/allocation-endpoint.tf
@@ -23,7 +23,7 @@ resource "random_string" "endpoint_random_string" {
 }
 
 resource "google_endpoints_service" "endpoints_service" {
-  for_each     = var.game_gke_clusters
+  for_each     = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
   service_name = "${each.key}-${random_string.endpoint_random_string.result}.endpoints.${var.project}.cloud.goog"
   grpc_config = templatefile(
     "${path.module}/files/agones/api_config.yaml.tpl", {
@@ -35,7 +35,7 @@ resource "google_endpoints_service" "endpoints_service" {
 }
 
 resource "google_endpoints_service_iam_binding" "endpoints_service_binding" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   service_name = google_endpoints_service.endpoints_service[each.key].service_name
   role         = "roles/servicemanagement.serviceController"
@@ -53,7 +53,7 @@ resource "google_service_account_iam_binding" "workload-identity-binding" {
     "serviceAccount:${var.project}.svc.id.goog[${var.allocation_endpoint.agones_namespace}/agones-allocator]",
   ]
 
-  depends_on = [module.agones_gke_clusters]
+  depends_on = [module.agones_gke_standard_clusters, module.agones_gke_autopilot_clusters]
 }
 
 resource "google_service_account" "ae_sa" {
@@ -66,7 +66,7 @@ resource "google_service_account_key" "ae_sa_key" {
 }
 
 resource "google_cloud_run_service_iam_binding" "binding" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   service  = google_cloud_run_service.aep_cloud_run[each.key].name
   project  = google_cloud_run_service.aep_cloud_run[each.key].project
@@ -79,9 +79,8 @@ resource "google_cloud_run_service_iam_binding" "binding" {
   ]
 }
 
-
 resource "google_cloud_run_service" "aep_cloud_run" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   project  = var.project
   name     = "allocation-endpoint-proxy-${each.key}"
@@ -97,7 +96,7 @@ resource "google_cloud_run_service" "aep_cloud_run" {
           name = "CLUSTERS_INFO"
           value = templatefile(
             "${path.module}/files/agones/clusters_info.tpl", {
-              name      = data.google_container_cluster.game-demo-agones-gke[each.key].name
+              name      = data.google_container_cluster.game-demo-agones[each.key].name
               ip        = google_compute_address.allocation-endpoint[each.key].address
               weight    = var.allocation_endpoint.weight
               namespace = var.allocation_endpoint.agones_namespace
@@ -196,15 +195,15 @@ resource "google_secret_manager_secret_iam_member" "secret-access" {
 }
 
 resource "google_project_service" "allocator-service" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   service                    = google_endpoints_service.endpoints_service[each.key].id
   disable_dependent_services = true
 }
 
 resource "google_compute_address" "allocation-endpoint" {
   project  = var.project
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
   region   = each.value.region
   provider = google-beta
 
@@ -219,14 +218,14 @@ resource "google_compute_address" "allocation-endpoint" {
 resource "local_file" "agones-skaffold-file" {
   content = templatefile(
     "${path.module}/files/agones/skaffold.yaml.tpl", {
-      gke_clusters = var.game_gke_clusters
+      gke_clusters = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
   })
-  filename = "${path.module}/${var.platform_directory}/agones/install/skaffold.yaml"
+  filename = "${path.module}/${var.platform_directory}/agones/skaffold.yaml"
 }
 
 # Make cluster specific helm value for LB IP
 resource "local_file" "agones-ae-lb-file" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   content = templatefile(
     "${path.module}/files/agones/ae-lb-ip-patch.yaml.tpl", {
@@ -235,13 +234,13 @@ resource "local_file" "agones-ae-lb-file" {
       sa_email     = google_service_account.ae_sa.email
       location     = each.value.region
   })
-  filename = "${path.module}/${var.platform_directory}/agones/install/${each.key}/kustomization.yaml"
+  filename = "${path.module}/${var.platform_directory}/agones/${each.key}/kustomization.yaml"
 }
 
 # Create agones-system ns manifest as resource referenced by kustomization.yaml
 resource "local_file" "agones-ns-file" {
-  for_each = var.game_gke_clusters
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   content  = file("${path.module}/files/agones/agones-system.yaml")
-  filename = "${path.module}/${var.platform_directory}/agones/install/${each.key}/agones-system.yaml"
+  filename = "${path.module}/${var.platform_directory}/agones/${each.key}/agones-system.yaml"
 }
diff --git a/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl b/infrastructure/files/agones/ae-lb-ip-patch.yaml.tpl
@@ -1,13 +1,14 @@
 helmCharts:
   - name: agones
     repo: https://agones.dev/chart/stable
-    version: 1.29.0
+    version: 1.30.0
     releaseName: agones
     namespace: agones-system
     valuesInline:
       agones:
         crds:
           cleanupOnDelete: false
+        featureGates: "SplitControllerAndExtensions=true"
         allocator:
           disableMTLS: true
           disableTLS: true

diff --git a/infrastructure/open-match.tf b/infrastructure/open-match.tf
@@ -13,10 +13,6 @@
 // limitations under the License.
 
 
-data "google_compute_network" "vpc" {
-  name = var.vpc_name
-}
-
 resource "google_redis_instance" "open-match" {
   name           = "global-game-open-match"
   tier           = "STANDARD_HA"
@@ -26,7 +22,7 @@ resource "google_redis_instance" "open-match" {
   location_id             = "${var.services_gke_config.location}-a"
   alternative_location_id = "${var.services_gke_config.location}-f"
 
-  authorized_network      = data.google_compute_network.vpc.id
+  authorized_network      = google_compute_network.vpc.id
   transit_encryption_mode = "DISABLED"
   connect_mode            = "PRIVATE_SERVICE_ACCESS"
 
@@ -45,13 +41,13 @@ resource "google_compute_global_address" "private_service_range" {
   purpose       = "VPC_PEERING"
   address_type  = "INTERNAL"
   prefix_length = 16
-  network       = data.google_compute_network.vpc.id
+  network       = google_compute_network.vpc.id
 
   depends_on = [google_project_service.project]
 }
 
 resource "google_service_networking_connection" "private_service_connection" {
-  network                 = data.google_compute_network.vpc.id
+  network                 = google_compute_network.vpc.id
   service                 = "servicenetworking.googleapis.com"
   reserved_peering_ranges = [google_compute_global_address.private_service_range.name]
 

diff --git a/infrastructure/pipelines.tf b/infrastructure/pipelines.tf
@@ -54,8 +54,8 @@ resource "google_clouddeploy_delivery_pipeline" "services_pipeline" {
 
 ##### Agones Pipelines #####
 
-resource "google_clouddeploy_target" "agones" {
-  for_each = var.game_gke_clusters
+resource "google_clouddeploy_target" "agones-gke" {
+  for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
 
   location = var.clouddeploy_config.location
   name     = "${each.value.short_name}-agones-deploy"
@@ -70,7 +70,7 @@ resource "google_clouddeploy_target" "agones" {
   description = "Global Game: Agones Deploy Target - ${each.key}"
 
   gke {
-    cluster = data.google_container_cluster.game-demo-agones-gke[each.key].id
+    cluster = data.google_container_cluster.game-demo-agones[each.key].id
   }
 
   labels = {
@@ -83,7 +83,7 @@ resource "google_clouddeploy_target" "agones" {
   depends_on = [google_project_service.project]
 }
 
-resource "google_clouddeploy_delivery_pipeline" "agones" {
+resource "google_clouddeploy_delivery_pipeline" "agones-gke" {
   location = var.clouddeploy_config.location
   name     = "agones-deploy-pipeline"
 
@@ -103,9 +103,9 @@ resource "google_clouddeploy_delivery_pipeline" "agones" {
 
   serial_pipeline {
     dynamic "stages" {
-      for_each = var.game_gke_clusters
+      for_each = merge(var.game_gke_standard_clusters, var.game_gke_autopilot_clusters)
       content {
-        target_id = google_clouddeploy_target.agones[stages.key].target_id
+        target_id = google_clouddeploy_target.agones-gke[stages.key].target_id
         profiles  = [stages.key]
       }
     }

diff --git a/infrastructure/terraform.tfvars.sample b/infrastructure/terraform.tfvars.sample
@@ -42,36 +42,36 @@ vpc_regions = {
 # Game GKE Cluster Config Values
 ### NOTE: If you change the GKE Clusters, please make sure to change `cloudbuild.yaml` in 
 ### `platform/agones/install` as they are not dynamically created.
-game_gke_clusters = {
+game_gke_standard_clusters = {
   "global-game-us-central1-01" : {
     "short_name" : "us-central1-01",
     "region" : "us-central1",
     "machine_type" : "e2-standard-4"
   },
-  "global-game-us-central1-02" : {
-    "short_name" : "us-central1-02",
-    "region" : "us-central1",
-    "machine_type" : "e2-standard-4"
-  },
   "global-game-eu-west1-01" : {
     "short_name" : "europe-west1-01",
     "region" : "europe-west1",
     "machine_type" : "e2-standard-4"
   },
-  "global-game-eu-west1-02" : {
-    "short_name" : "europe-west1-02",
-    "region" : "europe-west1",
-    "machine_type" : "e2-standard-4"
-  },
   "global-game-asia-east1-01" : {
     "short_name" : "asia-east1-01",
     "region" : "asia-east1",
     "machine_type" : "e2-standard-4"
+  }
+}
+
+game_gke_autopilot_clusters = {
+  "global-game-us-central1-02" : {
+    "short_name" : "us-central1-02",
+    "region" : "us-central1",
+  },
+  "global-game-eu-west1-02" : {
+    "short_name" : "europe-west1-02",
+    "region" : "europe-west1",
   },
   "global-game-asia-east1-02" : {
     "short_name" : "asia-east1-02",
     "region" : "asia-east1",
-    "machine_type" : "e2-standard-4"
   }
 }
 

diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf
@@ -44,9 +44,14 @@ variable "vpc_regions" {
 
 ### Agones Variables ###
 
-variable "game_gke_clusters" {
+variable "game_gke_standard_clusters" {
   type        = map(any)
-  description = "GKE gameclusters & associated values"
+  description = "GKE Standard Game Clusters & Associated values"
+}
+
+variable "game_gke_autopilot_clusters" {
+  type        = map(any)
+  description = "GKE Autopilot Game Clusters & Associated values"
 }
 
 ### Cloud Deploy Variables ###

diff --git a/platform/agones/install/cloudbuild.yaml → platform/agones/cloudbuild.yaml b/platform/agones/install/cloudbuild.yaml → platform/agones/cloudbuild.yaml
@@ -24,6 +24,7 @@ steps:
         "--skaffold-version", "1.39",
         "--region", "us-central1"
       ]
+    timeout: 1800s
 
 substitutions:
   _RELEASE_NAME: rel-0001