Added Spanner Service Account, VPC additions, updated deploy instruct…

…ions, & re-arranged template files (#25) * Re-named template directory to better reflect usage * Added Spanner SA for application access * Added Cloud Router & NAT so GKE nodes can access container images outside of the local VPC * Added example nginx deploy to Spanner GKE cluster * Added instructions for running Spannerk GKE deploy * Re-named deploy directory to better match function
googleforgames · Feb 2, 2023 · 6089b85 · 6089b85
1 parent 59436fb
commit 6089b85
Show file tree

Hide file tree

Showing 15 changed files with 136 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -54,6 +54,8 @@ $ cd deploy/agones/install
 $ gcloud builds submit --config=cloudbuild.yaml --substitutions=_RELEASE_NAME=rel-1
 ```
 
+You can monitor the status of the deployment through the Cloud Logging URL returned by the `gcloud builds` command as well as the Kubernetes Engine/Worloads panel in the GCP Console. Once the Worloads have been marked as OK, you can proceed to apply the Allocation Endpoint Patch.
+
 #### Allocation Endpoint Patch
 After the Agones install has completed and the GKE Workloads show complete, run the Allocation Endpoint Patch Cloud Deploy to apply the appropriate endpoint patches to each cluster: 
 
@@ -62,6 +64,21 @@ $ cd deploy/agones/endpoint-patch/
 $ gcloud builds submit --config=cloudbuild.yaml
 ```
 
+***NOTE*** - The cloudbuild.yaml, kustomization.yaml & skaffold.yaml files will not exist until Terraform runs for the first time! The templates used for these files are stored in `files/agones/`.
+
+You can monitor the status of the deployment through the Cloud Logging URL returned by the `gcloud builds` comma
+nd as well as the Kubernetes Engine/Worloads panel in the GCP Console. Once the Worloads have been marked as O
+K, Agones should be avaialable. 
+
+### Deploy Spanner Applications to GKE Cluster
+
+#### Initial Deploy
+Replace the` _RELEASE_NAME` substitution with a unique build name. Cloudbuild will deploy Spanner applications using Cloud Deploy.
+
+```shell
+$ cd deploy/spanner/install
+$ gcloud builds submit --config=cloudbuild.yaml --substitutions=_RELEASE_NAME=rel-1
+```
 
 ## Licence
 

diff --git a/allocation-endpoint.tf b/allocation-endpoint.tf
@@ -16,12 +16,12 @@ resource "google_endpoints_service" "endpoints_service" {
   for_each     = var.game_gke_clusters
   service_name = "${each.key}.endpoints.${var.project}.cloud.goog"
   grpc_config = templatefile(
-    "${path.module}/files/allocation-endpoint/api_config.yaml.tpl", {
+    "${path.module}/files/agones/api_config.yaml.tpl", {
       service-name    = "${each.key}.endpoints.${var.project}.cloud.goog"
       service-account = google_service_account.ae_sa.email
     }
   )
-  protoc_output_base64 = filebase64("files/allocation-endpoint/agones_allocation_api_descriptor.pb")
+  protoc_output_base64 = filebase64("files/agones/agones_allocation_api_descriptor.pb")
 }
 
 resource "google_endpoints_service_iam_binding" "endpoints_service_binding" {
@@ -37,7 +37,7 @@ resource "google_endpoints_service_iam_binding" "endpoints_service_binding" {
 
 resource "google_service_account_iam_binding" "workload-identity-binding" {
   service_account_id = google_service_account.ae_sa.name
-  role = "roles/iam.workloadIdentityUser"
+  role               = "roles/iam.workloadIdentityUser"
 
   members = [
     "serviceAccount:${var.project}.svc.id.goog[${var.allocation_endpoint.agones_namespace}/agones-allocator]",
@@ -84,7 +84,7 @@ resource "google_cloud_run_service" "aep_cloud_run" {
         env {
           name = "CLUSTERS_INFO"
           value = templatefile(
-            "${path.module}/files/allocation-endpoint/clusters_info.tpl", {
+            "${path.module}/files/agones/clusters_info.tpl", {
               name      = data.google_container_cluster.game-demo-agones-gke[each.key].name
               ip        = google_compute_address.allocation-endpoint[each.key].address
               weight    = var.allocation_endpoint.weight
@@ -184,7 +184,7 @@ resource "google_project_service" "allocator-service" {
 
 resource "google_compute_address" "allocation-endpoint" {
   for_each = var.game_gke_clusters
-  region = each.value.region
+  region   = each.value.region
 
   name = "allocator-endpoint-ip-${each.key}"
 }
@@ -194,7 +194,7 @@ resource "local_file" "agones-skaffold-file" {
   for_each = var.game_gke_clusters
 
   content = templatefile(
-    "${path.module}/files/allocation-endpoint/skaffold.yaml.tpl", {
+    "${path.module}/files/agones/skaffold.yaml.tpl", {
       cluster_name = each.key
   })
   filename = "${path.module}/deploy/agones/install/skaffold-${each.key}.yaml"
@@ -205,7 +205,7 @@ resource "local_file" "agones-ae-lb-file" {
   for_each = var.game_gke_clusters
 
   content = templatefile(
-    "${path.module}/files/allocation-endpoint/ae-lb-ip-patch.yaml.tpl", {
+    "${path.module}/files/agones/ae-lb-ip-patch.yaml.tpl", {
       lb_ip = google_compute_address.allocation-endpoint[each.key].address
   })
   filename = "${path.module}/deploy/agones/install/${each.key}/kustomization.yaml"
@@ -217,7 +217,7 @@ resource "local_file" "patch-agones-manifest" {
   for_each = var.game_gke_clusters
 
   content = templatefile(
-    "${path.module}/files/allocation-endpoint/patch-agones-allocator.yaml.tpl", {
+    "${path.module}/files/agones/patch-agones-allocator.yaml.tpl", {
       project_id   = var.project
       location     = each.value.region
       cluster_name = each.key

diff --git a/deploy/services/install/cloudbuild.yaml b/deploy/services/install/cloudbuild.yaml
@@ -0,0 +1,37 @@
+# Copyright 2023 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+serviceAccount: projects/${PROJECT_ID}/serviceAccounts/cloudbuild-cicd@${PROJECT_ID}.iam.gserviceaccount.com
+steps:
+  - name: "gcr.io/cloud-builders/gke-deploy"
+    args:
+    - run
+    - --filename=spanner-system.yaml
+    - --location=us-central1
+    - --cluster=global-game-gke-spanner
+    - --output=output/spanner-deploy-us-central1-01
+
+  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    entrypoint: gcloud
+    args:
+      [
+        "deploy", "releases", "create", "${_RELEASE_NAME}",
+        "--delivery-pipeline", "global-game-spanner-deploy-pipeline",
+        "--region", "us-central1"
+      ]
+
+substitutions:
+  _RELEASE_NAME: rel-0001
+options:
+  logging: CLOUD_LOGGING_ONLY
diff --git a/deploy/services/install/nginx/kustomization.yaml b/deploy/services/install/nginx/kustomization.yaml
@@ -0,0 +1,8 @@
+# ./nginx/kustomization.yaml
+helmCharts:
+- name: nginx
+  repo: https://charts.bitnami.com/bitnami
+  releaseName: nginx
+  version: 13.2.23
+  releaseName: example-web
+  namespace: spanner
diff --git a/deploy/services/install/skaffold.yaml b/deploy/services/install/skaffold.yaml
@@ -0,0 +1,9 @@
+apiVersion: skaffold/v2beta29
+kind: Config
+deploy:
+  kustomize:
+    paths:
+      - "./nginx"
+    buildArgs: ["--enable-helm"]
+    flags:
+      apply: ['--server-side'] # Avoid the "Too long: must have at most 262144 bytes" problem
diff --git a/deploy/services/install/spanner-system.yaml b/deploy/services/install/spanner-system.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spanner
diff --git a/...location-endpoint/ae-lb-ip-patch.yaml.tpl → files/agones/ae-lb-ip-patch.yaml.tpl b/...location-endpoint/ae-lb-ip-patch.yaml.tpl → files/agones/ae-lb-ip-patch.yaml.tpl
diff --git a/...point/agones_allocation_api_descriptor.pb → ...gones/agones_allocation_api_descriptor.pb b/...point/agones_allocation_api_descriptor.pb → ...gones/agones_allocation_api_descriptor.pb
diff --git a/...s/allocation-endpoint/api_config.yaml.tpl → files/agones/api_config.yaml.tpl b/...s/allocation-endpoint/api_config.yaml.tpl → files/agones/api_config.yaml.tpl
diff --git a/files/allocation-endpoint/clusters_info.tpl → files/agones/clusters_info.tpl b/files/allocation-endpoint/clusters_info.tpl → files/agones/clusters_info.tpl
diff --git a/...-endpoint/patch-agones-allocator.yaml.tpl → files/agones/patch-agones-allocator.yaml.tpl b/...-endpoint/patch-agones-allocator.yaml.tpl → files/agones/patch-agones-allocator.yaml.tpl
diff --git a/files/allocation-endpoint/skaffold.yaml.tpl → files/agones/skaffold.yaml.tpl b/files/allocation-endpoint/skaffold.yaml.tpl → files/agones/skaffold.yaml.tpl
diff --git a/spanner.tf b/spanner.tf
@@ -26,3 +26,15 @@ resource "google_spanner_database" "spanner-database" {
   version_retention_period = "3d"
   deletion_protection      = false
 }
+
+resource "google_service_account" "spanner-sa" {
+  project      = var.project
+  account_id   = "spanner-sa"
+  display_name = "Spanner Service Account"
+}
+
+resource "google_project_iam_member" "spanner-sa" {
+  project = var.project
+  role    = "roles/spanner.databaseUser"
+  member  = "serviceAccount:${google_service_account.spanner-sa.email}"
+}
diff --git a/terraform.tfvars.sample b/terraform.tfvars.sample
@@ -14,6 +14,10 @@
 
 project = "PROJECT_ID"
 
+# VPC Regions & associated subnet CIDRs Values
+
+vpc_name = "global-game-vpc"
+
 vpc_regions = {
   "us-central1" : {
     "vpc_subnet_cidr" : "10.1.0.0/16"
@@ -26,6 +30,9 @@ vpc_regions = {
   }
 }
 
+# Game GKE Cluster Config Values
+### NOTE: If you change the GKE Clusters, please make sure to change `cloudbuild.yaml` in 
+### `agones/install` & `deploy/agones/endpoint-patch/` as they are not dynamically created.  
 game_gke_clusters = {
   "global-game-us-central1-01" : {
     "region" : "us-central1",
@@ -45,6 +52,7 @@ game_gke_clusters = {
   }
 }
 
+# GCP APIs to Enable
 gcp_project_services = [
   "clouddeploy.googleapis.com",
   "cloudbuild.googleapis.com",
@@ -58,15 +66,16 @@ gcp_project_services = [
   "run.googleapis.com"
 ]
 
-vpc_name = "global-game-vpc"
 
+# Spanner DB Config Values
 spanner_config = {
   db_name       = "global-game-spanner-db"
   instance_name = "global-game-spanner-instance"
   location      = "nam-eur-asia1"
   num_nodes     = 3
 }
 
+# Spanner GKE Config Values
 spanner_gke_config = {
   cluster_name = "global-game-gke-spanner"
   location     = "us-central1"
@@ -75,13 +84,19 @@ spanner_gke_config = {
   }
 }
 
+
+# Agones Application Service Account Name
 app_service_account_config = {
   name        = "app-service-account"
   description = "Global Multiplayer Game service account"
 }
 
+
+# Agones GKE Service Account Name
 k8s_service_account_id = "k8s-service-account"
 
+
+# Agones Allocation Endpoint Config Values
 allocation_endpoint = {
   name             = "agones-allocation-endpoint"
   proxy_image      = "us-docker.pkg.dev/agones-images/examples/allocation-endpoint-proxy:0.3"

diff --git a/vpc.tf b/vpc.tf
@@ -24,3 +24,28 @@ resource "google_compute_subnetwork" "subnet" {
   region        = each.key
   network       = google_compute_network.vpc.id
 }
+
+resource "google_compute_router" "vpc_router" {
+  for_each = var.vpc_regions
+  name     = "global-game-${each.key}-router"
+  region   = each.key
+  network  = google_compute_network.vpc.id
+
+  bgp {
+    asn = 64514
+  }
+}
+
+resource "google_compute_router_nat" "vpc_nat" {
+  for_each                           = var.vpc_regions
+  name                               = "global-game-${each.key}-nat"
+  router                             = google_compute_router.vpc_router[each.key].name
+  region                             = each.key
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+
+  log_config {
+    enable = true
+    filter = "ERRORS_ONLY"
+  }
+}