From c2561146ef02700899e54db211e38e750c21e314 Mon Sep 17 00:00:00 2001 From: Mark Mandel Date: Tue, 16 Jan 2024 14:31:34 -0800 Subject: [PATCH] Automate Building of Linux Client (#234) * Automate Building of Linux Client This automated both the building of the game launcher, a custom Unreal Engine Development image, and using that image to build both the client build and the dedicated server image - and storing all these artifacts in Cloud Storage. Closes #113 * Added creation of open-match namespace to Services GKE --------- Co-authored-by: Andrew Marcum --- README.md | 44 ++++++++---- game/.gcloudignore | 2 + game/.gitignore | 3 +- game/Dockerfile | 4 +- game/GameLauncher/app.ini.sample | 2 +- game/cloudbuild.yaml | 94 +++++++++++++++++++++----- game/unreal-engine/Dockerfile | 93 +++++++++++++++++++++++++ game/unreal-engine/cloudbuild.yaml | 51 ++++++++++++++ infrastructure/client.tf | 18 +++++ infrastructure/game-server.tf | 4 +- infrastructure/iam.tf | 3 +- infrastructure/terraform.tfvars.sample | 1 + infrastructure/variables.tf | 7 +- services/open-match/deployment.yaml | 18 +++++ services/skaffold.yaml | 1 + 15 files changed, 309 insertions(+), 36 deletions(-) create mode 100644 game/unreal-engine/Dockerfile create mode 100644 game/unreal-engine/cloudbuild.yaml create mode 100644 infrastructure/client.tf create mode 100644 services/open-match/deployment.yaml diff --git a/README.md b/README.md index 9dd7881..b21c9d2 100644 --- a/README.md +++ b/README.md @@ -116,15 +116,15 @@ cd global-multiplayer-demo export GAME_DEMO_HOME=$(pwd) ``` -#### Access to Unreal Engine Container Images +#### Access to Unreal Engine Source code. To build the Dedicated Game Server you will need access to the Unreal Engine GitHub organisation. To do so, follow: [Accessing Unreal Engine source code on GitHub](https://www.unrealengine.com/en-US/ue-on-github). -Once done, to pull down the [Unreal Development Containers](https://unrealcontainers.com/), you will also need to +Once done, to pull down the source code with Cloud Build, you will also need to create [a personal access token (classic)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token#creating-a-personal-access-token-classic) -with at least `read:packages` scope. +with at least `repo` scope. Leave the page open with this token, as we'll need it shortly. @@ -152,8 +152,9 @@ cp terraform.tfvars.sample terraform.tfvars You will need to now edit `terraform.tfvars` * Update with the ID of your Google Cloud Project, -* Updated and with the Client ID and Client secret created in the above step. -* Updated with the GitHub personal access token you created the above steps. +* Update and with the Client ID and Client secret created in the above step. +* Update with your GitHub username and the with the GitHub personal access token you + created with the above steps. You can edit other variables in this file, but we recommend leaving the default values for your first run before experimenting. @@ -240,9 +241,12 @@ This will: * Store those image in [Artifact Registry](https://cloud.google.com/artifact-registry) * Deploy them via Cloud Build to an Autopilot cluster. -### Dedicated Game Server +### Build Linux Games Client and Dedicated Game Server -To build the Unreal dedicated game server image, run the following command. +To build both the Unreal Client and Game launcher, as well as the Unreal dedicated game server image, run the following +command. + +> This will take ~5 hours on first run, and ~2 hours afterwards or so, so feel free to grab a cup of ☕ or a long nap. ```shell cd $GAME_DEMO_HOME/game @@ -251,13 +255,17 @@ gcloud builds submit --config=cloudbuild.yaml Cloud Build will deploy: +* Build a custom [Unreal Engine development image](https://unrealcontainers.com/docs/concepts/image-types#development-images) on first run, to have all the plugins we need for our game + client. +* Build a Linux version of the Unreal Game Client Game Launcher with appropriate configuration files. +* Store a zip of the Client in a Google Cloud Storage Bucket `gs://${PROJECT_ID}-release-artifacts`. * Build the image for the dedicated game server. * Store the image in [Artifact Registry](https://cloud.google.com/artifact-registry). * Start the staged rollout of the Agones Fleet to each regional set of clusters. -> This will take ~20 minutes or so, so feel free to grab a cup of ☕ +#### Deploy the Game Server to all Agones Clusters -Navigate to the +Once the build process is complete, navigate to the [agones-deploy-pipeline](https://console.cloud.google.com/deploy/delivery-pipelines/us-central1/global-game-agones-gameservers) delivery pipeline to review the rollout status. Cloud Build will create a Cloud Deploy release which automatically deploys the game server Agones Fleet to the `asia-east1` region first. @@ -268,8 +276,18 @@ The Fleet can be deployed to the next region in the queue via pressing the ## Replace RELEASE_NAME with the unique build name gcloud deploy releases promote --release=RELEASE_NAME --delivery-pipeline=global-game-agones-gameservers --region=us-central1 ``` +#### Retrieve Game Client + +The Cloud Build process will build and archive a `Client-${BUILD_ID}.zip` file in the Google Cloud Storage +Bucket `gs://${PROJECT_ID}-release-artifacts`. -### Game Client +Use the [Cloud Storage Browser](https://console.cloud.google.com/storage/browser/) or `gsutil` to download +the file and `unzip` it locally. + +Run `launcher` to run the Game Launcher, and see [Playing The Game](#playing-the-game) for details on how to play the +game once the launcher is up and running. + +### Editing or Building the Game Locally To build the Game Client for your host machine, you will need to [install Unreal Engine from source](https://docs.unrealengine.com/5.2/en-US/building-unreal-engine-from-source/), @@ -300,7 +318,7 @@ gcloud compute addresses list --filter=name=frontend-service --format="value(add JWT token can be obtained by accessing frontend api's ip address with '/login' path, such as "http://[IP_ADDRESS].sslip.io/login" and extracting it from the URL. -### Run the Game Launcher +#### Run the Game Launcher To run the game launcher, you will need to have [Go](https://go.dev/dl/) installed to run it, as well as the [prerequisites for the Fyne Go Cross Platform UI library](https://developer.fyne.io/started/). @@ -315,7 +333,7 @@ cp app.ini.sample app.ini gcloud compute addresses list --filter=name=frontend-service --format="value(address)" ``` -Edit the app.ini, and replace the `frontend_api` value with http://[IP_ADDRESS].sslip.io +Edit the app.ini, and replace the `frontend_api` value for ${IP_ADDRESS}. And update the `binary` field with the path to the executable of the client build for your operating system. @@ -325,6 +343,8 @@ Then run the following to start the launcher! go run main.go ``` +### Playing the Game + You will need three players to play a game, so you can use the "Instances" drop down to create more than one game client instance on your local machine. Depending on the capability of your graphics card, creating smaller resolution game client instances may be required. diff --git a/game/.gcloudignore b/game/.gcloudignore index 201d71d..9d5dc36 100644 --- a/game/.gcloudignore +++ b/game/.gcloudignore @@ -24,3 +24,5 @@ Packaged Saved *.code-workspace /Makefile +*.zip + diff --git a/game/.gitignore b/game/.gitignore index f86eec2..40115d1 100644 --- a/game/.gitignore +++ b/game/.gitignore @@ -25,4 +25,5 @@ Packaged Saved *.code-workspace /Makefile -/GameLauncher/app.ini \ No newline at end of file +/GameLauncher/app.ini +*.zip diff --git a/game/Dockerfile b/game/Dockerfile index 7cbc92c..352c42d 100644 --- a/game/Dockerfile +++ b/game/Dockerfile @@ -16,7 +16,9 @@ # Build a container for the Unreal Engine dedicated game server # -FROM ghcr.io/epicgames/unreal-engine:dev-slim-5.2.0 as builder +ARG BASE_IMAGE=ghcr.io/epicgames/unreal-engine:dev-slim-5.2.0 + +FROM $BASE_IMAGE as builder ARG SERVER_CONFIG=Development COPY --chown=ue4:ue4 . /tmp/project diff --git a/game/GameLauncher/app.ini.sample b/game/GameLauncher/app.ini.sample index 63abd8d..87aea5e 100644 --- a/game/GameLauncher/app.ini.sample +++ b/game/GameLauncher/app.ini.sample @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -frontend_api = http://[IP_ADDRESS].sslip.io +frontend_api = http://${IP_ADDRESS}.sslip.io callback_listen_port = 8082 [windows] diff --git a/game/cloudbuild.yaml b/game/cloudbuild.yaml index de02a20..fafded1 100644 --- a/game/cloudbuild.yaml +++ b/game/cloudbuild.yaml @@ -16,18 +16,74 @@ serviceAccount: projects/${PROJECT_ID}/serviceAccounts/cloudbuild-cicd@${PROJECT steps: # - # Building of the images + # Build the unreal editor build image if it does not exist. # + - name: gcr.io/cloud-builders/gcloud + id: build-unreal-editor-container + dir: /workspace/unreal-engine + script: | + gcloud container images describe $_BUILD_IMAGE || gcloud builds submit --substitutions _UNREAL_VERSION=$_UNREAL_VERSION --config=cloudbuild.yaml . + automapSubstitutions: true + waitFor: ['-'] - - name: gcr.io/cloud-builders/docker - id: github-login + # + # Game Launcher building + # + + - name: gcr.io/cloud-builders/gcloud + id: get-front-end + dir: /workspace/GameLauncher + script: | + echo "export IP_ADDRESS=$(gcloud compute addresses list --filter=name=frontend-service --format='value(address)')" > .env && \ + cat .env + waitFor: ['-'] + - name: golang:1.21.4 + id: linux-game-launcher + dir: /workspace/GameLauncher + script: | + apt update && \ + apt install -y golang gcc libgl1-mesa-dev xorg-dev zip gettext-base && \ + go build . && \ + . ./.env && \ + envsubst < app.ini.sample > app.ini && \ + cat app.ini && \ + zip -ur /workspace/Client-$BUILD_ID.zip launcher app.ini assets + automapSubstitutions: true + waitFor: + - get-front-end + + # + # Game Client Building + # + + - name: ${_BUILD_IMAGE} + id: linux-unreal-client + dir: /home/ue4 script: | - echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin - secretEnv: - - CR_PAT + df -h && \ + echo "who: $(whoami)" && \ + sudo mkdir -pv "/builder/home/Library/Logs/Unreal Engine/LocalBuildLogs" && sudo chown -R ue4:ue4 /builder && \ + touch "/builder/home/Library/Logs/Unreal Engine/LocalBuildLogs/empty.log" && \ + sudo cp -r /workspace /tmp/project && sudo chown -R ue4:ue4 /tmp/project && ls -l /tmp/project && \ + /home/ue4/UnrealEngine/Engine/Build/BatchFiles/RunUAT.sh BuildCookRun \ + -project=/tmp/project/Droidshooter.uproject -noP4 -clientconfig=Development \ + -utf8output -NoDebugInfo -AllMaps -platform=Linux -client -build -cook -pak -stage -prereqs -package \ + -archive -archivedirectory=/tmp/project/Packaged && \ + cd /tmp/project/Packaged/LinuxClient && sudo zip -ur /workspace/Client-$BUILD_ID.zip . + automapSubstitutions: true + waitFor: + - build-unreal-editor-container + - linux-game-launcher + + # + # Building of the dedicate game server images + # + - name: gcr.io/cloud-builders/docker - id: build-image - args: [ "build", ".", "-t", "${_UNREAL_SERVER_IMAGE}" ] + id: dedicated-server + args: [ "build", "--build-arg", "BASE_IMAGE=${_BUILD_IMAGE}", ".", "-t", "${_UNREAL_SERVER_IMAGE}" ] + waitFor: + - linux-unreal-client # # Deployment @@ -35,31 +91,35 @@ steps: - name: gcr.io/google.com/cloudsdktool/cloud-sdk id: cloud-deploy-release - entrypoint: bash - args: - - "-c" - - | + script: | gcloud deploy releases create deploy-$(date +'%Y%m%d%H%M%S') \ --annotations=cloud_build=https://console.cloud.google.com/cloud-build/builds/${BUILD_ID} \ --delivery-pipeline global-game-agones-gameservers \ --skaffold-file skaffold.yaml \ - --images droidshooter-server=${_UNREAL_SERVER_IMAGE} \ + --images droidshooter-server=$_UNREAL_SERVER_IMAGE \ --region us-central1 + automapSubstitutions: true + waitFor: + - dedicated-server artifacts: images: - ${_REGISTRY}/droidshooter-server + objects: + location: gs://${PROJECT_ID}-release-artifacts + paths: + - Client-${BUILD_ID}.zip substitutions: + _UNREAL_VERSION: 5.2.0 + _BUILD_IMAGE: us-docker.pkg.dev/${PROJECT_ID}/global-game-images/unreal-engine:${_UNREAL_VERSION} _UNREAL_SERVER_IMAGE: ${_REGISTRY}/droidshooter-server:${BUILD_ID} _REGISTRY: us-docker.pkg.dev/${PROJECT_ID}/global-game-images -availableSecrets: - secretManager: - - versionName: projects/${PROJECT_ID}/secrets/github-packages/versions/latest - env: CR_PAT options: dynamic_substitutions: true machineType: E2_HIGHCPU_32 logging: CLOUD_LOGGING_ONLY + diskSizeGb: '500' +timeout: 36000s tags: - global-game-demo - game diff --git a/game/unreal-engine/Dockerfile b/game/unreal-engine/Dockerfile new file mode 100644 index 0000000..af83ada --- /dev/null +++ b/game/unreal-engine/Dockerfile @@ -0,0 +1,93 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Credits to https://github.com/adamrehn/ue4-docker/blob/master/src/ue4docker/dockerfiles and +# https://unrealcontainers.com/docs/obtaining-images/write-your-own#writing-dockerfiles-for-linux-containers +# for how to accomplish this in a Dockerfile + +# Split prerequisites into it's own layer, so we can reuse it. +FROM nvidia/opengl:1.0-glvnd-devel-ubuntu22.04 as prerequisites + +# Disable interactive prompts during package installation + +ENV DEBIAN_FRONTEND=noninteractive +# build prerequisites +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + git-lfs \ + python3 \ + python3-dev \ + python3-pip \ + shared-mime-info \ + software-properties-common \ + sudo \ + tzdata \ + unzip \ + xdg-user-dirs \ + zip \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcairo2 \ + libfontconfig1 \ + libfreetype6 \ + libgbm1 \ + libglu1 \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libsm6 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxi6 \ + libxkbcommon-x11-0 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + libxv1 \ + x11-xkb-utils \ + xauth \ + xfonts-base \ + xkb-data && \ + rm -rf /var/lib/apt/lists/* + +# Unreal refuses to run as the root user, so create a non-root user with no password and allow them to run commands using sudo +RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ + passwd -d ue4 && \ + usermod -a -G audio,video,sudo ue4 +USER ue4 + +# Build Unreal Engine from source +FROM prerequisites as source + +COPY --chown=ue4:ue4 ./UnrealEngine /home/ue4/UnrealEngine + +WORKDIR /home/ue4/UnrealEngine +RUN whoami && ls -l .. && ls -l && ./Setup.sh -no-cache && sudo rm -rf /var/lib/apt/lists/* && ./GenerateProjectFiles.sh && make + +# Do some cleanup before moving to the final layer +RUN rm -r /home/ue4/UnrealEngine/Engine/Documentation && \ + cd /home/ue4/UnrealEngine/Engine/Binaries && find -name 'Win64' -exec rm -r {} \; || true && find -name 'Mac' -exec rm -r {} \; || true + +# Copy over to the final image only what is needed +FROM prerequisites + +COPY --from=source --chown=ue4:ue4 /home/ue4/UnrealEngine/Engine /home/ue4/UnrealEngine/Engine +WORKDIR /home/ue4/UnrealEngine diff --git a/game/unreal-engine/cloudbuild.yaml b/game/unreal-engine/cloudbuild.yaml new file mode 100644 index 0000000..3e3ef45 --- /dev/null +++ b/game/unreal-engine/cloudbuild.yaml @@ -0,0 +1,51 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +serviceAccount: projects/${PROJECT_ID}/serviceAccounts/cloudbuild-cicd@${PROJECT_ID}.iam.gserviceaccount.com +steps: + # Clone the UnrealEngine repository using a personal access token. + - name: gcr.io/cloud-builders/git + id: clone-unreal-engine + script: | + df -h + git clone --depth=1 --single-branch --branch "$_UNREAL_VERSION-release" https://$GITHUB_AUTH@github.com/EpicGames/UnrealEngine.git + cd UnrealEngine + ls -l + secretEnv: + - GITHUB_AUTH + automapSubstitutions: true + + - name: gcr.io/cloud-builders/docker + id: build-unreal-image + args: ['build', '.', '--tag', '${_BUILD_IMAGE}' ] + +images: + - ${_BUILD_IMAGE} + +availableSecrets: + secretManager: + - versionName: projects/${PROJECT_ID}/secrets/github-auth/versions/latest + env: GITHUB_AUTH +substitutions: + _BUILD_IMAGE: us-docker.pkg.dev/${PROJECT_ID}/global-game-images/unreal-engine:${_UNREAL_VERSION} + _UNREAL_VERSION: 5.2.0 +options: + dynamic_substitutions: true + machineType: E2_HIGHCPU_32 + diskSizeGb: '1000' + logging: CLOUD_LOGGING_ONLY +timeout: 18000s +tags: + - global-game-demo + - unreal diff --git a/infrastructure/client.tf b/infrastructure/client.tf new file mode 100644 index 0000000..18d28b9 --- /dev/null +++ b/infrastructure/client.tf @@ -0,0 +1,18 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +resource "google_storage_bucket" "release-artifacts" { + location = "US" + name = "${var.project}-release-artifacts" +} diff --git a/infrastructure/game-server.tf b/infrastructure/game-server.tf index 0144ed2..0783039 100644 --- a/infrastructure/game-server.tf +++ b/infrastructure/game-server.tf @@ -13,7 +13,7 @@ # limitations under the License. resource "google_secret_manager_secret" "secret_github_packages" { - secret_id = "github-packages" + secret_id = "github-auth" labels = { "environment" = var.resource_env_label @@ -28,7 +28,7 @@ resource "google_secret_manager_secret_version" "pat_1" { secret = google_secret_manager_secret.secret_github_packages.id enabled = true - secret_data = var.github_pat + secret_data = "${var.github_username}:${var.github_pat}" } resource "google_secret_manager_secret_iam_binding" "cloud_build_binding" { diff --git a/infrastructure/iam.tf b/infrastructure/iam.tf index 5dfd8a8..d91ae5b 100644 --- a/infrastructure/iam.tf +++ b/infrastructure/iam.tf @@ -40,7 +40,8 @@ resource "google_project_iam_member" "cloudbuild-sa-cloudbuild-roles" { "roles/storage.admin", "roles/iam.serviceAccountUser", "roles/spanner.databaseUser", - "roles/gkehub.editor" + "roles/gkehub.editor", + "roles/compute.viewer" ]) role = each.key member = "serviceAccount:${google_service_account.cloudbuild-sa.email}" diff --git a/infrastructure/terraform.tfvars.sample b/infrastructure/terraform.tfvars.sample index 1b39d06..71cce2d 100644 --- a/infrastructure/terraform.tfvars.sample +++ b/infrastructure/terraform.tfvars.sample @@ -142,4 +142,5 @@ app_service_account_config = { k8s_service_account_id = "k8s-service-account" # Github Personal Access Token +github_username = "" github_pat = "" diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index ca371fb..b898cdd 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -145,7 +145,12 @@ variable "open-match-matchfunction" { ### Dedicated Game Server Variables +variable "github_username" { + type = string + description = "The GitHub username that matches to the `github_pat` personal access token" +} + variable "github_pat" { type = string - description = "A GitHub personal access token (classic) with at least read:packages scope" + description = "A GitHub personal access token (classic) with at least repo scope" } diff --git a/services/open-match/deployment.yaml b/services/open-match/deployment.yaml new file mode 100644 index 0000000..e857265 --- /dev/null +++ b/services/open-match/deployment.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: open-match diff --git a/services/skaffold.yaml b/services/skaffold.yaml index f2a7632..7321cba 100644 --- a/services/skaffold.yaml +++ b/services/skaffold.yaml @@ -22,6 +22,7 @@ manifests: - profile/deployment.yaml - frontend/config.yaml - frontend/deployment.yaml + - open-match/deployment.yaml - open-match/director/director.yaml - open-match/matchfunction/matchfunction.yaml - open-match/matchfunction/config.yaml