Merge pull request #262 from murraystevenson/CI

CI : Add CI for `linux-gcc11` and `macos-arm64`

.github/workflows/main.yml

@@ -0,0 +1,126 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - '*_maintenance'
+  pull_request:
+    branches:
+      - '*'
+  release:
+      types: [published]
+
+jobs:
+
+  build:
+
+    strategy:
+
+      # Don't cancel other jobs in the build matrix if one job fails.
+      fail-fast: false
+
+      matrix:
+
+        # Rather than generate all permutations of various settings,
+        # we want to explicitly list each of the variants we want to
+        # test. We can use `name` to declare the names of our variants,
+        # and then use `include` to define their settings.
+
+        name: [
+          linux-gcc11,
+          macos-arm64,
+        ]
+
+        include:
+
+          - name: linux-gcc11
+            os: ubuntu-20.04
+            publish: true
+            containerImage: ghcr.io/gafferhq/build/build:3.0.0
+            jobs: 4
+
+          - name: macos-arm64
+            os: macos-14
+            publish: true
+            containerImage:
+            jobs: 3
+
+    runs-on: ${{ matrix.os }}
+
+    container: ${{ matrix.containerImage }}
+
+    env:
+      DEPENDENCIES_BUILD_DIR: "./build"
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - name: Install toolchain (Linux)
+      run: |
+        # The Docker container configures bash shells such that they enable the
+        # software collections we want. If we could set GitHub's
+        # `defaults.run.shell` to `bash` then all our build steps would pick up
+        # this environment automatically. But we can't do that because it will
+        # break future Windows builds, and we can't configure a different shell
+        # per platform because GitHub won't allow it. But we can run _this_
+        # Linux-only step in bash, and transfer the environment out to be used
+        # in later steps.
+        echo $PATH > $GITHUB_PATH
+        echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV
+      shell: bash
+      if: runner.os == 'Linux'
+
+    - name: Install toolchain (macOS)
+      run: |
+        # Choose the earliest Xcode version available on a macos-14 runner.
+        sudo xcode-select -s /Applications/Xcode_14.3.1.app/Contents/Developer
+        # Install build requirements.
+        sudo pip3 install scons==4.6.0 --break-system-packages
+        brew install gpatch
+        brew install bison
+        # Make sure bison is discoverable.
+        echo BISON_ROOT=/opt/homebrew/opt/bison >> $GITHUB_ENV
+        # Make the location of the macOS platform SDK obvious to CMake.
+        # OpenShadingLanguage needs a little extra help finding `cstddef`.
+        echo SDKROOT=`xcrun --sdk macosx --show-sdk-path` >> $GITHUB_ENV
+        # Remove CommandLineTools so there is no potential for conflict with
+        # our selected Xcode version.
+        sudo rm -rf /Library/Developer/CommandLineTools
+      shell: bash
+      if: runner.os == 'macOS'
+
+    - name: 'Install Python Modules'
+      run: |
+        python --version
+        pip install PyJWT==1.7.1 PyGitHub==1.45
+
+    - name: Set Custom Variables
+      run: |
+        .github/workflows/main/setBuildVars.py
+      env:
+        GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEPENDENCIES_BUILD_VARIANT: ${{ matrix.name }}
+      shell: bash
+
+    - name: Build Dependencies
+      run: |
+        ./build.py --cleanup --jobs ${{ matrix.jobs }} --buildDir ${{ env.DEPENDENCIES_BUILD_DIR }}/${{ env.DEPENDENCIES_BUILD_NAME }} --package ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }}
+      env:
+        PYTHONUTF8: 1
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.DEPENDENCIES_BUILD_NAME }}
+        path: ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }}
+        # Using compression-level 0 avoids compressing our already compressed
+        # package and results in a significantly faster upload.
+        compression-level: 0
+      if: matrix.publish
+
+    - name: Publish Release
+      run: |
+        python ./.github/workflows/main/publishRelease.py --archive ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} --repo ${{ github.repository }} --releaseId ${{ env.DEPENDENCIES_GITHUB_RELEASEID }}
+      if: matrix.publish && env.DEPENDENCIES_GITHUB_RELEASEID != ''
+      env:
+        GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/main/publishRelease.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+##########################################################################
+#
+#  Copyright (c) 2019, Cinesite VFX Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are
+#  met:
+#
+#      * Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      * Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials provided with
+#        the distribution.
+#
+#      * Neither the name of John Haddon nor the names of
+#        any other contributors to this software may be used to endorse or
+#        promote products derived from this software without specific prior
+#        written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+##########################################################################
+
+import github
+
+import argparse
+import datetime
+import os
+import subprocess
+import sys
+
+# A script to publish a build to a GitHub Release
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument(
+	"--repo",
+	required = True,
+	help = "The GitHub organisation/repo to post the build link to."
+)
+
+parser.add_argument(
+	"--releaseId",
+	type = int,
+	required = True,
+	help = "The release ID to publish the asset to."
+)
+
+parser.add_argument(
+	"--archive",
+	dest = "archive",
+	required = True,
+	help = "The path to the build archive to publish."
+)
+
+parser.add_argument(
+	"--github-access-token",
+	dest = "githubAccessToken",
+	default = os.environ.get( 'GITHUB_ACCESS_TOKEN', None ),
+	help = "A suitable access token to authenticate the GitHub API."
+)
+
+args = parser.parse_args()
+
+if not args.githubAccessToken :
+	parser.exit( 1, "No --github-access-token/GITHUB_ACCESS_TOKEN set" )
+
+if not os.path.exists( args.archive ) :
+	parser.exit( 1, "The specified archive '%s' does not exist." % args.archive )
+
+githubClient = github.Github( args.githubAccessToken )
+repo = githubClient.get_repo( args.repo )
+
+release = repo.get_release( args.releaseId )
+if not release :
+	parser.exit( 1, "Unable to find GitHub Release %s" % args.releaseId )
+
+print( "Uploading '%s' to release %s" % ( args.archive, args.releaseId ) )
+asset = release.upload_asset( args.archive, content_type="application/gzip" )
+
+print( "Success, %s available at %s" % ( args.archive, asset.browser_download_url ) )

.github/workflows/main/setBuildVars.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+##########################################################################
+#
+#  Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are
+#  met:
+#
+#      * Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      * Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials provided with
+#        the distribution.
+#
+#      * Neither the name of Cinesite VFX Ltd. nor the names of
+#        any other contributors to this software may be used to endorse or
+#        promote products derived from this software without specific prior
+#        written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+##########################################################################
+
+import datetime
+import github
+import os
+import re
+import sys
+import json
+
+# GitHub Action workflow variables can be populated at run-time by echoing special
+# strings to an env file. The following allows vars to be set:
+#
+#   https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files
+#
+#   echo var=value >> $GITHUB_ENV
+#
+# We make use of this mechanism to allow custom logic to define the build name
+# as well as determine the correct commit hash depending on the nature of the
+# trigger. These variables can be referenced in a pipeline yaml file downstream
+# of the step that runs this script.
+
+# Actions is somewhat sparse in what information is available via the GITHUB_*
+# env vars (from the github context). There is however a veritable treasure
+# trove in the .json file pointed to by GITHUB_EVENT_PATH. "The Internets" seem
+# to suggest this is the most reliable way of determining information about the
+# triggering commit. Some of the official context vars may vary after a retry,
+# etc. too.
+#
+# The contents of this file is based on the webhook payload so should be
+# relatively stable as it is part of that public API.
+
+with open( os.environ["GITHUB_EVENT_PATH"] ) as f :
+	eventData = json.load( f )
+
+## Source Branches
+
+buildBranch = os.environ.get( "GITHUB_BASE_REF", "" )
+sourceBranch = os.environ.get( "GITHUB_HEAD_REF", buildBranch )
+
+## Source Commit Hash
+
+commit = os.environ["GITHUB_SHA"]
+
+## Pull Request builds
+
+# Actions merges the branch into its target in PR build, so GITHUB_SHA isn't
+# correct as it references the ephemeral merge.  We also want to extract the
+# pull request number for the build name.
+
+pullRequest = ""
+if os.environ.get( "GITHUB_EVENT_NAME" ) == "pull_request" :
+	commit = eventData["pull_request"]["head"]["sha"]
+	pullRequest = eventData["pull_request"]["number"]
+
+## Tag builds
+
+# A variety of build types may be in service of a tag (ie: release publish
+# or manual retry for a specific tag).
+
+tag = ""
+if "/tags" in os.environ["GITHUB_REF"] :
+	tag = os.environ["GITHUB_REF"].replace( "refs/tags/", "" )
+
+## Release ID
+
+# To allow builds to be published to a release, we need to lookup the ID of any
+# release that matches the tag we're building, if there is one.
+
+releaseId = ""
+
+if tag :
+
+	githubClient = github.Github( os.environ.get( 'GITHUB_ACCESS_TOKEN' ) )
+	repo = githubClient.get_repo( os.environ.get( 'GITHUB_REPOSITORY' ) )
+
+	for r in repo.get_releases() :
+		if r.tag_name == tag :
+			releaseId = r.id
+			break
+
+if releaseId :
+
+	# Check that the version specified by the SConstruct matches the
+	# version in the tag.
+	version = ""
+	versionRe = re.compile( r"^__version = (\S+)" )
+	with open( "build.py" ) as buildPy :
+		for line in buildPy.readlines() :
+			versionMatch = versionRe.match( line )
+			if versionMatch :
+				version = versionMatch.group( 2 ).strip( "'\"" )
+
+	if version != tag :
+		sys.stderr.write( "Tag \"{}\" does not match build.py __version \"{}\"\n".format( tag, version ) )
+		sys.exit( 1 )
+
+## Build Name
+
+# We have a couple of naming conventions for builds, depending on the nature of the trigger.
+
+formatVars = {
+	"variant" : os.environ["DEPENDENCIES_BUILD_VARIANT"],
+	"timestamp" : datetime.datetime.now().strftime( "%Y_%m_%d_%H%M" ),
+	"pullRequest" : pullRequest,
+	"shortCommit" : commit[:8],
+	"tag" : tag,
+	"branch" : re.sub( r"[^a-zA-Z0-9_]", "", sourceBranch )
+}
+
+nameFormats = {
+	"default" : "gafferDependencies-{timestamp}-{shortCommit}-{variant}",
+	"pull_request" : "gafferDependencies-pr{pullRequest}-{branch}-{timestamp}-{shortCommit}-{variant}",
+	"release" : "gafferDependencies-{tag}-{variant}"
+}
+
+trigger = os.environ.get( 'GITHUB_EVENT_NAME', '' )
+
+# If we have a releaseID (and tag) then we always use release naming convention
+# to allow manual re-runs of release builds that fail for <reasons>.
+if tag and releaseId :
+	print( "Have Release ID %s for tag %s, using release naming." % ( releaseId, tag ) )
+	trigger = "release"
+
+buildName = nameFormats.get( trigger, nameFormats['default'] ).format( **formatVars )
+
+## Set vars in the downstream workflow environment
+
+with open( os.environ["GITHUB_ENV"], "a" ) as f :
+
+	print( "Setting $DEPENDENCIES_BUILD_NAME to '%s'" % buildName )
+	f.write( 'DEPENDENCIES_BUILD_NAME=%s\n' % buildName )
+
+	print( "Setting $DEPENDENCIES_SOURCE_COMMIT to '%s'" % commit )
+	f.write( 'DEPENDENCIES_SOURCE_COMMIT=%s\n' % commit )
+
+	print( "Setting $DEPENDENCIES_GITHUB_RELEASEID to '%s'" % releaseId )
+	f.write( 'DEPENDENCIES_GITHUB_RELEASEID=%s\n' % releaseId )
+
+	packageExtension = "zip" if os.name == "nt" else "tar.gz"
+	print( "Setting $PACKAGE_EXTENSION to '%s'" %packageExtension )
+	f.write( 'PACKAGE_EXTENSION=%s\n' %packageExtension )