diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..d138f06686 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,126 @@ +name: CI + +on: + push: + branches: + - main + - '*_maintenance' + pull_request: + branches: + - '*' + release: + types: [published] + +jobs: + + build: + + strategy: + + # Don't cancel other jobs in the build matrix if one job fails. + fail-fast: false + + matrix: + + # Rather than generate all permutations of various settings, + # we want to explicitly list each of the variants we want to + # test. We can use `name` to declare the names of our variants, + # and then use `include` to define their settings. + + name: [ + linux-gcc11, + macos-arm64, + ] + + include: + + - name: linux-gcc11 + os: ubuntu-20.04 + publish: true + containerImage: ghcr.io/gafferhq/build/build:3.0.0 + jobs: 4 + + - name: macos-arm64 + os: macos-14 + publish: true + containerImage: + jobs: 3 + + runs-on: ${{ matrix.os }} + + container: ${{ matrix.containerImage }} + + env: + DEPENDENCIES_BUILD_DIR: "./build" + steps: + + - uses: actions/checkout@v4 + + - name: Install toolchain (Linux) + run: | + # The Docker container configures bash shells such that they enable the + # software collections we want. If we could set GitHub's + # `defaults.run.shell` to `bash` then all our build steps would pick up + # this environment automatically. But we can't do that because it will + # break future Windows builds, and we can't configure a different shell + # per platform because GitHub won't allow it. But we can run _this_ + # Linux-only step in bash, and transfer the environment out to be used + # in later steps. + echo $PATH > $GITHUB_PATH + echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH >> $GITHUB_ENV + shell: bash + if: runner.os == 'Linux' + + - name: Install toolchain (macOS) + run: | + # Choose the earliest Xcode version available on a macos-14 runner. + sudo xcode-select -s /Applications/Xcode_14.3.1.app/Contents/Developer + # Install build requirements. + sudo pip3 install scons==4.6.0 --break-system-packages + brew install gpatch + brew install bison + # Make sure bison is discoverable. + echo BISON_ROOT=/opt/homebrew/opt/bison >> $GITHUB_ENV + # Make the location of the macOS platform SDK obvious to CMake. + # OpenShadingLanguage needs a little extra help finding `cstddef`. + echo SDKROOT=`xcrun --sdk macosx --show-sdk-path` >> $GITHUB_ENV + # Remove CommandLineTools so there is no potential for conflict with + # our selected Xcode version. + sudo rm -rf /Library/Developer/CommandLineTools + shell: bash + if: runner.os == 'macOS' + + - name: 'Install Python Modules' + run: | + python --version + pip install PyJWT==1.7.1 PyGitHub==1.45 + + - name: Set Custom Variables + run: | + .github/workflows/main/setBuildVars.py + env: + GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPENDENCIES_BUILD_VARIANT: ${{ matrix.name }} + shell: bash + + - name: Build Dependencies + run: | + ./build.py --cleanup --jobs ${{ matrix.jobs }} --buildDir ${{ env.DEPENDENCIES_BUILD_DIR }}/${{ env.DEPENDENCIES_BUILD_NAME }} --package ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} + env: + PYTHONUTF8: 1 + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.DEPENDENCIES_BUILD_NAME }} + path: ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} + # Using compression-level 0 avoids compressing our already compressed + # package and results in a significantly faster upload. + compression-level: 0 + if: matrix.publish + + - name: Publish Release + run: | + python ./.github/workflows/main/publishRelease.py --archive ${{ env.DEPENDENCIES_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} --repo ${{ github.repository }} --releaseId ${{ env.DEPENDENCIES_GITHUB_RELEASEID }} + if: matrix.publish && env.DEPENDENCIES_GITHUB_RELEASEID != '' + env: + GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main/publishRelease.py b/.github/workflows/main/publishRelease.py new file mode 100644 index 0000000000..a44f9d5471 --- /dev/null +++ b/.github/workflows/main/publishRelease.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright (c) 2019, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import github + +import argparse +import datetime +import os +import subprocess +import sys + +# A script to publish a build to a GitHub Release + +parser = argparse.ArgumentParser() + +parser.add_argument( + "--repo", + required = True, + help = "The GitHub organisation/repo to post the build link to." +) + +parser.add_argument( + "--releaseId", + type = int, + required = True, + help = "The release ID to publish the asset to." +) + +parser.add_argument( + "--archive", + dest = "archive", + required = True, + help = "The path to the build archive to publish." +) + +parser.add_argument( + "--github-access-token", + dest = "githubAccessToken", + default = os.environ.get( 'GITHUB_ACCESS_TOKEN', None ), + help = "A suitable access token to authenticate the GitHub API." +) + +args = parser.parse_args() + +if not args.githubAccessToken : + parser.exit( 1, "No --github-access-token/GITHUB_ACCESS_TOKEN set" ) + +if not os.path.exists( args.archive ) : + parser.exit( 1, "The specified archive '%s' does not exist." % args.archive ) + +githubClient = github.Github( args.githubAccessToken ) +repo = githubClient.get_repo( args.repo ) + +release = repo.get_release( args.releaseId ) +if not release : + parser.exit( 1, "Unable to find GitHub Release %s" % args.releaseId ) + +print( "Uploading '%s' to release %s" % ( args.archive, args.releaseId ) ) +asset = release.upload_asset( args.archive, content_type="application/gzip" ) + +print( "Success, %s available at %s" % ( args.archive, asset.browser_download_url ) ) diff --git a/.github/workflows/main/setBuildVars.py b/.github/workflows/main/setBuildVars.py new file mode 100755 index 0000000000..3eaef9c91e --- /dev/null +++ b/.github/workflows/main/setBuildVars.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of Cinesite VFX Ltd. nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import datetime +import github +import os +import re +import sys +import json + +# GitHub Action workflow variables can be populated at run-time by echoing special +# strings to an env file. The following allows vars to be set: +# +# https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files +# +# echo var=value >> $GITHUB_ENV +# +# We make use of this mechanism to allow custom logic to define the build name +# as well as determine the correct commit hash depending on the nature of the +# trigger. These variables can be referenced in a pipeline yaml file downstream +# of the step that runs this script. + +# Actions is somewhat sparse in what information is available via the GITHUB_* +# env vars (from the github context). There is however a veritable treasure +# trove in the .json file pointed to by GITHUB_EVENT_PATH. "The Internets" seem +# to suggest this is the most reliable way of determining information about the +# triggering commit. Some of the official context vars may vary after a retry, +# etc. too. +# +# The contents of this file is based on the webhook payload so should be +# relatively stable as it is part of that public API. + +with open( os.environ["GITHUB_EVENT_PATH"] ) as f : + eventData = json.load( f ) + +## Source Branches + +buildBranch = os.environ.get( "GITHUB_BASE_REF", "" ) +sourceBranch = os.environ.get( "GITHUB_HEAD_REF", buildBranch ) + +## Source Commit Hash + +commit = os.environ["GITHUB_SHA"] + +## Pull Request builds + +# Actions merges the branch into its target in PR build, so GITHUB_SHA isn't +# correct as it references the ephemeral merge. We also want to extract the +# pull request number for the build name. + +pullRequest = "" +if os.environ.get( "GITHUB_EVENT_NAME" ) == "pull_request" : + commit = eventData["pull_request"]["head"]["sha"] + pullRequest = eventData["pull_request"]["number"] + +## Tag builds + +# A variety of build types may be in service of a tag (ie: release publish +# or manual retry for a specific tag). + +tag = "" +if "/tags" in os.environ["GITHUB_REF"] : + tag = os.environ["GITHUB_REF"].replace( "refs/tags/", "" ) + +## Release ID + +# To allow builds to be published to a release, we need to lookup the ID of any +# release that matches the tag we're building, if there is one. + +releaseId = "" + +if tag : + + githubClient = github.Github( os.environ.get( 'GITHUB_ACCESS_TOKEN' ) ) + repo = githubClient.get_repo( os.environ.get( 'GITHUB_REPOSITORY' ) ) + + for r in repo.get_releases() : + if r.tag_name == tag : + releaseId = r.id + break + +if releaseId : + + # Check that the version specified by the SConstruct matches the + # version in the tag. + version = "" + versionRe = re.compile( r"^__version = (\S+)" ) + with open( "build.py" ) as buildPy : + for line in buildPy.readlines() : + versionMatch = versionRe.match( line ) + if versionMatch : + version = versionMatch.group( 2 ).strip( "'\"" ) + + if version != tag : + sys.stderr.write( "Tag \"{}\" does not match build.py __version \"{}\"\n".format( tag, version ) ) + sys.exit( 1 ) + +## Build Name + +# We have a couple of naming conventions for builds, depending on the nature of the trigger. + +formatVars = { + "variant" : os.environ["DEPENDENCIES_BUILD_VARIANT"], + "timestamp" : datetime.datetime.now().strftime( "%Y_%m_%d_%H%M" ), + "pullRequest" : pullRequest, + "shortCommit" : commit[:8], + "tag" : tag, + "branch" : re.sub( r"[^a-zA-Z0-9_]", "", sourceBranch ) +} + +nameFormats = { + "default" : "gafferDependencies-{timestamp}-{shortCommit}-{variant}", + "pull_request" : "gafferDependencies-pr{pullRequest}-{branch}-{timestamp}-{shortCommit}-{variant}", + "release" : "gafferDependencies-{tag}-{variant}" +} + +trigger = os.environ.get( 'GITHUB_EVENT_NAME', '' ) + +# If we have a releaseID (and tag) then we always use release naming convention +# to allow manual re-runs of release builds that fail for . +if tag and releaseId : + print( "Have Release ID %s for tag %s, using release naming." % ( releaseId, tag ) ) + trigger = "release" + +buildName = nameFormats.get( trigger, nameFormats['default'] ).format( **formatVars ) + +## Set vars in the downstream workflow environment + +with open( os.environ["GITHUB_ENV"], "a" ) as f : + + print( "Setting $DEPENDENCIES_BUILD_NAME to '%s'" % buildName ) + f.write( 'DEPENDENCIES_BUILD_NAME=%s\n' % buildName ) + + print( "Setting $DEPENDENCIES_SOURCE_COMMIT to '%s'" % commit ) + f.write( 'DEPENDENCIES_SOURCE_COMMIT=%s\n' % commit ) + + print( "Setting $DEPENDENCIES_GITHUB_RELEASEID to '%s'" % releaseId ) + f.write( 'DEPENDENCIES_GITHUB_RELEASEID=%s\n' % releaseId ) + + packageExtension = "zip" if os.name == "nt" else "tar.gz" + print( "Setting $PACKAGE_EXTENSION to '%s'" %packageExtension ) + f.write( 'PACKAGE_EXTENSION=%s\n' %packageExtension ) diff --git a/Changes.md b/Changes.md index 27a622271d..9daa688648 100644 --- a/Changes.md +++ b/Changes.md @@ -1,3 +1,10 @@ +9.x.x (relative to 8.x.x) +----- + +- CI : Added CI for linux-gcc11 and macos-arm64 (Xcode 14.3.1). +- LLVM : Updated to version 15.0.7. +- Cycles : Disabled CUDA binary generation for Kepler and Maxwell architecture GPUs. + 8.0.0 (relative to 7.0.0) ----- diff --git a/Cycles/patches/cudaBinaries.patch b/Cycles/patches/cudaBinaries.patch new file mode 100644 index 0000000000..07aa827d95 --- /dev/null +++ b/Cycles/patches/cudaBinaries.patch @@ -0,0 +1,10 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -120,7 +120,7 @@ + mark_as_advanced(WITH_CYCLES_DEVICE_CUDA) + + option(WITH_CYCLES_CUDA_BINARIES "Build Cycles NVIDIA CUDA binaries" OFF) +- set(CYCLES_CUDA_BINARIES_ARCH sm_30 sm_35 sm_37 sm_50 sm_52 sm_60 sm_61 sm_70 sm_75 sm_86 sm_89 compute_75 CACHE STRING "CUDA architectures to build binaries for") ++ set(CYCLES_CUDA_BINARIES_ARCH sm_60 sm_61 sm_70 sm_75 sm_86 sm_89 compute_75 CACHE STRING "CUDA architectures to build binaries for") + option(WITH_CYCLES_CUDA_BUILD_SERIAL "Build cubins one after another (useful on machines with limited RAM)" OFF) + option(WITH_CUDA_DYNLOAD "Dynamically load CUDA libraries at runtime (for developers, makes cuda-gdb work)" ON) diff --git a/LLVM/config.py b/LLVM/config.py index e0c8d6df03..3937bfec7c 100644 --- a/LLVM/config.py +++ b/LLVM/config.py @@ -2,8 +2,9 @@ "downloads" : [ - "https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/llvm-11.1.0.src.tar.xz", - "https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang-11.1.0.src.tar.xz" + "https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/llvm-15.0.7.src.tar.xz", + "https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/clang-15.0.7.src.tar.xz", + "https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.7/cmake-15.0.7.src.tar.xz" ], @@ -14,6 +15,7 @@ "commands" : [ "mv ../clang* tools/clang", + "mv ../cmake* ../cmake", "mkdir build", "cd build &&" " cmake" @@ -22,6 +24,10 @@ " -DCMAKE_BUILD_TYPE=Release" " -DLLVM_ENABLE_RTTI=ON" " -DLLVM_ENABLE_LIBXML2=OFF" + " -DLLVM_ENABLE_ZSTD=OFF" + " -DLLVM_INCLUDE_BENCHMARKS=OFF" + " -DLLVM_INCLUDE_TESTS=OFF" + " -DLLVM_TARGETS_TO_BUILD='Native'" " ..", "cd build && make install -j {jobs}" diff --git a/LLVM/patches/missingLimits.patch b/LLVM/patches/missingLimits.patch deleted file mode 100644 index 0d5535830b..0000000000 --- a/LLVM/patches/missingLimits.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- a/utils/benchmark/src/benchmark_register.h 2023-11-24 23:44:35.321178665 +0000 -+++ b/utils/benchmark/src/benchmark_register.h 2023-11-24 23:44:51.271329652 +0000 -@@ -1,6 +1,7 @@ - #ifndef BENCHMARK_REGISTER_H - #define BENCHMARK_REGISTER_H - -+#include - #include - - #include "check.h" \ No newline at end of file diff --git a/LibWebP/config.py b/LibWebP/config.py index c9f16accf1..4395fefd81 100644 --- a/LibWebP/config.py +++ b/LibWebP/config.py @@ -20,9 +20,9 @@ "manifest" : [ "include/webp", - "lib/libwebp{sharedLibraryExtension}*", - "lib/libwebpdemux{sharedLibraryExtension}*", - "lib/libsharpyuv{sharedLibraryExtension}*", + "lib/libwebp*{sharedLibraryExtension}*", + "lib/libwebpdemux*{sharedLibraryExtension}*", + "lib/libsharpyuv*{sharedLibraryExtension}*", ], diff --git a/OpenImageIO/config.py b/OpenImageIO/config.py index 5b331dbe2c..9265df759f 100644 --- a/OpenImageIO/config.py +++ b/OpenImageIO/config.py @@ -45,9 +45,16 @@ " -D USE_QT=NO" " ..", "cd gafferBuild && make install -j {jobs} VERBOSE=1", + "{extraCommands}", ], + "variables" : { + + "extraCommands" : "", + + }, + "manifest" : [ "bin/maketx", @@ -60,4 +67,14 @@ ], + "platform:macos" : { + + "variables" : { + + "extraCommands" : "mv {buildDir}/lib/python{pythonVersion}/site-packages/OpenImageIO {pythonLibDir}/python{pythonVersion}/site-packages/OpenImageIO" + + }, + + }, + } diff --git a/OpenShadingLanguage/config.py b/OpenShadingLanguage/config.py index 3bf64a5d78..226fc85f97 100644 --- a/OpenShadingLanguage/config.py +++ b/OpenShadingLanguage/config.py @@ -41,11 +41,13 @@ " ..", "cd gafferBuild && make install -j {jobs} VERBOSE=1", "cp {buildDir}/share/doc/OSL/osl-languagespec.pdf {buildDir}/doc", + "{extraCommands}", ], "variables" : { + "extraCommands" : "", "useBatched" : "b8_AVX,b8_AVX2,b8_AVX2_noFMA,b8_AVX512,b8_AVX512_noFMA,b16_AVX512,b16_AVX512_noFMA", }, @@ -67,8 +69,11 @@ "platform:macos" : { "variables" : { + + "extraCommands" : "mv {buildDir}/lib/python{pythonVersion}/site-packages/oslquery.so {pythonLibDir}/python{pythonVersion}/site-packages/oslquery.so", "useBatched" : "0", - } + + }, } diff --git a/PsUtil/config.py b/PsUtil/config.py index b48b8bf9d4..13acd48e45 100644 --- a/PsUtil/config.py +++ b/PsUtil/config.py @@ -22,14 +22,30 @@ "commands" : [ - "{buildDir}/bin/python setup.py install --root / --prefix {buildDir}", + "{buildDir}/bin/python setup.py install --root / --prefix {buildDir} {extraArgs}", ], + "variables" : { + + "extraArgs" : "", + + }, + "manifest" : [ "lib/python{pythonVersion}/site-packages/psutil", ], + "platform:macos" : { + + "variables" : { + + "extraArgs" : "--install-lib {pythonLibDir}/python{pythonVersion}/site-packages", + + } + + }, + } diff --git a/Python/config.py b/Python/config.py index c51df6a4c4..d10ccad443 100644 --- a/Python/config.py +++ b/Python/config.py @@ -28,7 +28,7 @@ "commands" : [ - "./configure --prefix={buildDir} {libraryType} --enable-unicode=ucs4 --with-ensurepip=install --with-system-ffi", + "./configure --prefix={buildDir} {libraryType} --with-ensurepip=install", "make -j {jobs}", "make install", diff --git a/build.py b/build.py index a905ec5071..700458e077 100755 --- a/build.py +++ b/build.py @@ -93,7 +93,7 @@ def __projects() : configFiles = glob.glob( "*/config.py" ) return sorted( [ os.path.split( f )[0] for f in configFiles ] ) -def __decompress( archive ) : +def __decompress( archive, cleanup = False ) : if os.path.splitext( archive )[1] == ".zip" : with zipfile.ZipFile( archive ) as f : @@ -114,6 +114,9 @@ def __decompress( archive ) : f.extractall() files = f.getnames() + if cleanup : + os.unlink( archive ) + dirs = { f.split( "/" )[0] for f in files } if len( dirs ) == 1 : # Well behaved archive with single top-level @@ -261,7 +264,7 @@ def decorated( *args, **kw ) : return decorated @__preserveCurrentDirectory -def __buildProject( project, config, buildDir ) : +def __buildProject( project, config, buildDir, cleanup ) : sys.stderr.write( "Building project {}\n".format( project ) ) @@ -287,8 +290,9 @@ def __buildProject( project, config, buildDir ) : shutil.rmtree( workingDir ) os.makedirs( workingDir ) os.chdir( workingDir ) + fullWorkingDir = os.getcwd() - decompressedArchives = [ __decompress( "../../" + a ) for a in archives ] + decompressedArchives = [ __decompress( "../../" + a, cleanup ) for a in archives ] os.chdir( config.get( "workingDir", decompressedArchives[0] ) ) if config["license"] is not None : @@ -320,6 +324,9 @@ def __buildProject( project, config, buildDir ) : os.remove( link[0] ) os.symlink( link[1], link[0] ) + if cleanup : + shutil.rmtree( fullWorkingDir ) + def __checkConfigs( projects, configs ) : def walk( project, configs ) : @@ -336,7 +343,7 @@ def walk( project, configs ) : for project in projects : walk( project, configs ) -def __buildProjects( projects, configs, buildDir ) : +def __buildProjects( projects, configs, buildDir, cleanup ) : digestsFilename = os.path.join( buildDir, ".digests" ) if os.path.isfile( digestsFilename ) : @@ -357,7 +364,7 @@ def walk( project, configs, buildDir ) : if digests.get( project ) == configs[project]["digest"].hexdigest() : sys.stderr.write( "Project {} is up to date : skipping\n".format( project ) ) else : - __buildProject( project, configs[project], buildDir ) + __buildProject( project, configs[project], buildDir, cleanup ) digests[project] = configs[project]["digest"].hexdigest() with open( digestsFilename, "w" ) as digestsFile : json.dump( digests, digestsFile, indent = 4 ) @@ -407,7 +414,7 @@ def walk( project, configs, buildDir ) : json.dump( projectManifest, file, indent = 4 ) rootName = os.path.basename( package ).replace( ".tar.gz", "" ) - with tarfile.open( package, "w:gz" ) as file : + with tarfile.open( package, "w:gz", compresslevel = 6 ) as file : for m in files : file.add( os.path.join( buildDir, m ), arcname = os.path.join( rootName, m ) ) @@ -440,6 +447,12 @@ def walk( project, configs, buildDir ) : help = "The number of build jobs to run in parallel. Defaults to cpu_count." ) +parser.add_argument( + "--cleanup", + action = "store_true", + help = "Delete archive files after extraction and 'working' directories after each project build completes." +) + for project in __projects() : config = __loadJSON( project ) @@ -484,7 +497,7 @@ def walk( project, configs, buildDir ) : __checkConfigs( args.projects, configs ) buildDir = variables["buildDir"].format( **variables ) -__buildProjects( args.projects, configs, buildDir ) +__buildProjects( args.projects, configs, buildDir, args.cleanup ) if args.package : __buildPackage( args.projects, configs, buildDir, args.package.format( **variables ) )