Allow fetching previously built artifacts from S3

buildlist.sh

@@ -46,6 +46,7 @@ pkg_list_file="$_RET"
 logmust cd "$TOP"
 logmust make clean
 logmust mkdir artifacts
+logmust mkdir artifacts/cache
 
 #
 # Auto-generate the default revision for all the packages. It will be the
@@ -80,13 +81,41 @@ PACKAGES=("${_RET_LIST[@]}")
 for pkg in "${PACKAGES[@]}"; do
 	# shellcheck disable=SC2086
 	logmust ./buildpkg.sh $build_flags "$pkg"
+	echo ""
 done
 
 logmust build-info-pkg/build-package.sh "$pkg_list"
 logmust cp build-info-pkg/artifacts/* artifacts/
 
+fetched_from_cache=false
 for pkg in "${PACKAGES[@]}"; do
 	logmust cp "packages/$pkg/tmp/artifacts"/* artifacts/
+
+	#
+	# Cache each package's artifacts in a separate directory so that they
+	# can be easily retrieved by future linux-pkg builds. Note that those
+	# artifacts are not consumed by appliance-build.
+	#
+	logmust mkdir -p "artifacts/cache/$pkg/artifacts"
+	logmust cp "packages/$pkg/tmp/artifacts"/* \
+		"artifacts/cache/$pkg/artifacts/"
+	if [[ -f "packages/$pkg/tmp/build_info" ]]; then
+		logmust cp "packages/$pkg/tmp/build_info" \
+			"artifacts/cache/$pkg/"
+	fi
+	if [[ -f "packages/$pkg/tmp/fetched-from-cache" ]]; then
+		fetched_from_cache=true
+	fi
 done
 
+if $fetched_from_cache; then
+	echo_warn "The following package artifacts were not built, but" \
+		"fetched from $CACHED_ARTIFACTS_URL:"
+	for pkg in "${PACKAGES[@]}"; do
+		if [[ -f "packages/$pkg/tmp/fetched-from-cache" ]]; then
+			echo_warn " - $pkg"
+		fi
+	done
+fi
+
 echo_success "Packages have been built successfully."

buildpkg.sh

@@ -108,6 +108,43 @@ function inititalize_from_upstream() {
 	fi
 }
 
+function fetch_cached_artifacts() {
+	if [[ "$PACKAGE_CUSTOMIZED" != "false" ]]; then
+		echo_warn "Package $PACKAGE has custom build parameters."
+		return 1
+	fi
+
+	if [[ -z "$CACHED_ARTIFACTS_URL" ]]; then
+		echo_warn "CACHED_ARTIFACTS_URL is unset."
+		return 1
+	fi
+
+	if ! aws s3 ls "$CACHED_ARTIFACTS_URL" &>/dev/null; then
+		echo_warn "Invalid cached artifacts url: $CACHED_ARTIFACTS_URL"
+		return 1
+	fi
+
+	local cache_url="$CACHED_ARTIFACTS_URL/cache/$PACKAGE/"
+	if ! aws s3 ls "$cache_url" &>/dev/null; then
+		echo_warn "No cached artifacts for package (url: $cache_url)."
+		return 1
+	fi
+
+	logmust mkdir "$WORKDIR/cached"
+	logmust aws s3 cp --recursive --no-progress \
+		"$cache_url" "$WORKDIR/cached/"
+	logmust rm -rf "$WORKDIR/artifacts"
+	logmust cp -r "$WORKDIR/cached/artifacts" "$WORKDIR/artifacts"
+	if [[ -f "$WORKDIR/cached/build_info" ]]; then
+		logmust cp "$WORKDIR/cached/build_info" "$WORKDIR/build_info"
+	else
+		logmust touch "$WORKDIR/build_info"
+	fi
+	logmust bash -c "echo 'Fetched from $cache_url' >>'$WORKDIR/build_info'"
+	logmust touch "$WORKDIR/fetched-from-cache"
+	echo "Fetched artifacts for $PACKAGE from $cache_url"
+}
+
 function usage() {
 	[[ $# != 0 ]] && echo "$(basename "$0"): $*"
 	echo "Usage: $(basename "$0") [-i | -u [-M]] [-ch] [-g pkg_git_url]"
@@ -171,6 +208,15 @@ PACKAGE=$1
 $DO_UPDATE_PACKAGE && $do_initialize && usage "-i and -u are exclusive" >&2
 ! $do_merge && ! $DO_UPDATE_PACKAGE && usage "-M requires -u" >&2
 
+if [[ "$REUSE_CACHED_ARTIFACTS" == "always" ]] ||
+	[[ "$REUSE_CACHED_ARTIFACTS" == "on-failure" ]]; then
+	if $DO_UPDATE_PACKAGE || $do_initialize; then
+		echo_warn "REUSE_CACHED_ARTIFACTS unset because" \
+			"-i or -u is passed."
+		REUSE_CACHED_ARTIFACTS=""
+	fi
+fi
+
 logmust check_package_exists "$PACKAGE"
 
 #
@@ -192,6 +238,21 @@ logmust sudo rm -rf "$WORKDIR"
 logmust mkdir "$WORKDIR"
 logmust mkdir "$WORKDIR/artifacts"
 
+if [[ "$REUSE_CACHED_ARTIFACTS" == "always" ]]; then
+	#
+	# Fetch artifacts from cache. We execute in a subshell so that
+	# failures do not terminate this script.
+	#
+	echo "REUSE_CACHED_ARTIFACTS == 'always': fetch artifacts from" \
+		"previous build"
+	if (fetch_cached_artifacts); then
+		echo_success "Package $PACKAGE fetched from previous build."
+		exit 0
+	else
+		echo_warn "Package artifacts not fetched from previous build."
+	fi
+fi
+
 if $do_initialize; then
 	logmust inititalize_from_upstream
 	echo_success "Repository initialized from upstream in $WORKDIR/repo"
@@ -234,7 +295,24 @@ if $do_checkstyle; then
 	stage checkstyle
 fi
 logmust cd "$WORKDIR"
-stage build
+build_failed=false
+#
+# We execute the build stage in a subshell so that a failure doesn't
+# terminate this script.
+#
+(stage build) || build_failed=true
+if $build_failed && [[ $REUSE_CACHED_ARTIFACTS == "on-failure" ]]; then
+	echo_warn "Build failed, but REUSE_CACHED_ARTIFACTS == 'on-failure'"
+	echo_warn "Attempting to fetch artifacts from previous build."
+	if (logmust fetch_cached_artifacts); then
+		echo_success "Package $PACKAGE fetched from previous build."
+		exit 0
+	else
+		echo_error "Package artifacts not fetched from previous build."
+	fi
+elif $build_failed; then
+	exit 1
+fi
 logmust rm "$WORKDIR/building"
 
 echo_success "Package $PACKAGE has been built successfully."

lib/common.sh

@@ -27,6 +27,7 @@ function enable_colors() {
 	[[ -t 1 ]] && flags="" || flags="-T xterm"
 	FMT_RED="$(tput $flags setaf 1)"
 	FMT_GREEN="$(tput $flags setaf 2)"
+	FMT_YELLOW="$(tput $flags setaf 3)"
 	FMT_BOLD="$(tput $flags bold)"
 	FMT_NF="$(tput $flags sgr0)"
 	COLORS_ENABLED=true
@@ -35,6 +36,7 @@ function enable_colors() {
 function disable_colors() {
 	FMT_RED=""
 	FMT_GREEN=""
+	FMT_YELLOW=""
 	FMT_BOLD=""
 	FMT_NF=""
 	COLORS_ENABLED=false
@@ -64,6 +66,10 @@ function echo_success() {
 	echo -e "${FMT_BOLD}${FMT_GREEN}Success: $*${FMT_NF}"
 }
 
+function echo_warn() {
+	echo -e "${FMT_BOLD}${FMT_YELLOW}Warning: $*${FMT_NF}"
+}
+
 function echo_bold() {
 	echo -e "${FMT_BOLD}$*${FMT_NF}"
 }
@@ -325,13 +331,17 @@ function get_package_config_from_env() {
 
 	echo "get_package_config_from_env(): using prefix: ${PACKAGE_PREFIX}_"
 
+	export PACKAGE_CUSTOMIZED=false
+
 	var="${PACKAGE_PREFIX}_GIT_URL"
 	if [[ -n "$PARAM_PACKAGE_GIT_URL" ]]; then
 		PACKAGE_GIT_URL="$PARAM_PACKAGE_GIT_URL"
 		echo "PARAM_PACKAGE_GIT_URL passed from '-g'"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_GIT_URL="${!var}"
 		echo "PACKAGE_GIT_URL set to value of ${var}"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "$DEFAULT_PACKAGE_GIT_URL" ]]; then
 		PACKAGE_GIT_URL="$DEFAULT_PACKAGE_GIT_URL"
 		echo "PACKAGE_GIT_URL set to value of DEFAULT_PACKAGE_GIT_URL"
@@ -341,9 +351,11 @@ function get_package_config_from_env() {
 	if [[ -n "$PARAM_PACKAGE_GIT_BRANCH" ]]; then
 		PACKAGE_GIT_BRANCH="$PARAM_PACKAGE_GIT_BRANCH"
 		echo "PARAM_PACKAGE_GIT_BRANCH passed from '-b'"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_GIT_BRANCH="${!var}"
 		echo "PACKAGE_GIT_BRANCH set to value of ${var}"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "$DEFAULT_PACKAGE_GIT_BRANCH" ]]; then
 		PACKAGE_GIT_BRANCH="$DEFAULT_PACKAGE_GIT_BRANCH"
 		echo "PACKAGE_GIT_BRANCH set to value of" \
@@ -359,9 +371,11 @@ function get_package_config_from_env() {
 	if [[ -n "$PARAM_PACKAGE_VERSION" ]]; then
 		PACKAGE_VERSION="$PARAM_PACKAGE_VERSION"
 		echo "PACKAGE_VERSION passed from '-v'"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_VERSION="${!var}"
 		echo "PACKAGE_VERSION set to value of ${var}"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "$DEFAULT_PACKAGE_VERSION" ]]; then
 		PACKAGE_VERSION="$DEFAULT_PACKAGE_VERSION"
 		echo "PACKAGE_VERSION set to value of DEFAULT_PACKAGE_VERSION"
@@ -371,9 +385,11 @@ function get_package_config_from_env() {
 	if [[ -n "$PARAM_PACKAGE_REVISION" ]]; then
 		PACKAGE_REVISION="$PARAM_PACKAGE_REVISION"
 		echo "PACKAGE_REVISION passed from '-r'"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "${!var}" ]]; then
 		PACKAGE_REVISION="${!var}"
 		echo "PACKAGE_REVISION set to value of ${var}"
+		PACKAGE_CUSTOMIZED=true
 	elif [[ -n "$DEFAULT_PACKAGE_REVISION" ]]; then
 		PACKAGE_REVISION="$DEFAULT_PACKAGE_REVISION"
 		echo "PACKAGE_REVISION set to value of DEFAULT_PACKAGE_REVISION"