Merge pull request #10092 from EOSIO/release/2.0.x

Merge release/2.0.x into master

.cicd/build.sh

@@ -3,7 +3,8 @@ set -eo pipefail
 [[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :evergreen_tree: Configuring Environment'
 . ./.cicd/helpers/general.sh
 mkdir -p "$BUILD_DIR"
-CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"Release\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DBUILD_MONGO_DB_PLUGIN=\"true\""
+[[ -z "$DCMAKE_BUILD_TYPE" ]] && export DCMAKE_BUILD_TYPE='Release'
+CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DBUILD_MONGO_DB_PLUGIN=\"true\""
 if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then
     # You can't use chained commands in execute
     if [[ "$GITHUB_ACTIONS" == 'true' ]]; then

.cicd/create-docker-from-binary.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euo pipefail
+
+buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' .
+echo ":done: download successful"
+
+SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g')
+SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_')
+echo "$SANITIZED_BRANCH"
+echo "$SANITIZED_TAG"
+
+# do docker build
+echo ":docker::build: Building image..."
+DOCKERHUB_REGISTRY="docker.io/eosio/eosio"
+
+BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest}
+DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ."
+echo "$ $DOCKER_BUILD_GEN"
+eval $DOCKER_BUILD_GEN
+
+#tag and push on each destination AWS & DOCKERHUB
+
+EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY")
+for REG in ${EOSIO_REGS[@]}; do
+    DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT"
+    DOCKER_TAG_BRANCH="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_BRANCH"
+    echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH"
+    eval $DOCKER_TAG_COMMIT 
+    eval $DOCKER_TAG_BRANCH
+    DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT"
+    DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH"
+    echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH"
+    eval $DOCKER_PUSH_COMMIT 
+    eval $DOCKER_PUSH_BRANCH
+    CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT"
+    CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH"
+    echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$"
+    eval $CLEAN_IMAGE_COMMIT 
+    eval $CLEAN_IMAGE_BRANCH
+    if [[ ! -z "$SANITIZED_TAG" ]]; then
+        DOCKER_TAG="docker tag eosio_image $REG:$SANITIZED_TAG"
+        DOCKER_REM="docker rmi $REG:$SANITIZED_TAG"
+        echo -e "$ \n Tagging Image: \n$DOCKER_TAG \n Cleaning Up: \n$DOCKER_REM"
+        eval $DOCKER_TAG 
+        eval $DOCKER_REM
+    fi
+done
+
+DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG"
+echo "Clean up base image"
+eval $DOCKER_GEN

.cicd/docker-tag.sh

@@ -9,23 +9,17 @@ echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length'
 echo ${#CONTRACT_REGISTRIES[*]}
 echo '$ echo ${CONTRACT_REGISTRIES[*]} # array'
 echo ${CONTRACT_REGISTRIES[*]}
+export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
 # pull
 echo '+++ :arrow_down: Pulling Container(s)'
-for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
-    if [[ ! -z "$REGISTRY" ]]; then
-        echo "Pulling from '$REGISTRY'."
-        IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
-        DOCKER_PULL_COMMAND="docker pull '$IMAGE'"
-        echo "$ $DOCKER_PULL_COMMAND"
-        eval $DOCKER_PULL_COMMAND
-    fi
-done
+DOCKER_PULL_COMMAND="docker pull '$IMAGE'"
+echo "$ $DOCKER_PULL_COMMAND"
+eval $DOCKER_PULL_COMMAND
 # tag
 echo '+++ :label: Tagging Container(s)'
 for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
     if [[ ! -z "$REGISTRY" ]]; then
         echo "Tagging for registry $REGISTRY."
-        IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
         DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'"
         echo "$ $DOCKER_TAG_COMMAND"
         eval $DOCKER_TAG_COMMAND
@@ -56,15 +50,18 @@ echo '--- :put_litter_in_its_place: Cleaning Up'
 for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
     if [[ ! -z "$REGISTRY" ]]; then
         echo "Cleaning up from $REGISTRY."
-        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'"
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH' || :"
+        echo "$ $DOCKER_RMI_COMMAND"
+        eval $DOCKER_RMI_COMMAND
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :"
         echo "$ $DOCKER_RMI_COMMAND"
         eval $DOCKER_RMI_COMMAND
         if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then
-            DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG'"
+            DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :"
             echo "$ $DOCKER_RMI_COMMAND"
             eval $DOCKER_RMI_COMMAND
         fi
-        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE'"
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE' || :"
         echo "$ $DOCKER_RMI_COMMAND"
         eval $DOCKER_RMI_COMMAND
     fi

.cicd/generate-base-images.sh

@@ -5,39 +5,44 @@ set -eo pipefail
 # search for base image in docker registries
 echo '--- :docker: Build or Pull Base Image :minidisc:'
 echo "Looking for '$HASHED_IMAGE_TAG' container in our registries."
-EXISTS_ALL='true'
-EXISTS_DOCKER_HUB='false'
-EXISTS_ECR='false'
-for REGISTRY in ${CI_REGISTRIES[*]}; do
-    if [[ ! -z "$REGISTRY" ]]; then
-        MANIFEST_COMMAND="docker manifest inspect '$REGISTRY:$HASHED_IMAGE_TAG'"
-        echo "$ $MANIFEST_COMMAND"
-        set +e
-        eval $MANIFEST_COMMAND
-        MANIFEST_INSPECT_EXIT_STATUS="$?"
-        set -eo pipefail
-        if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then
-            if [[ "$(echo "$REGISTRY" | grep -icP '[.]amazonaws[.]com/')" != '0' ]]; then
-                EXISTS_ECR='true'
-            elif [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then
-                EXISTS_DOCKER_HUB='true'
-            fi
-        else
-            EXISTS_ALL='false'
-        fi
+export EXISTS_DOCKER_HUB='false'
+export EXISTS_MIRROR='false'
+MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'"
+echo "$ $MANIFEST_COMMAND"
+set +e
+eval $MANIFEST_COMMAND
+MANIFEST_INSPECT_EXIT_STATUS="$?"
+set -eo pipefail
+if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then
+    if [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then
+        export EXISTS_DOCKER_HUB='true'
+    else
+        export EXISTS_MIRROR='true'
+    fi
+fi
+# pull and copy as-necessary
+if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then
+    DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
+    echo "$ $DOCKER_PULL_COMMAND"
+    eval $DOCKER_PULL_COMMAND
+    # copy, if necessary
+    if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then
+        # tag
+        DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        echo "$ $DOCKER_TAG_COMMAND"
+        eval $DOCKER_TAG_COMMAND
+        # push
+        DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        echo "$ $DOCKER_PUSH_COMMAND"
+        eval $DOCKER_PUSH_COMMAND
+        export EXISTS_DOCKER_HUB='true'
     fi
-done
-# copy, if possible, since it is so much faster
-if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_BASE_IMAGE" != 'true' && ! -z "$MIRROR_REGISTRY" ]]; then
-    echo 'Attempting copy from Docker Hub to the mirror instead of a new base image build.'
+elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then
     DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
     echo "$ $DOCKER_PULL_COMMAND"
-    set +e
     eval $DOCKER_PULL_COMMAND
-    DOCKER_PULL_EXIT_STATUS="$?"
-    set -eo pipefail
-    if [[ "$DOCKER_PULL_EXIT_STATUS" == '0' ]]; then
-        echo 'Pull from Docker Hub worked! Pushing to mirror.'
+    # copy, if necessary
+    if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then
         # tag
         DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_TAG_COMMAND"
@@ -46,22 +51,19 @@ if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_
         DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_PUSH_COMMAND"
         eval $DOCKER_PUSH_COMMAND
-        EXISTS_ALL='true'
-        EXISTS_ECR='true'
-    else
-        echo 'Pull from Docker Hub failed, rebuilding base image from scratch.'
+        export EXISTS_MIRROR='true'
     fi
 fi
 # esplain yerself
-if [[ "$EXISTS_ALL" == 'false' ]]; then
+if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false' ]]; then
     echo 'Building base image from scratch.'
 elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then
     echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries."
 elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then
     echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries."
 fi
 # build, if neccessary
-if [[ "$EXISTS_ALL" == 'false' || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first
+if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first
     export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ."
     echo "$ $DOCKER_BUILD_COMMAND"
     eval $DOCKER_BUILD_COMMAND