diff --git a/.circleci/cb-publish-step-1-set-versions.sh b/.circleci/cb-publish-step-1-set-versions.sh new file mode 100755 index 00000000000..05b53a755ee --- /dev/null +++ b/.circleci/cb-publish-step-1-set-versions.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +git config --global user.name aws-amplify-bot +git config --global user.email aws@amazon.com + +if [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then + if [[ "$BRANCH_NAME" =~ ^tagged-release-without-e2e-tests\/.* ]]; then + # Remove tagged-release-without-e2e-tests/ + export NPM_TAG="${BRANCH_NAME/tagged-release-without-e2e-tests\//}" + elif [[ "$BRANCH_NAME" =~ ^tagged-release\/.* ]]; then + # Remove tagged-release/ + export NPM_TAG="${BRANCH_NAME/tagged-release\//}" + fi + if [ -z "$NPM_TAG" ]; then + echo "Tag name is missing. Name your branch with either tagged-release/ or tagged-release-without-e2e-tests/" + exit 1 + fi + + npx lerna version --exact --preid=$NPM_TAG --conventional-commits --conventional-prerelease --yes --no-push --include-merged-tags --message "chore(release): Publish tagged release $NPM_TAG [ci skip]" --no-commit-hooks --force-publish '@aws-amplify/cli-internal' + +# @latest release +elif [[ "$BRANCH_NAME" == "release" ]]; then + # create release commit and release tags + npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]" --no-commit-hooks --force-publish '@aws-amplify/cli-internal' + +# release candidate or local publish for testing / building binary +else + # create release commit and release tags + npx lerna version --preid=rc.$(git rev-parse --short HEAD) --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" --no-commit-hooks --force-publish '@aws-amplify/cli-internal' +fi diff --git a/.circleci/cb-publish-step-2-verdaccio.sh b/.circleci/cb-publish-step-2-verdaccio.sh new file mode 100755 index 00000000000..c27881e6d6d --- /dev/null +++ b/.circleci/cb-publish-step-2-verdaccio.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly +# this causes the script to keep running even after failed publishes +# this function forces failed publishes to exit on failure +function lernaPublishExitOnFailure { + # exit on failure + set -e + # run lerna publish with the args that were passed to this function + # duplicate stdout to a temp file + # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code) + npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results +} + +npmRegistryUrl=$(npm get registry) +if [[ "$npmRegistryUrl" =~ ^http://localhost ]]; then + # registy URL update changes .yarnrc.yml file + git update-index --assume-unchanged .yarnrc.yml + + echo "Publishing to local registry under latest tag" + lernaPublishExitOnFailure from-git --yes --no-push +else + echo "NPM registry url is not pointing to localhost, $npmRegistryUrl" + exit 1 +fi diff --git a/.circleci/cb-publish-step-3-npm.sh b/.circleci/cb-publish-step-3-npm.sh new file mode 100755 index 00000000000..76a26c29dff --- /dev/null +++ b/.circleci/cb-publish-step-3-npm.sh @@ -0,0 +1,70 @@ +#!/bin/bash -e + +# lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly +# this causes the script to keep running even after failed publishes +# this function forces failed publishes to exit on failure +function lernaPublishExitOnFailure { + # exit on failure + set -e + # run lerna publish with the args that were passed to this function + # duplicate stdout to a temp file + # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code) + npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results +} + +# verifies that binaries are uploaded and available before publishing to NPM +function verifyPkgIsAvailable { + # exit on failure + set -e + + # read version of @aws-amplify/cli + desiredPkgVersion=$(npx lerna list --scope @aws-amplify/cli --json | jq -r '.[0].version') + + # check binaries + # send HEAD requests to check for binary presence + # curl --fail exits with non-zero code and makes this script fail + curl -I --fail https://$PKG_CLI_CLOUDFRONT_URL/$desiredPkgVersion/amplify-pkg-linux-x64.tgz + curl -I --fail https://$PKG_CLI_CLOUDFRONT_URL/$desiredPkgVersion/amplify-pkg-linux-arm64.tgz + curl -I --fail https://$PKG_CLI_CLOUDFRONT_URL/$desiredPkgVersion/amplify-pkg-macos-x64.tgz + curl -I --fail https://$PKG_CLI_CLOUDFRONT_URL/$desiredPkgVersion/amplify-pkg-win-x64.tgz +} + +if [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then + if [[ "$BRANCH_NAME" =~ ^tagged-release-without-e2e-tests\/.* ]]; then + # Remove tagged-release-without-e2e-tests/ + export NPM_TAG="${BRANCH_NAME/tagged-release-without-e2e-tests\//}" + elif [[ "$BRANCH_NAME" =~ ^tagged-release\/.* ]]; then + # Remove tagged-release/ + export NPM_TAG="${BRANCH_NAME/tagged-release\//}" + fi + if [ -z "$NPM_TAG" ]; then + echo "Tag name is missing. Name your branch with either tagged-release/ or tagged-release-without-e2e-tests/" + exit 1 + fi + + # verify that binary has been uploaded + verifyPkgIsAvailable + + echo "Publishing to NPM under $NPM_TAG tag" + lernaPublishExitOnFailure from-git --yes --no-push --dist-tag=$NPM_TAG + +# @latest release +elif [[ "$BRANCH_NAME" == "release" ]]; then + # verify that binary has been uploaded + verifyPkgIsAvailable + + # publish versions that were just computed + lernaPublishExitOnFailure from-git --yes --no-push + +# release candidate or local publish for testing / building binary +elif [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]]; then + + # verify that binary has been uploaded + verifyPkgIsAvailable + + # publish versions that were just computed + lernaPublishExitOnFailure from-git --yes --no-push --dist-tag rc +else + echo "branch name" "$BRANCH_NAME" "did not match any branch publish rules." + exit 1 +fi diff --git a/.circleci/cb-publish-step-4-push-to-git.sh b/.circleci/cb-publish-step-4-push-to-git.sh new file mode 100755 index 00000000000..4625fb966ab --- /dev/null +++ b/.circleci/cb-publish-step-4-push-to-git.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e + +git config --global user.name aws-amplify-bot +git config --global user.email aws@amazon.com + +if [[ "$BRANCH_NAME" =~ ^tagged-release ]] || [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]]; then + # push release commit + git push origin "$BRANCH_NAME" + + # push release tags + git tag --points-at HEAD | xargs git push origin + +# @latest release +elif [[ "$BRANCH_NAME" == "release" ]]; then + # push release commit + git push origin "$BRANCH_NAME" + + # push release tags + git tag --points-at HEAD | xargs git push origin + + # fast forward main to release + git fetch origin main + git checkout main + git merge release --ff-only + git push origin main + + # fast forward hotfix to release + git fetch origin hotfix + git checkout hotfix + git merge release --ff-only + git push origin hotfix +else + echo "branch name" "$BRANCH_NAME" "did not match any branch publish rules." + exit 1 +fi diff --git a/.circleci/local_publish_helpers_codebuild.sh b/.circleci/local_publish_helpers_codebuild.sh new file mode 100644 index 00000000000..899930efcc7 --- /dev/null +++ b/.circleci/local_publish_helpers_codebuild.sh @@ -0,0 +1,333 @@ +#!/bin/bash + +custom_registry_url=http://localhost:4873 +default_verdaccio_package=verdaccio@5.1.2 + +function startLocalRegistry { + # Start local registry + tmp_registry_log="$(mktemp)" + echo "Registry output file: $tmp_registry_log" + (cd && nohup npx ${VERDACCIO_PACKAGE:-$default_verdaccio_package} -c $1 &>$tmp_registry_log &) + # Wait for Verdaccio to boot + attempts=0 + until grep -q 'http address' $tmp_registry_log + do + attempts=$((attempts+1)) + echo "Waiting for Verdaccio, attempt $attempts" + sleep 1 + + if (( attempts > 60 )); then + echo "Verdaccio didn't start"; + exit 1 + fi + done +} + +function uploadPkgCliCodeBuild { + cd out/ + export hash=$(git rev-parse HEAD | cut -c 1-12) + export version=$(./amplify-pkg-linux-x64 --version) + + if [[ "$BRANCH_NAME" == "release" ]] || [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then + aws s3 cp amplify-pkg-win-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-win-x64-$(echo $hash).tgz + aws s3 cp amplify-pkg-macos-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-macos-x64-$(echo $hash).tgz + aws s3 cp amplify-pkg-linux-arm64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-arm64-$(echo $hash).tgz + aws s3 cp amplify-pkg-linux-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64-$(echo $hash).tgz + + ALREADY_EXISTING_FILES="$(set -o pipefail && aws s3 ls s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64 | ( egrep -v "amplify-pkg-linux-x64-.*" || true ) | wc -l | xargs)" + INCORRECT_PERMISSIONS=$? + + if [ INCORRECT_PERMISSIONS -ne "0" ]; then + echo "Insufficient permissions to list s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64" + exit 1 + fi + + if [ ALREADY_EXISTING_FILES -ne "0" ]; then + echo "Cannot overwrite existing file at s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64.tgz" + exit 1 + fi + + aws s3 cp amplify-pkg-win-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-win-x64.tgz + aws s3 cp amplify-pkg-macos-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-macos-x64.tgz + aws s3 cp amplify-pkg-linux-arm64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-arm64.tgz + aws s3 cp amplify-pkg-linux-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64.tgz + + else + aws s3 cp amplify-pkg-linux-x64.tgz s3://$PKG_CLI_BUCKET_NAME/$(echo $version)/amplify-pkg-linux-x64-$(echo $hash).tgz + fi + + cd .. +} + +function generatePkgCli { + cd pkg + + # install package depedencies + cp ../yarn.lock ./ + yarn workspaces focus --production + + # Optimize package size + find . \ + -name "*.d.ts" -or -name "*.js.map" -or -name "*.d.ts.map" -or \ + -iname "readme.md" -or -iname "changelog.md" -or -iname "history.md" \ + | xargs rm + + # Restore .d.ts files required by @aws-amplify/codegen-ui at runtime + cp ../node_modules/typescript/lib/*.d.ts node_modules/typescript/lib/ + + # replace DEV binary entry point with production one + cp ../node_modules/@aws-amplify/cli-internal/bin/amplify.production.template node_modules/@aws-amplify/cli-internal/bin/amplify + + # Transpile code for packaging + npx babel node_modules --extensions '.js,.jsx,.es6,.es,.ts' --copy-files --include-dotfiles -d ../build/node_modules + + # Include third party licenses + cp ../Third_Party_Licenses.txt ../build/node_modules + + # Build pkg cli + cp package.json ../build/node_modules/package.json + + if [[ "$@" =~ 'arm' ]]; then + npx pkg --no-bytecode --public-packages "*" --public -t node18-linux-arm64 ../build/node_modules -o ../out/amplify-pkg-linux-arm64 + tar -czvf ../out/amplify-pkg-linux-arm64.tgz ../out/amplify-pkg-linux-arm64 + fi + + if [[ "$@" =~ 'linux' ]]; then + npx pkg -t node18-linux-x64 ../build/node_modules -o ../out/amplify-pkg-linux-x64 + tar -czvf ../out/amplify-pkg-linux-x64.tgz ../out/amplify-pkg-linux-x64 + fi + + if [[ "$@" =~ 'macos' ]]; then + npx pkg -t node18-macos-x64 ../build/node_modules -o ../out/amplify-pkg-macos-x64 + tar -czvf ../out/amplify-pkg-macos-x64.tgz ../out/amplify-pkg-macos-x64 + fi + + if [[ "$@" =~ 'win' ]]; then + npx pkg -t node18-win-x64 ../build/node_modules -o ../out/amplify-pkg-win-x64.exe + tar -czvf ../out/amplify-pkg-win-x64.tgz ../out/amplify-pkg-win-x64.exe + fi + + cd .. +} + +function verifyPkgCli { + echo "Human readable sizes" + du -h out/* + echo "Sizes in bytes" + wc -c out/* + + function verifySinglePkg { + binary_name=$1 + compressed_binary_name=$2 + binary_threshold_in_bytes=$3 + + # Compressed binary size is not deterministic enough to have stricter threshold. + # I.e. it depends on how compression algorithm can compress bytecode and there are cases where compressed size + # grows even if uncompressed size drops. We don't have control on bytecode and compression. + # Therefore we check if compression gets past half of original size as sanity check. + compressed_binary_threshold_in_bytes=$((binary_threshold_in_bytes/2)) + + binary_size=$(wc -c out/$binary_name | awk '{print $1}') + compressed_binary_size=$(wc -c out/$compressed_binary_name | awk '{print $1}') + + if (( binary_size > binary_threshold_in_bytes )); then + echo "$binary_name size has grown over $binary_threshold_in_bytes bytes" + exit 1 + fi + + if (( compressed_binary_size > compressed_binary_threshold_in_bytes )); then + echo "$compressed_binary_name size has grown over $compressed_binary_threshold_in_bytes bytes" + exit 1 + fi + } + + verifySinglePkg "amplify-pkg-linux-x64" "amplify-pkg-linux-x64.tgz" $((750 * 1024 * 1024)) + verifySinglePkg "amplify-pkg-macos-x64" "amplify-pkg-macos-x64.tgz" $((750 * 1024 * 1024)) + verifySinglePkg "amplify-pkg-win-x64.exe" "amplify-pkg-win-x64.tgz" $((750 * 1024 * 1024)) + verifySinglePkg "amplify-pkg-linux-arm64" "amplify-pkg-linux-arm64.tgz" $((600 * 1024 * 1024)) +} + +function unsetNpmRegistryUrl { + # Restore the original NPM and Yarn registry URLs + npm set registry "https://registry.npmjs.org/" + yarn config set npmRegistryServer "https://registry.npmjs.org/" +} + +function unsetSudoNpmRegistryUrl { + # Restore the original NPM and Yarn registry URLs + sudo npm set registry "https://registry.npmjs.org/" + sudo yarn config set npmRegistryServer "https://registry.npmjs.org/" +} + +function changeNpmGlobalPath { + mkdir -p ~/.npm-global/{bin,lib} + npm config set prefix '~/.npm-global' + export PATH=~/.npm-global/bin:$PATH +} + +function changeSudoNpmGlobalPath { + mkdir -p ~/.npm-global-sudo + npm config set prefix '~/.npm-global-sudo' + export PATH=~/.npm-global/bin:$PATH +} + +function setNpmRegistryUrlToLocal { + # Set registry to local registry + npm set registry "$custom_registry_url" + yarn config set npmRegistryServer "$custom_registry_url" +} + +function setSudoNpmRegistryUrlToLocal { + # Set registry to local registry + sudo npm set registry "$custom_registry_url" + sudo yarn config set npmRegistryServer "$custom_registry_url" +} + +function useChildAccountCredentials { + if [[ ! -z "$USE_PARENT_ACCOUNT" ]]; then + echo "Using parent account credentials" + return + fi + export AWS_PAGER="" + parent_acct=$(aws sts get-caller-identity | jq -cr '.Account') + child_accts=$(aws organizations list-accounts | jq -c "[.Accounts[].Id | select(. != \"$parent_acct\")]") + org_size=$(echo $child_accts | jq 'length') + pick_acct=$(echo $child_accts | jq -cr ".[$RANDOM % $org_size]") + session_id=$((1 + $RANDOM % 10000)) + if [[ -z "$pick_acct" || -z "$session_id" ]]; then + echo "Unable to find a child account. Falling back to parent AWS account" + return + fi + creds=$(aws sts assume-role --role-arn arn:aws:iam::${pick_acct}:role/OrganizationAccountAccessRole --role-session-name testSession${session_id} --duration-seconds 3600) + if [ -z $(echo $creds | jq -c -r '.AssumedRoleUser.Arn') ]; then + echo "Unable to assume child account role. Falling back to parent AWS account" + return + fi + echo "Using account credentials for $(echo $creds | jq -c -r '.AssumedRoleUser.Arn')" + export AWS_ACCESS_KEY_ID=$(echo $creds | jq -c -r ".Credentials.AccessKeyId") + export AWS_SECRET_ACCESS_KEY=$(echo $creds | jq -c -r ".Credentials.SecretAccessKey") + export AWS_SESSION_TOKEN=$(echo $creds | jq -c -r ".Credentials.SessionToken") +} + +function retry { + MAX_ATTEMPTS=2 + SLEEP_DURATION=5 + FIRST_RUN=true + n=0 + FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt" + rm -f $FAILED_TEST_REGEX_FILE + until [ $n -ge $MAX_ATTEMPTS ] + do + echo "Attempting $@ with max retries $MAX_ATTEMPTS" + setAwsAccountCredentials + "$@" && break + n=$[$n+1] + FIRST_RUN=false + echo "Attempt $n completed." + sleep $SLEEP_DURATION + done + if [ $n -ge $MAX_ATTEMPTS ]; then + echo "failed: ${@}" >&2 + exit 1 + fi + + resetAwsAccountCredentials + TEST_SUITE=${TEST_SUITE:-"TestSuiteNotSet"} + # if a test takes a long time to complete, the token may expire before reaching this call, but we should still allow the test to pass + aws cloudwatch put-metric-data --metric-name FlakyE2ETests --namespace amplify-cli-e2e-tests --unit Count --value $n --dimensions testFile=$TEST_SUITE || true + echo "Attempt $n succeeded." + exit 0 # don't fail the step if putting the metric fails +} + +function resetAwsAccountCredentials { + if [ -z "$AWS_ACCESS_KEY_ID_ORIG" ]; then + echo "AWS Access Key environment variable is already set" + else + export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_ORIG + fi + if [ -z "$AWS_SECRET_ACCESS_KEY_ORIG" ]; then + echo "AWS Secret Access Key environment variable is already set" + else + export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_ORIG + fi + if [ -z "$AWS_SESSION_TOKEN_ORIG" ]; then + echo "AWS Session Token environment variable is already set" + else + export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN_ORIG + fi +} + +function setAwsAccountCredentials { + resetAwsAccountCredentials + export AWS_ACCESS_KEY_ID_ORIG=$AWS_ACCESS_KEY_ID + export AWS_SECRET_ACCESS_KEY_ORIG=$AWS_SECRET_ACCESS_KEY + export AWS_SESSION_TOKEN_ORIG=$AWS_SESSION_TOKEN + # introduce a delay of up to 1 minute to allow for more even spread aws list-accounts calls due to throttling + sleep $[ ( $RANDOM % 60 ) + 1 ]s + if [[ "$OSTYPE" == "msys" ]]; then + # windows provided by circleci has this OSTYPE + useChildAccountCredentials + else + echo "OSTYPE is $OSTYPE" + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o awscliv2.zip >/dev/null + export PATH=$PATH:$(pwd)/aws/dist + useChildAccountCredentials + fi +} + +function runE2eTestCb { + _setupCoverage + FAILED_TEST_REGEX_FILE="./amplify-e2e-reports/amplify-e2e-failed-test.txt" + + if [ -f $FAILED_TEST_REGEX_FILE ]; then + # read the content of failed tests + failedTests=$(<$FAILED_TEST_REGEX_FILE) + if [[ ! -z "$DISABLE_COVERAGE" ]]; then + echo Running WITHOUT coverage + yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE -t "$failedTests" + else + NODE_V8_COVERAGE=$E2E_TEST_COVERAGE_DIR yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE -t "$failedTests" + fi + else + if [[ ! -z "$DISABLE_COVERAGE" ]]; then + echo Running WITHOUT coverage + yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE + else + NODE_V8_COVERAGE=$E2E_TEST_COVERAGE_DIR yarn e2e --forceExit --no-cache --maxWorkers=4 $TEST_SUITE + fi + fi +} + +function _setupCoverage { + _teardownCoverage + echo "Setup Coverage ($E2E_TEST_COVERAGE_DIR)" + if [ ! -d $E2E_TEST_COVERAGE_DIR ] + then + mkdir -p $E2E_TEST_COVERAGE_DIR + fi +} + +function _teardownCoverage { + if [ -d $E2E_TEST_COVERAGE_DIR ] + then + echo "Teardown Coverage ($E2E_TEST_COVERAGE_DIR)" + rm -r $E2E_TEST_COVERAGE_DIR + fi +} + +function checkPackageVersionsInLocalNpmRegistry { + cli_internal_version=$(npm view @aws-amplify/cli-internal version) + cli_version=$(npm view @aws-amplify/cli version) + + echo "@aws-amplify/cli-internal version: $cli_internal_version" + echo "@aws-amplify/cli version: $cli_version" + + if [[ $cli_internal_version != $cli_version ]]; then + echo "Versions did not match." + echo "Manual fix: add a proper conventional commit that touches the amplify-cli-npm package to correct its version bump. For example https://github.com/aws-amplify/amplify-cli/commit/6f14792d1db424aa428ec4836fed7d6dd5cccfd0" + exit 1 + else + echo "Versions matched." + fi +} diff --git a/.circleci/publish-codebuild.sh b/.circleci/publish-codebuild.sh deleted file mode 100755 index ac9464cbf6a..00000000000 --- a/.circleci/publish-codebuild.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -e -export BRANCH_NAME="$(git symbolic-ref HEAD --short 2>/dev/null)" -if [ "$BRANCH_NAME" = "" ] ; then - BRANCH_NAME="$(git rev-parse HEAD | xargs git name-rev | cut -d' ' -f2 | sed 's/remotes\/origin\///g')"; -fi -git checkout $BRANCH_NAME -echo "fetching tags" -git fetch --tags https://github.com/aws-amplify/amplify-cli - -# lerna has a bug (https://github.com/lerna/lerna/issues/1066) where failed publishes do not set the exit code properly -# this causes the script to keep running even after failed publishes -# this function forces failed publishes to exit on failure -function lernaPublishExitOnFailure { - # exit on failure - set -e - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - # registy URL update changes .yarnrc.yml file - git update-index --assume-unchanged .yarnrc.yml - fi - - # run lerna publish with the args that were passed to this function - # duplicate stdout to a temp file - # grep the temp file for the lerna err token and return exit 1 if found (-v option inverts grep exit code) - npx lerna publish "$@" | tee /tmp/publish-results && grep -qvz "lerna ERR!" < /tmp/publish-results -} - -# verifies that binaries are uploaded and available before publishing to NPM -function verifyPkgIsAvailable { - # exit on failure - set -e - - # read version of @aws-amplify/cli - desiredPkgVersion=$(npx lerna list --scope @aws-amplify/cli --json | jq -r '.[0].version') - - # check binaries - # send HEAD requests to check for binary presence - # curl --fail exits with non-zero code and makes this script fail - curl -I --fail https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-linux-x64.tgz - curl -I --fail https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-linux-arm64.tgz - curl -I --fail https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-macos-x64.tgz - curl -I --fail https://package.cli.amplify.aws/$desiredPkgVersion/amplify-pkg-win-x64.tgz -} - -if [ -z "$GITHUB_EMAIL" ]; then - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - git config --global user.email not@used.com - else - echo "GITHUB_EMAIL email is missing" - exit 1 - fi -else - git config --global user.email $GITHUB_EMAIL -fi - -if [ -z "$GITHUB_USER" ]; then - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - git config --global user.name "Doesnt Matter" - else - echo "GITHUB_USER email is missing" - exit 1 - fi -else - git config --global user.name $GITHUB_USER -fi - -if [[ "$BRANCH_NAME" =~ ^tagged-release ]]; then - if [[ "$BRANCH_NAME" =~ ^tagged-release-without-e2e-tests\/.* ]]; then - # Remove tagged-release-without-e2e-tests/ - export NPM_TAG="${BRANCH_NAME/tagged-release-without-e2e-tests\//}" - elif [[ "$BRANCH_NAME" =~ ^tagged-release\/.* ]]; then - # Remove tagged-release/ - export NPM_TAG="${BRANCH_NAME/tagged-release\//}" - fi - if [ -z "$NPM_TAG" ]; then - echo "Tag name is missing. Name your branch with either tagged-release/ or tagged-release-without-e2e-tests/" - exit 1 - fi - - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - echo "Publishing to local registry under latest tag" - lernaPublishExitOnFailure --exact --preid=$NPM_TAG --conventional-commits --conventional-prerelease --no-push --yes --include-merged-tags - else - echo "Publishing to NPM under $NPM_TAG tag" - lernaPublishExitOnFailure --exact --dist-tag=$NPM_TAG --preid=$NPM_TAG --conventional-commits --conventional-prerelease --message "chore(release): Publish tagged release $NPM_TAG [ci skip]" --yes --include-merged-tags - fi - -# @latest release -elif [[ "$BRANCH_NAME" == "release" ]]; then - # create release commit and release tags - npx lerna version --exact --conventional-commits --conventional-graduate --yes --no-push --include-merged-tags --message "chore(release): Publish latest [ci skip]" - - if [[ "$LOCAL_PUBLISH_TO_LATEST" != "true" ]]; then - # verify that binary has been uploaded - verifyPkgIsAvailable - fi - - # publish versions that were just computed - lernaPublishExitOnFailure from-git --yes --no-push - - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - echo "Published packages to verdaccio" - echo "Exiting without pushing release commit or release tags" - exit 0 - fi - - # push release commit - git push origin "$BRANCH_NAME" - - # push release tags - git tag --points-at HEAD | xargs git push origin - - # fast forward main to release - git fetch origin main - git checkout main - git merge release --ff-only - git push origin main - - # fast forward hotfix to release - git fetch origin hotfix - git checkout hotfix - git merge release --ff-only - git push origin hotfix - -# release candidate or local publish for testing / building binary -elif [[ "$BRANCH_NAME" =~ ^run-e2e-with-rc\/.* ]] || [[ "$BRANCH_NAME" =~ ^release_rc\/.* ]] || [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - - # force @aws-amplify/cli-internal to be versioned in case this pipeline run does not have any commits that modify the CLI packages - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - force_publish_local_args="--force-publish '@aws-amplify/cli-internal'" - fi - # create release commit and release tags - npx lerna version --preid=rc.$(git rev-parse --short HEAD) --exact --conventional-prerelease --conventional-commits --yes --no-push --include-merged-tags --message "chore(release): Publish rc [ci skip]" $(echo $force_publish_local_args) --no-commit-hooks - - - # if publishing locally to verdaccio - if [[ "$LOCAL_PUBLISH_TO_LATEST" == "true" ]]; then - # publish to verdaccio with no dist tag (default to latest) - lernaPublishExitOnFailure from-git --yes --no-push - echo "Published packages to verdaccio" - echo "Exiting without pushing release commit or release tags" - exit 0 - fi - - # verify that binary has been uploaded - verifyPkgIsAvailable - - # publish versions that were just computed - lernaPublishExitOnFailure from-git --yes --no-push --dist-tag rc - - # push release commit - git push origin "$BRANCH_NAME" - - # push release tags - git tag --points-at HEAD | xargs git push origin -else - echo "branch name" "$BRANCH_NAME" "did not match any branch publish rules. Skipping publish" -fi diff --git a/.husky/pre-push b/.husky/pre-push index 33f88e8c09b..9af535e1b0b 100755 --- a/.husky/pre-push +++ b/.husky/pre-push @@ -7,5 +7,5 @@ CYAN='\033[0;36m' NC='\033[0m' # No color echo "${GREEN}Running pre-push hook${NC}" -echo "${CYAN}Running yarn build-tests-changed && yarn split-e2e-tests${NC}" -yarn build-tests-changed && yarn split-e2e-tests +echo "${CYAN}Running yarn build-tests-changed" +yarn build-tests-changed diff --git a/codebuild_specs/amplify_general_config_tests.yml b/codebuild_specs/amplify_general_config_tests.yml index cffa2c0785e..4b6b4006ae6 100644 --- a/codebuild_specs/amplify_general_config_tests.yml +++ b/codebuild_specs/amplify_general_config_tests.yml @@ -10,3 +10,6 @@ phases: build: commands: - source ./shared-scripts.sh && _amplifyGeneralConfigTests +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/amplify_install_test.yml b/codebuild_specs/amplify_install_test.yml index 2e9102193fb..dcdb663a6c7 100644 --- a/codebuild_specs/amplify_install_test.yml +++ b/codebuild_specs/amplify_install_test.yml @@ -15,5 +15,8 @@ phases: # i.e. not buffer content in memory while installing binary - ulimit -Sv 1000000 - npm install -g @aws-amplify/cli - - source .circleci/local_publish_helpers.sh && unsetNpmRegistryUrl + - source .circleci/local_publish_helpers_codebuild.sh && unsetNpmRegistryUrl - amplify version +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/amplify_sudo_install_test.yml b/codebuild_specs/amplify_sudo_install_test.yml index 66caf8249fd..43756b5c2d5 100644 --- a/codebuild_specs/amplify_sudo_install_test.yml +++ b/codebuild_specs/amplify_sudo_install_test.yml @@ -11,5 +11,8 @@ phases: commands: - source ./shared-scripts.sh && _amplifySudoInstallTestSetup - sudo npm install -g @aws-amplify/cli - - source .circleci/local_publish_helpers.sh && unsetSudoNpmRegistryUrl + - source .circleci/local_publish_helpers_codebuild.sh && unsetSudoNpmRegistryUrl # - amplify version +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/build_tests_standalone.yml b/codebuild_specs/build_tests_standalone.yml index f2409d537b4..6953e269e4e 100644 --- a/codebuild_specs/build_tests_standalone.yml +++ b/codebuild_specs/build_tests_standalone.yml @@ -6,3 +6,6 @@ phases: commands: - echo "Build Tests Standalone" - source ./shared-scripts.sh && _buildTestsStandalone +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/cleanup_resources.yml b/codebuild_specs/cleanup_resources.yml index c9249fc7446..552912cfc6c 100644 --- a/codebuild_specs/cleanup_resources.yml +++ b/codebuild_specs/cleanup_resources.yml @@ -5,3 +5,6 @@ phases: build: commands: - echo cleanup running +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/deployment_verification_post_release.yml b/codebuild_specs/deployment_verification_post_release.yml new file mode 100644 index 00000000000..dddebba497d --- /dev/null +++ b/codebuild_specs/deployment_verification_post_release.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _deploymentVerificationPostRelease + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/deployment_verification_rc_or_tagged.yml b/codebuild_specs/deployment_verification_rc_or_tagged.yml new file mode 100644 index 00000000000..59135009e72 --- /dev/null +++ b/codebuild_specs/deployment_verification_rc_or_tagged.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _deploymentVerificationRCOrTagged + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/e2e_workflow.yml b/codebuild_specs/e2e_workflow.yml index 12235137686..5c06dc13222 100644 --- a/codebuild_specs/e2e_workflow.yml +++ b/codebuild_specs/e2e_workflow.yml @@ -66,6 +66,10 @@ batch: compute-type: BUILD_GENERAL1_LARGE - identifier: verify_versions_match buildspec: codebuild_specs/verify_versions_match.yml + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE - identifier: run_e2e_tests_linux buildspec: codebuild_specs/run_e2e_tests_linux.yml env: diff --git a/codebuild_specs/e2e_workflow_base.yml b/codebuild_specs/e2e_workflow_base.yml index a0b4bb72159..7cb8fa87234 100644 --- a/codebuild_specs/e2e_workflow_base.yml +++ b/codebuild_specs/e2e_workflow_base.yml @@ -70,6 +70,10 @@ batch: compute-type: BUILD_GENERAL1_LARGE - identifier: verify_versions_match buildspec: codebuild_specs/verify_versions_match.yml + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE - identifier: integration_test buildspec: codebuild_specs/integration_test.yml env: diff --git a/codebuild_specs/lint.yml b/codebuild_specs/lint.yml index 7363c6c10d0..4dd41678530 100644 --- a/codebuild_specs/lint.yml +++ b/codebuild_specs/lint.yml @@ -5,3 +5,6 @@ phases: build: commands: - source ./shared-scripts.sh && _lint +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/mock_e2e_tests.yml b/codebuild_specs/mock_e2e_tests.yml index df964536b1b..51f4cacf9f7 100644 --- a/codebuild_specs/mock_e2e_tests.yml +++ b/codebuild_specs/mock_e2e_tests.yml @@ -9,3 +9,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs build_linux requirePrevJobsToSucceed - source ./shared-scripts.sh && _mockE2ETests +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/publish_to_npm.yml b/codebuild_specs/publish_to_npm.yml new file mode 100644 index 00000000000..d2afae54a99 --- /dev/null +++ b/codebuild_specs/publish_to_npm.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash + git-credential-helper: yes +phases: + build: + commands: + - source ./shared-scripts.sh && _publishToNpm +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/release_workflows/github_prerelease.yml b/codebuild_specs/release_workflows/github_prerelease.yml new file mode 100644 index 00000000000..a8f14403ccc --- /dev/null +++ b/codebuild_specs/release_workflows/github_prerelease.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _githubPrerelease + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/release_workflows/github_prerelease_install_sanity_check.yml b/codebuild_specs/release_workflows/github_prerelease_install_sanity_check.yml new file mode 100644 index 00000000000..eff54e84f33 --- /dev/null +++ b/codebuild_specs/release_workflows/github_prerelease_install_sanity_check.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _githubPrereleaseInstallSanityCheck + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/release_workflows/github_release.yml b/codebuild_specs/release_workflows/github_release.yml new file mode 100644 index 00000000000..2fe2313d8c1 --- /dev/null +++ b/codebuild_specs/release_workflows/github_release.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source ./shared-scripts.sh && _githubRelease + +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/release_workflows/hotfix_workflow.yml b/codebuild_specs/release_workflows/hotfix_workflow.yml new file mode 100644 index 00000000000..cff4c99c3d5 --- /dev/null +++ b/codebuild_specs/release_workflows/hotfix_workflow.yml @@ -0,0 +1,67 @@ +version: 0.2 +env: + shell: bash + compute-type: BUILD_GENERAL1_MEDIUM + variables: + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + CDK_DEFAULT_REGION: us-east-1 + CLI_REGION: us-east-1 + AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out' + AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64' + +batch: + fast-fail: false + build-graph: + - identifier: build_tests_standalone + buildspec: codebuild_specs/build_tests_standalone.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_linux + buildspec: codebuild_specs/build_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: test + buildspec: codebuild_specs/test.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: mock_e2e_tests + buildspec: codebuild_specs/mock_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: validate_cdk_version + buildspec: codebuild_specs/validate_cdk_version.yml + - identifier: verify_api_extract + buildspec: codebuild_specs/verify_api_extract.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_yarn_lock + buildspec: codebuild_specs/verify_yarn_lock.yml + - identifier: publish_to_local_registry + buildspec: codebuild_specs/publish_to_local_registry.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_arm + buildspec: codebuild_specs/build_pkg_binaries_arm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_linux + buildspec: codebuild_specs/build_pkg_binaries_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_macos + buildspec: codebuild_specs/build_pkg_binaries_macos.yml + env: + compute-type: BUILD_GENERAL1_LARGE + depend-on: + - publish_to_local_registry + - identifier: build_pkg_binaries_win + buildspec: codebuild_specs/build_pkg_binaries_win.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_versions_match + buildspec: codebuild_specs/verify_versions_match.yml + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE diff --git a/codebuild_specs/release_workflows/post_publish_push_to_git.yml b/codebuild_specs/release_workflows/post_publish_push_to_git.yml new file mode 100644 index 00000000000..e20a0c7a6a5 --- /dev/null +++ b/codebuild_specs/release_workflows/post_publish_push_to_git.yml @@ -0,0 +1,8 @@ +version: 0.2 +env: + shell: bash + git-credential-helper: yes +phases: + build: + commands: + - source ./shared-scripts.sh && _postPublishPushToGit diff --git a/codebuild_specs/release_workflows/release_rc_workflow.yml b/codebuild_specs/release_workflows/release_rc_workflow.yml new file mode 100644 index 00000000000..31f1febf702 --- /dev/null +++ b/codebuild_specs/release_workflows/release_rc_workflow.yml @@ -0,0 +1,83 @@ +version: 0.2 +env: + shell: bash + compute-type: BUILD_GENERAL1_MEDIUM + git-credential-helper: yes + variables: + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + CDK_DEFAULT_REGION: us-east-1 + CLI_REGION: us-east-1 + AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out' + AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64' + +batch: + fast-fail: false + build-graph: + - identifier: build_linux + buildspec: codebuild_specs/build_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: validate_cdk_version + buildspec: codebuild_specs/validate_cdk_version.yml + - identifier: verify_yarn_lock + buildspec: codebuild_specs/verify_yarn_lock.yml + - identifier: publish_to_local_registry + buildspec: codebuild_specs/publish_to_local_registry.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_arm + buildspec: codebuild_specs/build_pkg_binaries_arm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_linux + buildspec: codebuild_specs/build_pkg_binaries_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_macos + buildspec: codebuild_specs/build_pkg_binaries_macos.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_win + buildspec: codebuild_specs/build_pkg_binaries_win.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: upb + buildspec: codebuild_specs/upload_pkg_binaries.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_versions_match + buildspec: codebuild_specs/verify_versions_match.yml + - identifier: publish_to_npm + buildspec: codebuild_specs/publish_to_npm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - upb + - publish_to_local_registry + - verify_versions_match + - validate_cdk_version + - verify_yarn_lock + - verify_pkg_cli + - identifier: deployment_verification_rc_or_tagged + buildspec: codebuild_specs/deployment_verification_rc_or_tagged.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - publish_to_npm + - identifier: post_publish_push_to_git + buildspec: codebuild_specs/release_workflows/post_publish_push_to_git.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - publish_to_npm diff --git a/codebuild_specs/release_workflows/release_workflow.yml b/codebuild_specs/release_workflows/release_workflow.yml new file mode 100644 index 00000000000..7324bc32f70 --- /dev/null +++ b/codebuild_specs/release_workflows/release_workflow.yml @@ -0,0 +1,100 @@ +version: 0.2 +env: + shell: bash + compute-type: BUILD_GENERAL1_MEDIUM + git-credential-helper: yes + variables: + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + CDK_DEFAULT_REGION: us-east-1 + CLI_REGION: us-east-1 + AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out' + AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64' + +batch: + fast-fail: false + build-graph: + - identifier: build_linux + buildspec: codebuild_specs/build_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: validate_cdk_version + buildspec: codebuild_specs/validate_cdk_version.yml + - identifier: verify_yarn_lock + buildspec: codebuild_specs/verify_yarn_lock.yml + - identifier: publish_to_local_registry + buildspec: codebuild_specs/publish_to_local_registry.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_arm + buildspec: codebuild_specs/build_pkg_binaries_arm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_linux + buildspec: codebuild_specs/build_pkg_binaries_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_macos + buildspec: codebuild_specs/build_pkg_binaries_macos.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_win + buildspec: codebuild_specs/build_pkg_binaries_win.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: upb + buildspec: codebuild_specs/upload_pkg_binaries.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_versions_match + buildspec: codebuild_specs/verify_versions_match.yml + - identifier: github_prerelease + buildspec: codebuild_specs/release_workflows/github_prerelease.yml + env: + compute-type: BUILD_GENERAL1_LARGE + debug-session: true + depend-on: + - upb + - identifier: github_prerelease_install_sanity_check + buildspec: codebuild_specs/release_workflows/github_prerelease_install_sanity_check.yml + env: + compute-type: BUILD_GENERAL1_LARGE + debug-session: true + depend-on: + - github_prerelease + - identifier: publish_to_npm + buildspec: codebuild_specs/publish_to_npm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + debug-session: true + depend-on: + - upb + - verify_versions_match + - github_prerelease_install_sanity_check + - publish_to_local_registry + - validate_cdk_version + - verify_yarn_lock + - verify_pkg_cli + - identifier: post_publish_push_to_git + buildspec: codebuild_specs/release_workflows/post_publish_push_to_git.yml + depend-on: + - publish_to_npm + - identifier: github_release + buildspec: codebuild_specs/release_workflows/github_release.yml + env: + compute-type: BUILD_GENERAL1_LARGE + debug-session: true + depend-on: + - post_publish_push_to_git + - identifier: deployment_verification_post_release + buildspec: codebuild_specs/deployment_verification_post_release.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - github_release diff --git a/codebuild_specs/release_workflows/tagged_release_without_e2e_workflow.yml b/codebuild_specs/release_workflows/tagged_release_without_e2e_workflow.yml new file mode 100644 index 00000000000..284627c01f8 --- /dev/null +++ b/codebuild_specs/release_workflows/tagged_release_without_e2e_workflow.yml @@ -0,0 +1,81 @@ +version: 0.2 +env: + shell: bash + compute-type: BUILD_GENERAL1_MEDIUM + variables: + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + CDK_DEFAULT_REGION: us-east-1 + CLI_REGION: us-east-1 + AMPLIFY_DIR: '$CODEBUILD_SRC_DIR/out' + AMPLIFY_PATH: '$CODEBUILD_SRC_DIR/out/amplify-pkg-linux-x64' + +batch: + fast-fail: false + build-graph: + - identifier: build_linux + buildspec: codebuild_specs/build_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: validate_cdk_version + buildspec: codebuild_specs/validate_cdk_version.yml + - identifier: verify_yarn_lock + buildspec: codebuild_specs/verify_yarn_lock.yml + - identifier: publish_to_local_registry + buildspec: codebuild_specs/publish_to_local_registry.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_arm + buildspec: codebuild_specs/build_pkg_binaries_arm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_linux + buildspec: codebuild_specs/build_pkg_binaries_linux.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_macos + buildspec: codebuild_specs/build_pkg_binaries_macos.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: build_pkg_binaries_win + buildspec: codebuild_specs/build_pkg_binaries_win.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: upb + buildspec: codebuild_specs/upload_pkg_binaries.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_pkg_cli + buildspec: codebuild_specs/verify_pkg_cli.yml + env: + compute-type: BUILD_GENERAL1_LARGE + - identifier: verify_versions_match + buildspec: codebuild_specs/verify_versions_match.yml + - identifier: publish_to_npm + buildspec: codebuild_specs/publish_to_npm.yml + env: + compute-type: BUILD_GENERAL1_LARGE + debug-session: true + depend-on: + - upb + - verify_versions_match + - publish_to_local_registry + - validate_cdk_version + - verify_yarn_lock + - verify_pkg_cli + - identifier: deployment_verification_rc_or_tagged + buildspec: codebuild_specs/deployment_verification_rc_or_tagged.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - publish_to_npm + - identifier: post_publish_push_to_git + buildspec: codebuild_specs/release_workflows/post_publish_push_to_git.yml + env: + compute-type: BUILD_GENERAL1_LARGE + git-credential-helper: yes + debug-session: true + depend-on: + - publish_to_npm diff --git a/codebuild_specs/scripts-windows/run-e2e-windows.sh b/codebuild_specs/scripts-windows/run-e2e-windows.sh index 6b608e450ab..0e1304e59d5 100644 --- a/codebuild_specs/scripts-windows/run-e2e-windows.sh +++ b/codebuild_specs/scripts-windows/run-e2e-windows.sh @@ -10,7 +10,7 @@ export AMPLIFY_DIR=$CODEBUILD_SRC_DIR\\out export AMPLIFY_PATH=$CODEBUILD_SRC_DIR\\out\\amplify.exe export NODE_OPTIONS=--max-old-space-size=5120 -source .circleci/local_publish_helpers.sh +source .circleci/local_publish_helpers_codebuild.sh source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh # source $BASH_ENV diff --git a/codebuild_specs/test.yml b/codebuild_specs/test.yml index f112bb8d21a..4e6cae52675 100644 --- a/codebuild_specs/test.yml +++ b/codebuild_specs/test.yml @@ -8,3 +8,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs build_linux requirePrevJobsToSucceed - source ./shared-scripts.sh && _testLinux +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/validate_cdk_version.yml b/codebuild_specs/validate_cdk_version.yml index 0919312000b..6c0dfd0ddb2 100644 --- a/codebuild_specs/validate_cdk_version.yml +++ b/codebuild_specs/validate_cdk_version.yml @@ -6,3 +6,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs build_linux requirePrevJobsToSucceed - source ./shared-scripts.sh && _validateCDKVersion +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/verify_api_extract.yml b/codebuild_specs/verify_api_extract.yml index 253eccb4c02..854f554a796 100644 --- a/codebuild_specs/verify_api_extract.yml +++ b/codebuild_specs/verify_api_extract.yml @@ -6,3 +6,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs build_linux requirePrevJobsToSucceed - source ./shared-scripts.sh && _verifyAPIExtract +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/verify_pkg_cli.yml b/codebuild_specs/verify_pkg_cli.yml new file mode 100644 index 00000000000..b26a8849c6b --- /dev/null +++ b/codebuild_specs/verify_pkg_cli.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: bash +phases: + build: + commands: + - source shared-scripts.sh && _waitForJobs $CODEBUILD_SRC_DIR/codebuild_specs/wait_upb.json requirePrevJobsToSucceed + - source ./shared-scripts.sh && _verifyPkgCLI +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/verify_versions_match.yml b/codebuild_specs/verify_versions_match.yml index 175366e1bd4..a8d64d2321e 100644 --- a/codebuild_specs/verify_versions_match.yml +++ b/codebuild_specs/verify_versions_match.yml @@ -6,3 +6,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs publish_to_local_registry requirePrevJobsToSucceed - source ./shared-scripts.sh && _verifyVersionsMatch +artifacts: + files: + - 'shared-scripts.sh' diff --git a/codebuild_specs/verify_yarn_lock.yml b/codebuild_specs/verify_yarn_lock.yml index ba66b74c16d..80bab34bdbe 100644 --- a/codebuild_specs/verify_yarn_lock.yml +++ b/codebuild_specs/verify_yarn_lock.yml @@ -6,3 +6,6 @@ phases: commands: - source shared-scripts.sh && _waitForJobs build_linux requirePrevJobsToSucceed - source ./shared-scripts.sh && _verifyYarnLock +artifacts: + files: + - 'shared-scripts.sh' diff --git a/package.json b/package.json index 3310d56f365..747a403574d 100644 --- a/package.json +++ b/package.json @@ -35,16 +35,25 @@ "lint-fix": "git diff --name-only --cached --diff-filter d | grep -E '\\.(js|jsx|ts|tsx)$' | xargs eslint --fix --quiet", "mergewords": "yarn ts-node ./scripts/handle-dict-conflicts.ts", "pkg-all-local": "yarn verdaccio-clean && yarn verdaccio-start && yarn verdaccio-connect && yarn publish-to-verdaccio && yarn pkg-all && yarn verdaccio-disconnect", - "pkg-all": "bash -c 'source .circleci/local_publish_helpers.sh && generatePkgCli'", + "pkg-all": "bash -c 'source .circleci/local_publish_helpers_codebuild.sh && generatePkgCli'", "pkg-clean": "rimraf build out pkg/node_modules pkg/yarn.lock", "postinstall": "husky install", "prettier-changes": "git diff --name-only --diff-filter MRA dev | xargs yarn prettier --write", "prettier-check": "yarn prettier --check .", "prettier-write": "yarn prettier --write .", "production-build": "lerna run build --concurrency 3 --stream", - "promote-rc": "./scripts/promote-rc.sh", + "promote-rc-local": "bash -c 'source ./scripts/cloud-release.sh && ReleaseLocal'", + "promote-rc-beta": "bash -c 'source ./scripts/cloud-release.sh && ReleaseBeta'", + "promote-rc": "bash -c 'source ./scripts/cloud-release.sh && ReleaseProd'", + "promote-rc-circleci": "./scripts/promote-rc.sh", "publish-to-verdaccio": "lerna publish --yes --no-commit-hooks --no-push --exact --dist-tag=latest --conventional-commits --no-git-tag-version --no-verify-access", - "release-rc": "./scripts/release-rc.sh", + "release-rc-local": "bash -c 'source ./scripts/cloud-release.sh && RCLocal'", + "release-rc-beta": "bash -c 'source ./scripts/cloud-release.sh && RCBeta'", + "release-rc": "bash -c 'source ./scripts/cloud-release.sh && RCProd'", + "release-rc-circleci": "./scripts/release-rc.sh", + "tagged-release-without-e2e-local": "bash -c 'source ./scripts/cloud-release.sh && TaggedRCLocal'", + "tagged-release-without-e2e-beta": "bash -c 'source ./scripts/cloud-release.sh && TaggedRCBeta'", + "tagged-release-without-e2e-prod": "bash -c 'source ./scripts/cloud-release.sh && TaggedRCProd'", "rm-aa-dev-link": "rimraf -f \".bin/amplify-app-dev\"", "rm-dev-link": "rimraf -f \".bin/amplify-dev\"", "setup-dev-win": "yarn build && yarn link-win && yarn link-aa-win", @@ -62,9 +71,9 @@ "update-test-timing-data": "ts-node ./scripts/cci-get-job-metrics.ts && ts-node ./scripts/cci-extract-test-timings-from-job-metrics.ts", "update-versions": "lerna version --yes --no-commit-hooks --no-push --exact --conventional-commits --no-git-tag-version", "verdaccio-clean": "rimraf ../verdaccio-cache", - "verdaccio-connect": "bash -c 'source .circleci/local_publish_helpers.sh && setNpmRegistryUrlToLocal'", - "verdaccio-disconnect": "bash -c 'source .circleci/local_publish_helpers.sh && unsetNpmRegistryUrl'", - "verdaccio-start": "bash -c 'source .circleci/local_publish_helpers.sh && startLocalRegistry \"$(pwd)/.circleci/verdaccio.yaml\"'", + "verdaccio-connect": "bash -c 'source .circleci/local_publish_helpers_codebuild.sh && setNpmRegistryUrlToLocal'", + "verdaccio-disconnect": "bash -c 'source .circleci/local_publish_helpers_codebuild.sh && unsetNpmRegistryUrl'", + "verdaccio-start": "bash -c 'source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry \"$(pwd)/.circleci/verdaccio.yaml\"'", "verdaccio-stop": "kill -9 $(lsof -n -t -iTCP:4873 -sTCP:LISTEN) || true", "verify-api-extract:clean": "yarn clean && yarn build && yarn verify-api-extract", "verify-api-extract": "yarn extract-api && ./scripts/verify-extract-api.sh", diff --git a/scripts/cloud-cli-utils.sh b/scripts/cloud-cli-utils.sh new file mode 100644 index 00000000000..949354b8d54 --- /dev/null +++ b/scripts/cloud-cli-utils.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# set exit on error to true +set -e +# load .env +set -o allexport +source ./scripts/.env set + +function authenticate { + account_number=$1 + role_name=$2 + profile_name=$3 + echo Authenticating terminal... + mwinit --aea + echo Loading account credentials for Account $account_number with Role: $role_name... + ada cred update --profile="${profile_name}" --account="${account_number}" --role=${role_name} --provider=isengard --once + aws configure set region us-east-1 --profile $profile_name +} +function triggerProjectBatch { + account_number=$1 + role_name=$2 + profile_name=$3 + project_name=$4 + target_branch=$5 + authenticate $account_number $role_name $profile_name + echo AWS Account: $account_number + echo Project: $project_name + echo Target Branch: $target_branch + RESULT=$(aws codebuild start-build-batch --profile="${profile_name}" --project-name $project_name --source-version=$target_branch \ + --environment-variables-override name=BRANCH_NAME,value=$target_branch,type=PLAINTEXT \ + --query 'buildBatch.id' --output text) + echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$account_number/projects/$project_name/batch/$RESULT?region=us-east-1" +} diff --git a/scripts/cloud-e2e.sh b/scripts/cloud-e2e.sh index 86bf1f56269..e8b69ad6f90 100644 --- a/scripts/cloud-e2e.sh +++ b/scripts/cloud-e2e.sh @@ -1,49 +1,22 @@ #!/bin/bash -# this file is used to automate the process of triggering an e2e build for each environment type - -# set exit on error to true -set -e -# load .env -set -o allexport -source ./scripts/.env set +source ./scripts/cloud-cli-utils.sh export CURR_BRANCH=$(git branch --show-current) -function authenticate { - echo Authenticating terminal... - mwinit --aea - echo Loading E2E account credentials... - ada cred update --profile="${CLOUD_E2E_PROFILE}" --account="${CLOUD_E2E_ACCOUNT}" --role=CodeBuildE2E --provider=isengard --once - aws configure set region us-east-1 --profile $CLOUD_E2E_PROFILE -} -function triggerBuild { - echo Submitting CodeBuild Request to AWS Account: $CLOUD_E2E_ACCOUNT - echo Current branch is: $CURR_BRANCH - echo E2E Target branch is: $TARGET_BRANCH - RESULT=$(aws codebuild start-build-batch --profile="${CLOUD_E2E_PROFILE}" --project-name AmplifyCLI-E2E-Testing --source-version=$TARGET_BRANCH --environment-variables-override name=BRANCH_NAME,value=$TARGET_BRANCH,type=PLAINTEXT --query 'buildBatch.id' --output text) - echo "https://us-east-1.console.aws.amazon.com/codesuite/codebuild/$CLOUD_E2E_ACCOUNT/projects/AmplifyCLI-E2E-Testing/batch/$RESULT?region=us-east-1" -} +export E2E_ROLE_NAME=CodeBuildE2E +export E2E_PROFILE_NAME=AmplifyCLIE2E +export E2E_PROJECT_NAME=AmplifyCLI-E2E-Testing + function cloudE2ELocal { echo Running Local E2E Test Suite - export CLOUD_E2E_PROFILE=AmplifyCLIE2ELocal - export CLOUD_E2E_ACCOUNT=$E2E_ACCOUNT_LOCAL - export TARGET_BRANCH=$CURR_BRANCH - authenticate - triggerBuild + triggerProjectBatch $E2E_ACCOUNT_LOCAL $E2E_ROLE_NAME "${E2E_PROFILE_NAME}Local" $E2E_PROJECT_NAME $CURR_BRANCH } function cloudE2EBeta { echo Running Beta E2E Test Suite - export CLOUD_E2E_PROFILE=AmplifyCLIE2EBeta - export CLOUD_E2E_ACCOUNT=$E2E_ACCOUNT_BETA - export TARGET_BRANCH=$CURR_BRANCH - authenticate - triggerBuild + triggerProjectBatch $E2E_ACCOUNT_BETA $E2E_ROLE_NAME "${E2E_PROFILE_NAME}Beta" $E2E_PROJECT_NAME $CURR_BRANCH } function cloudE2E { echo Running Prod E2E Test Suite - export CLOUD_E2E_PROFILE=AmplifyCLIE2E - export CLOUD_E2E_ACCOUNT=$E2E_ACCOUNT_PROD export TARGET_BRANCH=run-cb-e2e/$USER/$CURR_BRANCH - authenticate git push $(git remote -v | grep aws-amplify/amplify-cli | head -n1 | awk '{print $1;}') $CURR_BRANCH:$TARGET_BRANCH --no-verify --force-with-lease - triggerBuild + triggerProjectBatch $E2E_ACCOUNT_PROD $E2E_ROLE_NAME "${E2E_PROFILE_NAME}Prod" $E2E_PROJECT_NAME $TARGET_BRANCH } diff --git a/scripts/cloud-release.sh b/scripts/cloud-release.sh new file mode 100644 index 00000000000..b5e14c91038 --- /dev/null +++ b/scripts/cloud-release.sh @@ -0,0 +1,101 @@ +#!/bin/bash +source ./scripts/cloud-cli-utils.sh +export RELEASE_ROLE_NAME=CodebuildRelease +export RELEASE_PROFILE_NAME=AmplifyCLIRelease +export RC_PROJECT_NAME=RC +export TAGGED_RC_PROJECT_NAME=TaggedReleaseWithoutE2E +export RELEASE_PROJECT_NAME=Release + +############################## RC ############################## +function RCLocal { + echo Running Local RC + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + rc_sha=$(git rev-parse --short "$0") + branch_name="release_rc/$rc_sha" + git checkout -B "$branch_name" "$rc_sha" + git push "origin" "$branch_name" + triggerProjectBatch $RELEASE_ACCOUNT_LOCAL $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Local" $RC_PROJECT_NAME $branch_name +} +function RCBeta { + echo Running Beta RC + echo You must be on the Beta repository to perform this action, or build will fail. + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + rc_sha=$(git rev-parse --short "$0") + branch_name="release_rc/$rc_sha" + git checkout -B "$branch_name" "$rc_sha" + git push "origin" "$branch_name" + triggerProjectBatch $RELEASE_ACCOUNT_BETA $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Beta" $RC_PROJECT_NAME $branch_name +} +function RCProd { + echo Running Prod RC + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + source ./scripts/release-rc-codebuild.sh $0 + branch_name=$(git branch --show-current) + triggerProjectBatch $RELEASE_ACCOUNT_PROD $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Prod" $RC_PROJECT_NAME $branch_name +} +############################## Tagged RC ############################## +# Follow the steps here https://quip-amazon.com/RX9eASbegQzo/Tagged-release-steps +# and create an upstream branch (not in your fork, but in parent) +# with the name tagged-release-without-e2e-tests/ +function TaggedRCLocal { + echo Running Local Tagged RC + branch_name=$(git branch --show-current) + triggerProjectBatch $RELEASE_ACCOUNT_LOCAL $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Local" $TAGGED_RC_PROJECT_NAME $branch_name +} +function TaggedRCBeta { + echo Running Beta Tagged RC + branch_name=$(git branch --show-current) + triggerProjectBatch $RELEASE_ACCOUNT_BETA $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Beta" $TAGGED_RC_PROJECT_NAME $branch_name +} +function TaggedRCProd { + echo Running Prod Tagged RC + branch_name=$(git branch --show-current) + triggerProjectBatch $RELEASE_ACCOUNT_PROD $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Prod" $TAGGED_RC_PROJECT_NAME $branch_name +} +############################## RELEASE ############################## +function ReleaseLocal { + echo Running Local Release + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + rc_sha=$(git rev-parse --short "$0") + rc_branch="release_rc/$rc_sha" + git checkout "$rc_branch" + git push "origin" "$rc_branch"~1:refs/heads/release + branch_name=release + triggerProjectBatch $RELEASE_ACCOUNT_LOCAL $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Local" $RELEASE_PROJECT_NAME $branch_name +} +function ReleaseBeta { + echo Running Beta Release + echo You must be on the Beta repository to perform this action, or build will fail. + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + rc_sha=$(git rev-parse --short "$0") + rc_branch="release_rc/$rc_sha" + git checkout "$rc_branch" + git push "origin" "$rc_branch"~1:refs/heads/release + branch_name=release + triggerProjectBatch $RELEASE_ACCOUNT_BETA $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Beta" $RELEASE_PROJECT_NAME $branch_name +} +function ReleaseProd { + echo Running Prod Release + if [[ $0 == 'bash' || -z $0 ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 + fi + source ./scripts/promote-rc-codebuild.sh $0 + branch_name=release + triggerProjectBatch $RELEASE_ACCOUNT_PROD $RELEASE_ROLE_NAME "${RELEASE_PROFILE_NAME}Prod" $RELEASE_PROJECT_NAME $branch_name +} diff --git a/scripts/promote-rc-codebuild.sh b/scripts/promote-rc-codebuild.sh new file mode 100755 index 00000000000..1093f29e8be --- /dev/null +++ b/scripts/promote-rc-codebuild.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +repo_name="aws-amplify/amplify-cli" + +git remote update + +if [[ -z ${1+x} ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 +fi + +rc_sha=$(git rev-parse --short "$1") +remote_name=$(git remote -v | grep "$repo_name" | head -n1 | awk '{print $1;}') + +if [[ -z ${remote_name+x} ]]; then + echo "Could not determine remote name of" "$repo_name" "repository" + exit 1 +fi + +rc_branch="release_rc/$rc_sha" + +git fetch "$remote_name" "$rc_branch" +git checkout "$rc_branch" +git reset --hard "$remote_name"/"$rc_branch" +git push "$remote_name" "$rc_branch"~1:refs/heads/release diff --git a/scripts/release-rc-codebuild.sh b/scripts/release-rc-codebuild.sh new file mode 100755 index 00000000000..59fe9d9cfdd --- /dev/null +++ b/scripts/release-rc-codebuild.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +repo_name="aws-amplify/amplify-cli" + +git remote update + + +if [[ -z ${1+x} ]]; then + echo "Include the release candidate commit ref you wish to release as the first argument" + exit 1 +fi + +rc_sha=$(git rev-parse --short "$1") +remote_name=$(git remote -v | grep "$repo_name" | head -n1 | awk '{print $1;}') + +if [[ -z ${remote_name+x} ]]; then + echo "Could not determine remote name of" "$repo_name" "repository" + exit 1 +fi + +branch_name="release_rc/$rc_sha" + +git checkout -B "$branch_name" "$rc_sha" +git fetch "$remote_name" main +set +e +git merge "$remote_name"/main +merge_exit_code=$? +set -e +if [[ $merge_exit_code -gt 0 ]]; then + # could not automatically merge + echo "Resolve merge conflicts and resume release candidate publish by running 'kill -CONT $$'" + kill -TSTP $$ +fi +git push "$remote_name" "$branch_name" diff --git a/scripts/split-e2e-tests-v2.ts b/scripts/split-e2e-tests-v2.ts index 2063b78e7e0..21d749f5fd2 100644 --- a/scripts/split-e2e-tests-v2.ts +++ b/scripts/split-e2e-tests-v2.ts @@ -173,7 +173,7 @@ type CandidateJob = { tests: string[]; useParentAccount: boolean; // intentially leaving this here - accounts are randomly assigned to jobs - // by a via local_publish_helpers.sh script + // by a via local_publish_helpers_codebuild.sh script // account: string, }; diff --git a/shared-scripts.sh b/shared-scripts.sh index 83e8d0a7050..9df4ca441af 100644 --- a/shared-scripts.sh +++ b/shared-scripts.sh @@ -90,6 +90,7 @@ function _buildLinux { yarn --immutable yarn production-build yarn build-tests + ./.circleci/cb-publish-step-1-set-versions.sh storeCache $CODEBUILD_SRC_DIR repo storeCache $HOME/.cache .cache } @@ -142,7 +143,7 @@ function _verifyVersionsMatch { loadCache .cache $HOME/.cache loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath checkPackageVersionsInLocalNpmRegistry @@ -154,7 +155,7 @@ function _mockE2ETests { # make repo directory accessible to codebuild-user chown -R codebuild-user . - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh cd packages/amplify-util-mock/ # run mock e2e tests as codebuild-user, root can't run open search sudo -u codebuild-user bash -c 'export NODE_OPTIONS=--max-old-space-size=4096 && yarn e2e' @@ -165,17 +166,16 @@ function _publishToLocalRegistry { loadCache repo $CODEBUILD_SRC_DIR loadCache .cache $HOME/.cache - source ./.circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source ./.circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal - export LOCAL_PUBLISH_TO_LATEST=true - ./.circleci/publish-codebuild.sh + ./.circleci/cb-publish-step-2-verdaccio.sh unsetNpmRegistryUrl echo Generate Change Log # Leaving this breadcrumb here "git reset --soft HEAD~1" - # we commented this out because the publish script is now checking out the current branch, and this started to fail as a result + # we commented this out previously because the publish script is now checking out the current branch, and this started to fail as a result # if we run into problems in the future, we should revisit this - # git reset --soft HEAD~1 + git reset --soft HEAD~1 yarn ts-node scripts/unified-changelog.ts cat UNIFIED_CHANGELOG.md @@ -206,8 +206,8 @@ function _uploadPkgBinaries { echo Done loading binaries ls $CODEBUILD_SRC_DIR/out - source .circleci/local_publish_helpers.sh - uploadPkgCliForE2E + source .circleci/local_publish_helpers_codebuild.sh + uploadPkgCliCodeBuild storeCache $CODEBUILD_SRC_DIR/out all-binaries } @@ -222,7 +222,7 @@ function _buildBinaries { loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version loadCacheFile UNIFIED_CHANGELOG.md $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal generatePkgCli $binaryType @@ -242,7 +242,7 @@ function _install_packaged_cli_linux { function _convertCoverage { echo Convert Coverage - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath @@ -288,7 +288,7 @@ function _runE2ETestsLinux { # verify installation which amplify amplify version - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath amplify version @@ -305,7 +305,7 @@ function _unassumeTestAccountCredentials { function _runMigrationMultiEnvLayersTest { echo RUN E2E Tests Linux _loadE2ECache - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh changeNpmGlobalPath cd packages/amplify-migration-tests _loadTestAccountCredentials @@ -314,7 +314,7 @@ function _runMigrationMultiEnvLayersTest { function _runMigrationNonMultiEnvLayersTest { echo RUN E2E Tests Linux _loadE2ECache - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh changeNpmGlobalPath cd packages/amplify-migration-tests _loadTestAccountCredentials @@ -323,7 +323,7 @@ function _runMigrationNonMultiEnvLayersTest { function _runMigrationV8Test { echo RUN E2E Tests Linux _loadE2ECache - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh changeNpmGlobalPath cd packages/amplify-migration-tests unset IS_AMPLIFY_CI @@ -334,7 +334,7 @@ function _runMigrationV8Test { function _runMigrationV10Test { echo RUN E2E Tests Linux _loadE2ECache - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh changeNpmGlobalPath cd packages/amplify-migration-tests unset IS_AMPLIFY_CI @@ -345,7 +345,7 @@ function _runMigrationV10Test { function _runMigrationV12Test { echo RUN E2E Tests Linux _loadE2ECache - source .circleci/local_publish_helpers.sh + source .circleci/local_publish_helpers_codebuild.sh changeNpmGlobalPath cd packages/amplify-migration-tests unset IS_AMPLIFY_CI @@ -429,7 +429,7 @@ function _amplifySudoInstallTestSetup { loadCache repo $CODEBUILD_SRC_DIR loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache loadCache all-binaries $CODEBUILD_SRC_DIR/out - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setSudoNpmRegistryUrlToLocal changeSudoNpmGlobalPath # sudo npm install -g @aws-amplify/cli @@ -440,7 +440,7 @@ function _amplifyInstallTestSetup { loadCache repo $CODEBUILD_SRC_DIR loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache loadCache all-binaries $CODEBUILD_SRC_DIR/out - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath # limit memory for new processes to 1GB @@ -454,7 +454,7 @@ function _amplifyInstallTestSetup { function _amplifyConsoleIntegrationTests { loadCache repo $CODEBUILD_SRC_DIR loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath npm install -g @aws-amplify/cli @@ -616,12 +616,72 @@ function _waitForJobs { ts-node ./wait-for-all-codebuild.ts $CODEBUILD_RESOLVED_SOURCE_VERSION $file_path $PROJECT_NAME $account_for_failures cd .. } +function _verifyPkgCLI { + loadCache repo $CODEBUILD_SRC_DIR + loadCache repo-out-arm $CODEBUILD_SRC_DIR/out + loadCache repo-out-linux $CODEBUILD_SRC_DIR/out + loadCache repo-out-macos $CODEBUILD_SRC_DIR/out + loadCache repo-out-win $CODEBUILD_SRC_DIR/out + source .circleci/local_publish_helpers_codebuild.sh && verifyPkgCli +} +function _githubPrerelease { + loadCache repo $CODEBUILD_SRC_DIR + loadCache all-binaries $CODEBUILD_SRC_DIR/out + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + loadCacheFile UNIFIED_CHANGELOG.md $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md + cd out + mv amplify-pkg-macos-x64 amplify-pkg-macos + mv amplify-pkg-linux-x64 amplify-pkg-linux + mv amplify-pkg-win-x64.exe amplify-pkg-win.exe + tar zcvf amplify-pkg-macos.tgz amplify-pkg-macos + tar zcvf amplify-pkg-linux.tgz amplify-pkg-linux + tar zcvf amplify-pkg-win.exe.tgz amplify-pkg-win.exe + cd $CODEBUILD_SRC_DIR + echo Publish Amplify CLI GitHub prerelease + commit=$(git rev-parse HEAD~1) + version=$(cat .amplify-pkg-version) + yarn ts-node scripts/github-prerelease.ts $version $commit +} +function _githubPrereleaseInstallSanityCheck { + loadCache repo $CODEBUILD_SRC_DIR + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + echo Install packaged Amplify CLI + version=$(cat .amplify-pkg-version) + curl -sL https://aws-amplify.github.io/amplify-cli/install | version=v$version bash + echo "export PATH=$PATH:$HOME/.amplify/bin" >> $BASH_ENV + echo Sanity check install + amplify version +} +function _publishToNpm { + loadCache repo $CODEBUILD_SRC_DIR + loadCache all-binaries $CODEBUILD_SRC_DIR/out + + ./out/amplify-pkg-linux-x64 --version + echo Authenticate with npm + echo "//registry.npmjs.org/:_authToken=$NPM_PUBLISH_TOKEN" > ~/.npmrc + source ./.circleci/cb-publish-step-3-npm.sh +} +function _postPublishPushToGit { + loadCache repo $CODEBUILD_SRC_DIR + loadCache all-binaries $CODEBUILD_SRC_DIR/out + echo Push release commit and tags + source ./.circleci/cb-publish-step-4-push-to-git.sh +} +function _githubRelease { + loadCache repo $CODEBUILD_SRC_DIR + loadCache all-binaries $CODEBUILD_SRC_DIR/out + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + echo Publish Amplify CLI GitHub release + commit=$(git rev-parse HEAD~1) + version=$(cat .amplify-pkg-version) + yarn ts-node scripts/github-release.ts $version $commit +} function _amplifyGeneralConfigTests { _loadE2ECache _install_packaged_cli_linux amplify version - source .circleci/local_publish_helpers.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" + source .circleci/local_publish_helpers_codebuild.sh && startLocalRegistry "$CODEBUILD_SRC_DIR/.circleci/verdaccio.yaml" setNpmRegistryUrlToLocal changeNpmGlobalPath amplify version @@ -629,3 +689,17 @@ function _amplifyGeneralConfigTests { _loadTestAccountCredentials retry yarn general-config-e2e --no-cache --maxWorkers=3 --forceExit $TEST_SUITE } +function _deploymentVerificationPostRelease { + loadCache repo $CODEBUILD_SRC_DIR + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + echo Verify Release Deployment + version=$(cat .amplify-pkg-version) + yarn ts-node scripts/verify-deployment.ts -v $version +} +function _deploymentVerificationRCOrTagged { + loadCache repo $CODEBUILD_SRC_DIR + loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version + echo Verify Tagged or RC Deployment + version=$(cat .amplify-pkg-version) + yarn ts-node scripts/verify-deployment.ts --version $version --exclude-github +}