Finish the GitHub Actions publish workflow (#714)

Now, GitHub Actions should be able to publish to JFrog.
Also, fix the CI test Gradle configuration stuff (e.g. fail fast).

.github/scripts/build.sh

@@ -27,27 +27,29 @@ if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
   RELEASE_CANDIDATE=true
 fi
 
-# If the project version is being bumped in this PR, assert that the changelog contains an entry for it
-if (! $RELEASE_CANDIDATE) &&
-    (git diff ${TRAVIS_BRANCH}...HEAD -- gradle.properties | grep -F "+version=$VERSION" > /dev/null) &&
-    ! ( (cat CHANGELOG.md | grep -F "## [$VERSION] -" > /dev/null) &&
-        (cat CHANGELOG.md | grep -F "[$VERSION]: https" > /dev/null) ); then
-  echo "This change bumps the project version to $VERSION, but no changelog entry could be found for this version!"
-  echo 'Please update CHANGELOG.md using the changelog helper script.'
-  echo 'For more info, run: ./scripts/update-changelog --help'
-  exit 1
-fi
-
 # TODO: Is this needed on GitHub Actions? Travis aborts after 10 minutes of no output, not sure about GA
 # while sleep 9m; do echo "[Ping] Keeping Travis job alive ($((SECONDS / 60)) minutes)"; done &
 # WAITER_PID=$!
 
-# For PR builds, Skip module-specific tests if its module dependencies haven't been touched
+# For PR builds only...
 if [ ! -z "$GITHUB_HEAD_REF" ] && [ ! -z "$GITHUB_BASE_REF" ]; then
+  # Fetch the PR base ref so it can be used to compute diffs
+  git fetch origin ${GITHUB_BASE_REF}:${GITHUB_BASE_REF}
+  # If the project version is being bumped in this PR, assert that the changelog contains an entry for it
+  if (! $RELEASE_CANDIDATE) &&
+      (git diff ${GITHUB_BASE_REF}...HEAD -- gradle.properties | grep -F "+version=$VERSION" > /dev/null) &&
+      ! ( (cat CHANGELOG.md | grep -F "## [$VERSION] -" > /dev/null) &&
+          (cat CHANGELOG.md | grep -F "[$VERSION]: https" > /dev/null) ); then
+    echo "This change bumps the project version to $VERSION, but no changelog entry could be found for this version!"
+    echo 'Please update CHANGELOG.md using the changelog helper script.'
+    echo 'For more info, run: ./scripts/update-changelog --help'
+    exit 1
+  fi
+  # Skip module-specific tests if its module dependencies haven't been touched
   CONDITIONAL_TESTING_MODULES='d2 r2-int-test restli-int-test'
   echo "This is a PR build, so testing will be conditional for these subprojects: [${CONDITIONAL_TESTING_MODULES// /,}]"
   # If any Gradle file was touched, run all tests just to be safe
-  if (git diff ${TRAVIS_BRANCH}...HEAD --name-only | grep '\.gradle' > /dev/null); then
+  if (git diff ${GITHUB_BASE_REF}...HEAD --name-only | grep '\.gradle' > /dev/null); then
     echo "This PR touches a file matching *.gradle, so tests will be run for all subprojects."
   else
     # Have to prime the comma-separated list with a dummy value because list construction in bash is hard...
@@ -58,7 +60,7 @@ if [ ! -z "$GITHUB_HEAD_REF" ] && [ ! -z "$GITHUB_BASE_REF" ]; then
       MODULE_DEPENDENCIES="$(./scripts/get-module-dependencies $MODULE testRuntimeClasspath | tr '\n' ' ')"
       # Create regex to capture lines in the diff's paths, e.g. 'a b c' -> '^\(a\|b\|c\)/'
       PATH_MATCHING_REGEX="^\\($(echo $MODULE_DEPENDENCIES | sed -z 's/ \+/\\|/g;s/\\|$/\n/g')\\)/"
-      if [ ! -z "$PATH_MATCHING_REGEX" ] && ! (git diff ${TRAVIS_BRANCH}...HEAD --name-only | grep "$PATH_MATCHING_REGEX" > /dev/null); then
+      if [ ! -z "$PATH_MATCHING_REGEX" ] && ! (git diff ${GITHUB_BASE_REF}...HEAD --name-only | grep "$PATH_MATCHING_REGEX" > /dev/null); then
         echo "Computed as... [${MODULE_DEPENDENCIES// /,}]"
         echo "None of $MODULE's module dependencies have been touched, skipping tests for $MODULE."
         EXTRA_ARGS="${EXTRA_ARGS},$MODULE"
@@ -73,8 +75,8 @@ fi
 ./gradlew build $EXTRA_ARGS
 EXIT_CODE=$?
 
-# Kill the waiter job
 # TODO: Figure out if this can be removed as well for GitHub Actions
+# Kill the waiter job
 # kill $WAITER_PID
 
 if [ $EXIT_CODE != 0 ]; then

.github/scripts/publish.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# Ensure that this is being run in CI by GitHub Actions
+if [ "$CI" != "true" ] || [ "$GITHUB_ACTIONS" != "true" ]; then
+  echo "This script should only be run in CI by GitHub Actions."
+  exit 2
+fi
+
+# Ensure that the tag is named properly as a semver tag
+if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
+  echo "Tag $GITHUB_REF is NOT a valid semver tag (vX.Y.Z), please delete this tag."
+  exit 1
+fi
+
+# Ensure that the script is being run from the root project directory
+PROPERTIES_FILE='gradle.properties'
+if [ ! -f "$PROPERTIES_FILE" ]; then
+  echo "Could not find $PROPERTIES_FILE, are you sure this is being run from the root project directory?"
+  echo "PWD: ${PWD}"
+  exit 1
+fi
+
+# Determine the version being published
+VERSION=$(awk 'BEGIN { FS = "=" }; $1 == "version" { print $2 }' $PROPERTIES_FILE | awk '{ print $1 }')
+if [ -z "$VERSION" ]; then
+  echo "Could not read the version from $PROPERTIES_FILE, please fix it and try again."
+  exit 1
+fi
+
+# Determine if the version is a release candidate version
+RELEASE_CANDIDATE=false
+if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
+  RELEASE_CANDIDATE=true
+fi
+
+# Ensure the tag corresponds to the current version
+EXPECTED_TAG="v$VERSION"
+if [ "$GITHUB_REF" != "refs/tags/$EXPECTED_TAG" ]; then
+  echo "Attempting to publish Rest.li version $VERSION from tag $GITHUB_REF is illegal."
+  echo "Please delete this tag and publish instead from tag $EXPECTED_TAG"
+  exit 1
+fi
+
+# Ensure that the tag commit is an ancestor of master
+git fetch origin master:master
+git merge-base --is-ancestor $GITHUB_REF master
+if [ $? -ne 0 ]; then
+  echo "Tag $GITHUB_REF is NOT an ancestor of master!"
+  # Abort the deployment if it's not a release candidate tag
+  if $RELEASE_CANDIDATE; then
+    echo "Since this is a release candidate tag, the deployment will continue."
+  else
+    echo 'Please delete this tag and instead create a tag off a master commit.'
+    exit 1
+  fi
+fi
+
+# TODO: Is this needed on GitHub Actions? Travis aborts after 10 minutes of no output, not sure about GA
+# Output something every 9 minutes, otherwise Travis will abort after 10 minutes of no output
+# while sleep 9m; do echo "[Ping] Keeping Travis job alive ($((SECONDS / 60)) minutes)"; done &
+# WAITER_PID=$!
+
+# Publish to JFrog Artifactory
+echo "All checks passed, attempting to publish Rest.li $VERSION to JFrog Artifactory..."
+./gradlew artifactoryPublish
+EXIT_CODE=$?
+
+# TODO: Figure out if this can be removed as well for GitHub Actions
+# Kill the waiter job
+# kill $WAITER_PID
+
+if [ $EXIT_CODE = 0 ]; then
+  echo "Successfully published Rest.li $VERSION to JFrog Artifactory."
+else
+  # We used to roll back Bintray uploads on failure to publish, but it's not clear if this is needed for JFrog.
+  # TODO: If "partial uploads" can occur for JFrog, then here we would roll back the upload via the JFrog REST API.
+  # We did this before using: curl -X DELETE --user ${BINTRAY_USER}:${BINTRAY_KEY} --fail $DELETE_VERSION_URL
+
+  echo 'Failed to publish to JFrog Artifactory.'
+  echo "You can check https://linkedin.jfrog.io/ui/repos/tree/General/pegasus to ensure that $VERSION is not present."
+  echo 'Please retry the upload by restarting this GitHub Actions job.'
+
+  exit 1
+fi

.github/workflows/build.yml

@@ -15,6 +15,9 @@ jobs:
     name: Java ${{ matrix.java }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          # Need to fetch 2 commits for the PR (base commit and head merge commit) so we can compute the diff
+          fetch-depth: 2
       - uses: actions/setup-java@v2
         with:
           distribution: zulu

.github/workflows/publish.yml

@@ -22,5 +22,5 @@ jobs:
           distribution: zulu
           java-version: ${{ matrix.java }}
           # Do NOT use caching, since we want to ensure published artifacts are fresh
-      - run: ./gradlew -version  # TODO: Use actual publishing script
+      - run: ./.github/scripts/publish.sh
 

build.gradle

@@ -294,14 +294,15 @@ subprojects {
     }
   }
 
-  // Exclude tests which are known to be flaky in the Travis CI environment. Since all test tasks are TestNG tasks
-  // EXCEPT the integTest task, exclude the options from being evaluated by travis for the integTest task.
-  if (System.getenv('TRAVIS') == 'true' && System.getenv('USER') == 'travis') {
+  // Do some special test configuration for the CI environment
+  if (System.getenv('CI') == 'true' && System.getenv('GITHUB_ACTIONS') == 'true') {
     afterEvaluate {
       project.tasks.withType(Test).forEach {
+        // Exclude tests which are known to be flaky (only TestNG tests, not other tests e.g. Gradle int tests)
         if (it.options instanceof TestNGOptions) {
           it.options.excludeGroups 'ci-flaky'
         }
+        // Increase the Rest.li int test timeout
         it.systemProperties['test.httpRequestTimeout'] = '20000'
         // Make the build fail fast in CI (may have unintended consequences locally)
         it.failFast = true

scripts/release

@@ -127,4 +127,4 @@ if [ $? != 0 ]; then
   exit 1
 fi
 
-echo "Tag push complete. You can view the $TAG_NAME publish job here: https://travis-ci.com/github/linkedin/rest.li/builds"
+echo "Tag push complete. You can view the $TAG_NAME publish job here: https://github.com/linkedin/rest.li/actions/workflows/publish.yml"