ci: Even better cache keys and granular caching (#3534)

Just starting up services for Snuba or Sentry migrations takes up to a minute sometimes and we do this even when there are no migrations, just because one of the Sentry or Snuba migrations change. This patch splits the caches up so only the necessary one runs, saving further time. It also uses the `LATEST_TAG` as the cache key for upgrade tests as the image versions or data will never change for a certain tag once it is release.
getsentry · Jan 16, 2025 · 3913a9f · 3913a9f
1 parent 63b6c0a
commit 3913a9f
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 27 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -56,22 +56,13 @@ jobs:
           sudo curl -L https://github.com/docker/compose/releases/download/v2.26.0/docker-compose-`uname -s`-`uname -m` -o "/usr/local/lib/docker/cli-plugins/docker-compose"
           sudo chmod +x "/usr/local/lib/docker/cli-plugins/docker-compose"
 
-      - name: Compute Docker Volume Cache Key
-        id: cache_key
-        run: |
-          source .env
-          SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1)
-          echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
-          SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1)
-          echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
-
       - name: Restore DB Volumes Cache
         id: restore_cache
         uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
         with:
-          key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
+          key: db-volumes-v6-${{ env.LATEST_TAG }}
           restore-keys: |
-            db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
+            db-volumes-v6-${{ env.LATEST_TAG }}
             db-volumes-v6-
           volumes: |
             sentry-postgres
@@ -80,7 +71,10 @@ jobs:
 
       - name: Install ${{ env.LATEST_TAG }}
         env:
+          # Remove SKIP_DB_MIGRATIONS after releasing 25.1.1 or 25.2.0
           SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
+          SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
+          SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
         run: |
           # This is to compensate for a bug in upgrade-clickhouse where
           # if we have sentry-clickhouse volume without the rest, it fails

diff --git a/action.yaml b/action.yaml
@@ -54,34 +54,58 @@ runs:
         sudo curl -L https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-`uname -s`-`uname -m` -o "${{ env.COMPOSE_PATH }}/docker-compose"
         sudo chmod +x "${{ env.COMPOSE_PATH }}/docker-compose"
 
-    - name: Compute Docker Volume Cache Key
+    - name: Compute Docker Volume Cache Keys
       id: cache_key
       shell: bash
       run: |
         source ${{ github.action_path }}/.env
         # See https://explainshell.com/explain?cmd=ls%20-Rv1rpq
         # for that long `ls` command
-        SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1)
+        SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c '{ ls -Rv1rpq src/sentry/migrations/; sed -n "/KAFKA_TOPIC_TO_CLUSTER/,/}/p" src/sentry/conf/server.py; }' | md5sum | cut -d ' ' -f 1)
         echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
-        SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1)
+        SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c '{ ls -Rv1rpq snuba/snuba_migrations/**/*.py; sed -n "/^class Topic(Enum):/,/\\n\\n/p" snuba/utils/streams/topics.py; }' | md5sum | cut -d ' ' -f 1)
         echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT
 
-    - name: Restore DB Volumes Cache
-      id: restore_cache
+    - name: Restore Sentry Volume Cache
+      id: restore_cache_sentry
       uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
       with:
-        key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
+        key: db-volumes-sentry-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}
         restore-keys: |
-          db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
-          db-volumes-v6-
+          db-volumes-sentry-v1-
         volumes: |
           sentry-postgres
+
+    - name: Restore Snuba Volume Cache
+      id: restore_cache_snuba
+      uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
+      with:
+        key: db-volumes-snuba-v1-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
+        restore-keys: |
+          db-volumes-snuba-v1-
+        volumes: |
           sentry-clickhouse
+
+    - name: Restore Kafka Volume Cache
+      id: restore_cache_kafka
+      uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd
+      with:
+        key: db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
+        restore-keys: |
+          db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}
+          db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-
+          db-volumes-kafka-v1-
+        volumes: |
           sentry-kafka
 
     - name: Install self-hosted
       env:
-        SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }}
+        # Note that cache keys for Sentry and Snuba have their respective Kafka configs built into them
+        # and the Kafka volume cache is comprises both keys. This way we can omit the Kafka cache hit
+        # in here to still avoid running Sentry or Snuba migrations if only one of their Kafka config has
+        # changed. Heats up your head a bit but if you think about it, it makes sense.
+        SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache_sentry.outputs.cache-hit == 'true' && '1' || '' }}
+        SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache_snuba.outputs.cache-hit == 'true' && '1' || '' }}
       shell: bash
       run: |
         cd ${{ github.action_path }}
@@ -97,14 +121,28 @@ runs:
 
         ./install.sh --no-report-self-hosted-issues --skip-commit-check
 
-    - name: Save DB Volumes Cache
-      if: steps.restore_cache.outputs.cache-hit != 'true'
+    - name: Save Sentry Volume Cache
+      if: steps.restore_cache_sentry.outputs.cache-hit != 'true'
       uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
       with:
-        key: ${{ steps.restore_cache.outputs.cache-primary-key }}
+        key: ${{ steps.restore_cache_sentry.outputs.cache-primary-key }}
         volumes: |
           sentry-postgres
+
+    - name: Save Snuba Volume Cache
+      if: steps.restore_cache_snuba.outputs.cache-hit != 'true'
+      uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
+      with:
+        key: ${{ steps.restore_cache_snuba.outputs.cache-primary-key }}
+        volumes: |
           sentry-clickhouse
+
+    - name: Save Kafka Volume Cache
+      if: steps.restore_cache_kafka.outputs.cache-hit != 'true'
+      uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd
+      with:
+        key: ${{ steps.restore_cache_kafka.outputs.cache-primary-key }}
+        volumes: |
           sentry-kafka
 
     - name: Integration Test

diff --git a/install/bootstrap-snuba.sh b/install/bootstrap-snuba.sh
@@ -1,9 +1,9 @@
 echo "${_group}Bootstrapping and migrating Snuba ..."
 
-if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then
+if [[ -z "${SKIP_SNUBA_MIGRATIONS:-}" ]]; then
   $dcr snuba-api bootstrap --force
 else
-  echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS"
+  echo "Skipped DB migrations due to SKIP_SNUBA_MIGRATIONS=$SKIP_SNUBA_MIGRATIONS"
 fi
 
 echo "${_endgroup}"
diff --git a/install/set-up-and-migrate-database.sh b/install/set-up-and-migrate-database.sh
@@ -1,6 +1,6 @@
 echo "${_group}Setting up / migrating database ..."
 
-if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then
+if [[ -z "${SKIP_SENTRY_MIGRATIONS:-}" ]]; then
   # Fixes https://github.com/getsentry/self-hosted/issues/2758, where a migration fails due to indexing issue
   $dc up --wait postgres
 
@@ -31,6 +31,6 @@ with connection.cursor() as cursor:
     $dcr web upgrade --create-kafka-topics
   fi
 else
-  echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS"
+  echo "Skipped DB migrations due to SKIP_SENTRY_MIGRATIONS=$SKIP_SENTRY_MIGRATIONS"
 fi
 echo "${_endgroup}"