diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e471c84ba..5c01abc6c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,22 +56,13 @@ jobs: sudo curl -L https://github.com/docker/compose/releases/download/v2.26.0/docker-compose-`uname -s`-`uname -m` -o "/usr/local/lib/docker/cli-plugins/docker-compose" sudo chmod +x "/usr/local/lib/docker/cli-plugins/docker-compose" - - name: Compute Docker Volume Cache Key - id: cache_key - run: | - source .env - SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1) - echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT - SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1) - echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT - - name: Restore DB Volumes Cache id: restore_cache uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: - key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} + key: db-volumes-v6-${{ env.LATEST_TAG }} restore-keys: | - db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + db-volumes-v6-${{ env.LATEST_TAG }} db-volumes-v6- volumes: | sentry-postgres @@ -80,7 +71,10 @@ jobs: - name: Install ${{ env.LATEST_TAG }} env: + # Remove SKIP_DB_MIGRATIONS after releasing 25.1.1 or 25.2.0 SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }} + SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }} + SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }} run: | # This is to compensate for a bug in upgrade-clickhouse where # if we have sentry-clickhouse volume without the rest, it fails diff --git a/action.yaml b/action.yaml index 9f87b93d4f..924e4c856d 100644 --- a/action.yaml +++ b/action.yaml @@ -54,34 +54,58 @@ runs: sudo curl -L https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-`uname -s`-`uname -m` -o "${{ env.COMPOSE_PATH }}/docker-compose" sudo chmod +x "${{ env.COMPOSE_PATH }}/docker-compose" - - name: Compute Docker Volume Cache Key + - name: Compute Docker Volume Cache Keys id: cache_key shell: bash run: | source ${{ github.action_path }}/.env # See https://explainshell.com/explain?cmd=ls%20-Rv1rpq # for that long `ls` command - SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1) + SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c '{ ls -Rv1rpq src/sentry/migrations/; sed -n "/KAFKA_TOPIC_TO_CLUSTER/,/}/p" src/sentry/conf/server.py; }' | md5sum | cut -d ' ' -f 1) echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT - SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c 'ls -Rv1rpq snuba/snuba_migrations/**/*.py' | md5sum | cut -d ' ' -f 1) + SNUBA_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SNUBA_IMAGE -c '{ ls -Rv1rpq snuba/snuba_migrations/**/*.py; sed -n "/^class Topic(Enum):/,/\\n\\n/p" snuba/utils/streams/topics.py; }' | md5sum | cut -d ' ' -f 1) echo "SNUBA_MIGRATIONS_MD5=$SNUBA_MIGRATIONS_MD5" >> $GITHUB_OUTPUT - - name: Restore DB Volumes Cache - id: restore_cache + - name: Restore Sentry Volume Cache + id: restore_cache_sentry uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: - key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} + key: db-volumes-sentry-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} restore-keys: | - db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} - db-volumes-v6- + db-volumes-sentry-v1- volumes: | sentry-postgres + + - name: Restore Snuba Volume Cache + id: restore_cache_snuba + uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd + with: + key: db-volumes-snuba-v1-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + restore-keys: | + db-volumes-snuba-v1- + volumes: | sentry-clickhouse + + - name: Restore Kafka Volume Cache + id: restore_cache_kafka + uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd + with: + key: db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + restore-keys: | + db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + db-volumes-kafka-v1-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }}- + db-volumes-kafka-v1- + volumes: | sentry-kafka - name: Install self-hosted env: - SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }} + # Note that cache keys for Sentry and Snuba have their respective Kafka configs built into them + # and the Kafka volume cache is comprises both keys. This way we can omit the Kafka cache hit + # in here to still avoid running Sentry or Snuba migrations if only one of their Kafka config has + # changed. Heats up your head a bit but if you think about it, it makes sense. + SKIP_SENTRY_MIGRATIONS: ${{ steps.restore_cache_sentry.outputs.cache-hit == 'true' && '1' || '' }} + SKIP_SNUBA_MIGRATIONS: ${{ steps.restore_cache_snuba.outputs.cache-hit == 'true' && '1' || '' }} shell: bash run: | cd ${{ github.action_path }} @@ -97,14 +121,28 @@ runs: ./install.sh --no-report-self-hosted-issues --skip-commit-check - - name: Save DB Volumes Cache - if: steps.restore_cache.outputs.cache-hit != 'true' + - name: Save Sentry Volume Cache + if: steps.restore_cache_sentry.outputs.cache-hit != 'true' uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: - key: ${{ steps.restore_cache.outputs.cache-primary-key }} + key: ${{ steps.restore_cache_sentry.outputs.cache-primary-key }} volumes: | sentry-postgres + + - name: Save Snuba Volume Cache + if: steps.restore_cache_snuba.outputs.cache-hit != 'true' + uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd + with: + key: ${{ steps.restore_cache_snuba.outputs.cache-primary-key }} + volumes: | sentry-clickhouse + + - name: Save Kafka Volume Cache + if: steps.restore_cache_kafka.outputs.cache-hit != 'true' + uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd + with: + key: ${{ steps.restore_cache_kafka.outputs.cache-primary-key }} + volumes: | sentry-kafka - name: Integration Test diff --git a/install/bootstrap-snuba.sh b/install/bootstrap-snuba.sh index 20666302ad..496becd6c5 100644 --- a/install/bootstrap-snuba.sh +++ b/install/bootstrap-snuba.sh @@ -1,9 +1,9 @@ echo "${_group}Bootstrapping and migrating Snuba ..." -if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then +if [[ -z "${SKIP_SNUBA_MIGRATIONS:-}" ]]; then $dcr snuba-api bootstrap --force else - echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS" + echo "Skipped DB migrations due to SKIP_SNUBA_MIGRATIONS=$SKIP_SNUBA_MIGRATIONS" fi echo "${_endgroup}" diff --git a/install/set-up-and-migrate-database.sh b/install/set-up-and-migrate-database.sh index a1cc8323e8..2d4e13208e 100644 --- a/install/set-up-and-migrate-database.sh +++ b/install/set-up-and-migrate-database.sh @@ -1,6 +1,6 @@ echo "${_group}Setting up / migrating database ..." -if [[ -z "${SKIP_DB_MIGRATIONS:-}" ]]; then +if [[ -z "${SKIP_SENTRY_MIGRATIONS:-}" ]]; then # Fixes https://github.com/getsentry/self-hosted/issues/2758, where a migration fails due to indexing issue $dc up --wait postgres @@ -31,6 +31,6 @@ with connection.cursor() as cursor: $dcr web upgrade --create-kafka-topics fi else - echo "Skipped DB migrations due to SKIP_DB_MIGRATIONS=$SKIP_DB_MIGRATIONS" + echo "Skipped DB migrations due to SKIP_SENTRY_MIGRATIONS=$SKIP_SENTRY_MIGRATIONS" fi echo "${_endgroup}"