ci-hog.yml

# This workflow runs all of our backend django tests.
#
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
# .github/workflows/ci-backend-update-test-timing.yml action
name: Hog CI

on:
    push:
        branches:
            - master
        paths-ignore:
            - rust/**
            - livestream/**
    pull_request:
        paths-ignore:
            - rust/**
            - livestream/**

jobs:
    # Job to decide if we should run backend ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-24.04
        timeout-minutes: 5
        name: Determine need to run Hog checks
        # Set job outputs to values from filter step
        outputs:
            hog: ${{ steps.filter.outputs.hog }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v3

            - uses: dorny/paths-filter@v2
              id: filter
              with:
                  filters: |
                      hog:
                        # Avoid running tests for irrelevant changes
                        - 'common/hogvm/**/*'
                        - 'posthog/hogql/**/*'
                        - 'bin/hog'
                        - 'bin/hoge'
                        - requirements.txt
                        - requirements-dev.txt
                        - .github/workflows/ci-hog.yml

    hog-tests:
        needs: changes
        timeout-minutes: 30
        name: Hog tests
        runs-on: ubuntu-24.04
        if: needs.changes.outputs.hog == 'true'

        steps:
            # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined
            # there are backend changes, cancel previous runs
            - uses: n1hility/cancel-previous-runs@v3
              if: github.actor != 'posthog-bot'
              with:
                  token: ${{ secrets.GITHUB_TOKEN }}

            - uses: actions/checkout@v3
              with:
                  fetch-depth: 1

            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: 3.11.9
                  cache: 'pip'
                  cache-dependency-path: '**/requirements*.txt'
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            # uv is a fast pip alternative: https://github.com/astral-sh/uv/
            - run: pip install uv

            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl

            - name: Install Python dependencies
              run: |
                  uv pip install --system -r requirements.txt -r requirements-dev.txt

            - name: Install pnpm
              uses: pnpm/action-setup@v4

            - name: Set up Node.js
              uses: actions/setup-node@v4
              with:
                  node-version: 18

            - name: Check if ANTLR definitions are up to date
              run: |
                  cd ..
                  sudo apt-get install default-jre
                  mkdir antlr
                  cd antlr
                  curl -o antlr.jar https://www.antlr.org/download/antlr-$ANTLR_VERSION-complete.jar
                  export PWD=`pwd`
                  echo '#!/bin/bash' > antlr
                  echo "java -jar $PWD/antlr.jar \$*" >> antlr
                  chmod +x antlr
                  export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH"
                  export PATH="$PWD:$PATH"

                  cd ../posthog
                  antlr | grep "Version"
                  npm run grammar:build && git diff --exit-code
              env:
                  # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2),
                  # as apt-get is quite out of date. The same version must be set in common/hogql_parser/pyproject.toml
                  ANTLR_VERSION: '4.13.2'

            - name: Check if STL bytecode is up to date
              run: |
                  python -m common.hogvm.stl.compile
                  git diff --exit-code

            - name: Run HogVM Python tests
              run: |
                  pytest common/hogvm

            - name: Run HogVM TypeScript tests
              run: |
                  pnpm --filter=@posthog/hogvm install --frozen-lockfile
                  pnpm --filter=@posthog/hogvm test

            - name: Run Hog tests
              run: |
                  pnpm --filter=@posthog/hogvm install --frozen-lockfile
                  pnpm --filter=@posthog/hogvm build
                  cd common/hogvm
                  ./test.sh && git diff --exit-code

    check-package-version:
        name: Check HogVM TypeScript package version and detect an update
        needs: hog-tests
        if: needs.hog-tests.result == 'success' && needs.changes.outputs.hog == 'true'
        runs-on: ubuntu-24.04
        outputs:
            committed-version: ${{ steps.check-package-version.outputs.committed-version }}
            published-version: ${{ steps.check-package-version.outputs.published-version }}
            is-new-version: ${{ steps.check-package-version.outputs.is-new-version }}
        steps:
            - name: Checkout the repository
              uses: actions/checkout@v2
            - name: Check package version and detect an update
              id: check-package-version
              uses: PostHog/check-package-version@v2
              with:
                  path: common/hogvm/typescript

    release-hogvm:
        name: Release new HogVM TypeScript version
        runs-on: ubuntu-24.04
        needs: check-package-version
        if: needs.changes.outputs.hog == 'true' && needs.check-package-version.outputs.is-new-version == 'true'
        env:
            COMMITTED_VERSION: ${{ needs.check-package-version.outputs.committed-version }}
            PUBLISHED_VERSION: ${{ needs.check-package-version.outputs.published-version }}
        steps:
            - name: Checkout the repository
              uses: actions/checkout@v4
              with:
                  fetch-depth: 1
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: 3.11.9
                  cache: 'pip'
                  cache-dependency-path: '**/requirements*.txt'
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}
            - run: pip install uv
            - name: Install SAML (python3-saml) dependencies
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
            - name: Install Python dependencies
              run: |
                  uv pip install --system -r requirements.txt -r requirements-dev.txt
            - name: Install pnpm
              uses: pnpm/action-setup@v4
            - name: Set up Node 18
              uses: actions/setup-node@v4
              with:
                  node-version: 18
                  registry-url: https://registry.npmjs.org
            - name: Install package.json dependencies
              run: pnpm --filter=@posthog/hogvm install --frozen-lockfile
            - name: Publish the package in the npm registry
              run: cd common/hogvm/typescript && npm publish --access public
              env:
                  NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
            - name: Sleep 60 seconds to allow npm to update the package
              run: sleep 60

    update-versions:
        name: Update versions in package.json
        runs-on: ubuntu-24.04
        needs: release-hogvm
        if: always() # This ensures the job runs regardless of the result of release-hogvm
        steps:
            - name: Checkout the repository
              uses: actions/checkout@v4
              with:
                  fetch-depth: 1
                  repository: ${{ github.event.pull_request.head.repo.full_name }}
                  ref: ${{ github.event.pull_request.head.ref }}
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - name: Install pnpm
              uses: pnpm/action-setup@v4
            - name: Set up Node 18
              uses: actions/setup-node@v4
              with:
                  node-version: 18
                  registry-url: https://registry.npmjs.org

            - name: Check for version mismatches
              id: check-mismatch
              run: |
                  # Extract committed version
                  HOGVM_VERSION=$(jq -r '.version' common/hogvm/typescript/package.json)

                  # Compare dependencies in package.json
                  MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^')
                  PLUGIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' plugin-server/package.json | tr -d '^')

                  echo "HOGVM_VERSION=$HOGVM_VERSION"
                  echo "MAIN_VERSION=$MAIN_VERSION"
                  echo "PLUGIN_VERSION=$PLUGIN_VERSION"

                  # Set output if mismatches exist
                  if [[ "$HOGVM_VERSION" != "$MAIN_VERSION" || "$HOGVM_VERSION" != "$PLUGIN_VERSION" ]]; then
                      echo "mismatch=true" >> "$GITHUB_ENV"
                  else
                      echo "mismatch=false" >> "$GITHUB_ENV"
                  fi

            - name: Update package.json versions
              if: env.mismatch == 'true'
              run: |
                  VERSION=$(jq ".version" common/hogvm/typescript/package.json -r)

                  retry_pnpm_install() {
                      local retries=0
                      local max_retries=20 # 10 minutes total
                      local delay=30

                      while [[ $retries -lt $max_retries ]]; do
                          echo "Attempting pnpm install (retry $((retries+1))/$max_retries)..."
                          pnpm install --no-frozen-lockfile && break
                          echo "Install failed. Retrying in $delay seconds..."
                          sleep $delay
                          retries=$((retries + 1))
                      done

                      if [[ $retries -eq $max_retries ]]; then
                          echo "pnpm install failed after $max_retries attempts."
                          exit 1
                      fi
                  }

                  # Update main package.json
                  mv package.json package.old.json
                  jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
                  rm package.old.json

                  # Update plugin-server/package.json
                  cd plugin-server
                  mv package.json package.old.json
                  jq --indent 4 '.dependencies."@posthog/hogvm" = "^'$VERSION'"' package.old.json > package.json
                  rm package.old.json

                  retry_pnpm_install

            - name: Commit updated package.json files
              if: env.mismatch == 'true'
              uses: EndBug/add-and-commit@v9
              with:
                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "common/hogvm/typescript/package.json"]'
                  message: 'Update @posthog/hogvm version in package.json'
                  default_author: github_actions
                  github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}