worker-ci.yaml

name: Worker CI

on:
  push:
    branches:
      - main
      - dev

concurrency:
  group: ${{ github.ref }}-worker
  cancel-in-progress: true

env:
  IMAGE: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
  REGISTRY: ${{ vars.REGISTRY }}

jobs:
  build-worker:
    runs-on: ubuntu-latest
    if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~deploy') || contains(github.event.head_commit.message, '~sitemaps') || contains(github.event.head_commit.message, '~topics') || contains(github.event.head_commit.message, '~caches') || contains(github.event.head_commit.message, '~notifs') }}
    environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
    steps:
      - name: Print Job Description
        run: |
          echo building \"$IMAGE\" from branch \"${{ github.ref_name }}\"
      - name: Git Checkout
        uses: actions/checkout@v3
        with:
          ssh-key: ${{ secrets.GIT_SSH_KEY }}
          submodules: recursive
      - name: Build
        run: |
          docker build . --file Dockerfile.worker --tag $IMAGE:latest
      - name: Install doctl
        uses: digitalocean/action-doctl@v2
        with:
          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
      - name: Log in to doctl
        run: doctl registry login --expiry-seconds 600
      - name: Tag Docker Image
        run: docker tag $IMAGE:latest $REGISTRY/$IMAGE:latest
      - name: Push Docker Image
        run: docker push $REGISTRY/$IMAGE:latest

  deploy-worker-sitemaps:
    needs: build-worker
    runs-on: ubuntu-latest
    if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~sitemaps') }}
    environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
    env:
      BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
      CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
      CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-sitemaps
      DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
      ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
      ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
      FIREBASE_CREDENTIALS: ${{ secrets.FIREBASE_CREDENTIALS }}
      K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
      K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
      K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
      K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
      K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
      K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_SITEMAPS || 1 }}
      MAIL_HOST: ${{ vars.MAIL_HOST }}
      MAIL_PASS: ${{ secrets.MAIL_PASS }}
      MAIL_PORT: ${{ vars.MAIL_PORT }}
      MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
      MAIL_USER: ${{ secrets.MAIL_USER }}
      MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
      OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
      PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
      PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
      PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
      PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
      PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
      PORT: ${{ vars.PORT }}
      S3_BUCKET: ${{ vars.S3_BUCKET }}
      S3_KEY: ${{ secrets.S3_KEY }}
      S3_SECRET: ${{ secrets.S3_SECRET }}
      WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
      WORKER_QUEUE: sitemaps
    steps:
      - name: Print Job Description
        run: |
          echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
      - name: Git Checkout
        uses: actions/checkout@v3
        with:
          ssh-key: ${{ secrets.GIT_SSH_KEY }}
          submodules: recursive
      - name: Install doctl
        uses: digitalocean/action-doctl@v2
        with:
          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
      - name: Add kubectl cert
        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
      - name: Apply deployment changes
        run: |
          envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
      - name: Deploy Rollout
        run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
      - name: Verify deployment
        run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
        
  deploy-worker-topics:
    needs: build-worker
    runs-on: ubuntu-latest
    if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~topics') }}
    environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
    env:
      BACKOFF_INTERVAL: ${{ vars.BACKOFF_INTERVAL }}
      BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
      CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
      CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-topics
      DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
      DUPLICATE_LOOKBACK_INTERVAL: ${{ vars.DUPLICATE_LOOKBACK_INTERVAL }}
      ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
      ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
      K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
      K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
      K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
      K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
      K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
      K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_TOPICS || 1 }}
      MAIL_HOST: ${{ vars.MAIL_HOST }}
      MAIL_PASS: ${{ secrets.MAIL_PASS }}
      MAIL_PORT: ${{ vars.MAIL_PORT }}
      MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
      MAIL_USER: ${{ secrets.MAIL_USER }}
      MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
      OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
      PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
      PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
      PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
      PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
      PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
      PORT: ${{ vars.PORT }}
      RELATIONSHIP_THRESHOLD: ${{ vars.RELATIONSHIP_THRESHOLD }}
      S3_BUCKET: ${{ vars.S3_BUCKET }}
      S3_KEY: ${{ secrets.S3_KEY }}
      S3_SECRET: ${{ secrets.S3_SECRET }}
      WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
      WORKER_QUEUE: topics
    steps:
      - name: Print Job Description
        run: |
          echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
      - name: Git Checkout
        uses: actions/checkout@v3
        with:
          ssh-key: ${{ secrets.GIT_SSH_KEY }}
          submodules: recursive
      - name: Install doctl
        uses: digitalocean/action-doctl@v2
        with:
          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
      - name: Add kubectl cert
        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
      - name: Apply deployment changes
        run: |
          envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
      - name: Deploy Rollout
        run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
      - name: Verify deployment
        run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
        
  deploy-worker-caches:
    needs: build-worker
    runs-on: ubuntu-latest
    if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~caches') }}
    environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
    env:
      BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
      CACHE_HALFLIFE: ${{ vars.CACHE_HALFLIFE }}
      CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
      CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-caches
      DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
      ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
      ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
      K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
      K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
      K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
      K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
      K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
      K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_CACHES || 1 }}
      MAIL_HOST: ${{ vars.MAIL_HOST }}
      MAIL_PASS: ${{ secrets.MAIL_PASS }}
      MAIL_PORT: ${{ vars.MAIL_PORT }}
      MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
      MAIL_USER: ${{ secrets.MAIL_USER }}
      MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
      OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
      PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
      PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
      PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
      PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
      PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
      PORT: ${{ vars.PORT }}
      S3_BUCKET: ${{ vars.S3_BUCKET }}
      S3_KEY: ${{ secrets.S3_KEY }}
      S3_SECRET: ${{ secrets.S3_SECRET }}
      WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
      WORKER_QUEUE: caches
    steps:
      - name: Print Job Description
        run: |
          echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
      - name: Git Checkout
        uses: actions/checkout@v3
        with:
          ssh-key: ${{ secrets.GIT_SSH_KEY }}
          submodules: recursive
      - name: Install doctl
        uses: digitalocean/action-doctl@v2
        with:
          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
      - name: Add kubectl cert
        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
      - name: Apply deployment changes
        run: |
          envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
      - name: Deploy Rollout
        run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
      - name: Verify deployment
        run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE

  deploy-worker-notifs:
    needs: build-worker
    runs-on: ubuntu-latest
    if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~notifs') }}
    environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
    env:
      BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
      CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
      CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-notifs
      DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
      ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
      ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
      FIREBASE_CREDENTIALS: ${{ secrets.FIREBASE_CREDENTIALS }}
      K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
      K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
      K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
      K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
      K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
      K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_NOTIFS || 1 }}
      MAIL_HOST: ${{ vars.MAIL_HOST }}
      MAIL_PASS: ${{ secrets.MAIL_PASS }}
      MAIL_PORT: ${{ vars.MAIL_PORT }}
      MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
      MAIL_USER: ${{ secrets.MAIL_USER }}
      MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
      OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
      PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
      PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
      PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
      PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
      PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
      PORT: ${{ vars.PORT }}
      S3_BUCKET: ${{ vars.S3_BUCKET }}
      S3_KEY: ${{ secrets.S3_KEY }}
      S3_SECRET: ${{ secrets.S3_SECRET }}
      WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
      WORKER_QUEUE: notifs
    steps:
      - name: Print Job Description
        run: |
          echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
      - name: Git Checkout
        uses: actions/checkout@v3
        with:
          ssh-key: ${{ secrets.GIT_SSH_KEY }}
          submodules: recursive
      - name: Install doctl
        uses: digitalocean/action-doctl@v2
        with:
          token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
      - name: Add kubectl cert
        run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
      - name: Apply deployment changes
        run: |
          envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
      - name: Deploy Rollout
        run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
      - name: Verify deployment
        run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE