Skip to content

fix cached

fix cached #1652

Workflow file for this run

name: Worker CI
on:
push:
branches:
- main
- dev
concurrency:
group: ${{ github.ref }}-worker
cancel-in-progress: true
env:
IMAGE: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
REGISTRY: ${{ vars.REGISTRY }}
jobs:
build-worker:
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~deploy') || contains(github.event.head_commit.message, '~sitemaps') || contains(github.event.head_commit.message, '~topics') || contains(github.event.head_commit.message, '~caches') ||
contains(github.event.head_commit.message, '~media') ||
contains(github.event.head_commit.message, '~notifs') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
steps:
- name: Print Job Description
run: |
echo building \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Build
run: |
docker build . --file Dockerfile.worker --tag $IMAGE:latest
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Log in to doctl
run: doctl registry login --expiry-seconds 600
- name: Tag Docker Image
run: docker tag $IMAGE:latest $REGISTRY/$IMAGE:latest
- name: Push Docker Image
run: docker push $REGISTRY/$IMAGE:latest
deploy-worker-sitemaps:
needs: build-worker
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~sitemaps') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-sitemaps
DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
FIREBASE_CREDENTIALS: ${{ secrets.FIREBASE_CREDENTIALS }}
K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_SITEMAPS || 1 }}
MAIL_HOST: ${{ vars.MAIL_HOST }}
MAIL_PASS: ${{ secrets.MAIL_PASS }}
MAIL_PORT: ${{ vars.MAIL_PORT }}
MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
MAIL_USER: ${{ secrets.MAIL_USER }}
MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
PORT: ${{ vars.PORT }}
S3_BUCKET: ${{ vars.S3_BUCKET }}
S3_KEY: ${{ secrets.S3_KEY }}
S3_SECRET: ${{ secrets.S3_SECRET }}
WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
WORKER_QUEUE: sitemaps
steps:
- name: Print Job Description
run: |
echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Add kubectl cert
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
- name: Apply deployment changes
run: |
envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
- name: Deploy Rollout
run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
- name: Verify deployment
run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
deploy-worker-topics:
needs: build-worker
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~topics') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
env:
ANTHROPIC_API_KEY: $${{ secrets.ANTHROPIC_API_KEY }}
BACKOFF_INTERVAL: ${{ vars.BACKOFF_INTERVAL }}
BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-topics
DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
DUPLICATE_LOOKBACK_INTERVAL: ${{ vars.DUPLICATE_LOOKBACK_INTERVAL }}
ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_TOPICS || 1 }}
MAIL_HOST: ${{ vars.MAIL_HOST }}
MAIL_PASS: ${{ secrets.MAIL_PASS }}
MAIL_PORT: ${{ vars.MAIL_PORT }}
MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
MAIL_USER: ${{ secrets.MAIL_USER }}
MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
PORT: ${{ vars.PORT }}
RELATIONSHIP_THRESHOLD: ${{ vars.RELATIONSHIP_THRESHOLD }}
S3_BUCKET: ${{ vars.S3_BUCKET }}
S3_KEY: ${{ secrets.S3_KEY }}
S3_SECRET: ${{ secrets.S3_SECRET }}
TOPIC_RECALCULATE_RATE: ${{ vars.TOPIC_RECALCULATE_RATE }}
WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
WORKER_QUEUE: topics
steps:
- name: Print Job Description
run: |
echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Add kubectl cert
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
- name: Apply deployment changes
run: |
envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
- name: Deploy Rollout
run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
- name: Verify deployment
run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
deploy-worker-caches:
needs: build-worker
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~caches') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
CACHE_LIFESPAN: ${{ vars.CACHE_LIFESPAN }}
CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-caches
DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_CACHES || 1 }}
MAIL_HOST: ${{ vars.MAIL_HOST }}
MAIL_PASS: ${{ secrets.MAIL_PASS }}
MAIL_PORT: ${{ vars.MAIL_PORT }}
MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
MAIL_USER: ${{ secrets.MAIL_USER }}
MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
PORT: ${{ vars.PORT }}
S3_BUCKET: ${{ vars.S3_BUCKET }}
S3_KEY: ${{ secrets.S3_KEY }}
S3_SECRET: ${{ secrets.S3_SECRET }}
WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
WORKER_QUEUE: caches
steps:
- name: Print Job Description
run: |
echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Add kubectl cert
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
- name: Apply deployment changes
run: |
envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
- name: Deploy Rollout
run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
- name: Verify deployment
run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
deploy-worker-media:
needs: build-worker
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~media') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
BACKOFF_INTERVAL: ${{ vars.BACKOFF_INTERVAL }}
BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-media
DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
DUPLICATE_LOOKBACK_INTERVAL: ${{ vars.DUPLICATE_LOOKBACK_INTERVAL }}
ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_TOPICS || 1 }}
MAIL_HOST: ${{ vars.MAIL_HOST }}
MAIL_PASS: ${{ secrets.MAIL_PASS }}
MAIL_PORT: ${{ vars.MAIL_PORT }}
MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
MAIL_USER: ${{ secrets.MAIL_USER }}
MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
PORT: ${{ vars.PORT }}
RELATIONSHIP_THRESHOLD: ${{ vars.RELATIONSHIP_THRESHOLD }}
S3_BUCKET: ${{ vars.S3_BUCKET }}
S3_KEY: ${{ secrets.S3_KEY }}
S3_SECRET: ${{ secrets.S3_SECRET }}
WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
WORKER_QUEUE: media
steps:
- name: Print Job Description
run: |
echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Add kubectl cert
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
- name: Apply deployment changes
run: |
envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
- name: Deploy Rollout
run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
- name: Verify deployment
run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
deploy-worker-notifs:
needs: build-worker
runs-on: ubuntu-latest
if: ${{ contains(github.event.head_commit.message, '~worker') || contains(github.event.head_commit.message, '~notifs') }}
environment: ${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
BASE_DOMAIN: ${{ vars.BASE_DOMAIN }}
CHROMIUM_EXECUTABLE_PATH: ${{ vars.CHROMIUM_EXECUTABLE_PATH }}
CONTAINER_NAME: ${{ github.event.repository.name }}-worker-${{ fromJSON('{"main":"prod"}')[github.ref_name] || github.ref_name }}-notifs
DEEPAI_API_KEY: ${{ secrets.DEEPAI_API_KEY }}
ENABLED_FEATURES: ${{ vars.ENABLED_FEATURES }}
ERROR_REPORTING: $${{ vars.ERROR_REPORTING }}
FIREBASE_CREDENTIALS: ${{ secrets.FIREBASE_CREDENTIALS }}
K8S_NAMESPACE: ${{ secrets.K8S_NAMESPACE }}
K8S_MEMORY_REQUESTS: ${{ vars.K8S_MEMORY_REQUESTS }}
K8S_CPU_REQUESTS: ${{ vars.K8S_CPU_REQUESTS }}
K8S_MEMORY_LIMITS: ${{ vars.K8S_MEMORY_LIMITS }}
K8S_CPU_LIMITS: ${{ vars.K8S_CPU_LIMITS }}
K8S_REPLICA_COUNT_WORKER: ${{ vars.K8S_REPLICA_COUNT_WORKER_NOTIFS || 1 }}
MAIL_HOST: ${{ vars.MAIL_HOST }}
MAIL_PASS: ${{ secrets.MAIL_PASS }}
MAIL_PORT: ${{ vars.MAIL_PORT }}
MAIL_REPLY_TO: ${{ vars.MAIL_REPLY_TO }}
MAIL_USER: ${{ secrets.MAIL_USER }}
MAX_OPENAI_TOKEN_COUNT: ${{ vars.MAX_OPENAI_TOKEN_COUNT }}
OLD_NEWS_THRESHOLD: ${{ vars.OLD_NEWS_THRESHOLD }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PUBLISHER_FETCH_INTERVAL: ${{ vars.PUBLISHER_FETCH_INTERVAL }}
PUBLISHER_FETCH_LIMIT: ${{ vars.PUBLISHER_FETCH_LIMIT }}
PUBLISHER_MAX_ATTEMPT_LIMIT: ${{ vars.PUBLISHER_MAX_ATTEMPT_LIMIT }}
PG_CONNECTION_STRING: ${{ secrets.PG_CONNECTION_STRING }}
PLAY_HT_SECRET_KEY: ${{ secrets.PLAY_HT_SECRET_KEY }}
PLAY_HT_USER_ID: ${{ secrets.PLAY_HT_USER_ID }}
PORT: ${{ vars.PORT }}
S3_BUCKET: ${{ vars.S3_BUCKET }}
S3_KEY: ${{ secrets.S3_KEY }}
S3_SECRET: ${{ secrets.S3_SECRET }}
WORKER_RETRY_EXPRS: ${{ vars.WORKER_RETRY_EXPRS }}
WORKER_QUEUE: notifs
steps:
- name: Print Job Description
run: |
echo deploying \"$IMAGE\" from branch \"${{ github.ref_name }}\"
- name: Git Checkout
uses: actions/checkout@v3
with:
ssh-key: ${{ secrets.GIT_SSH_KEY }}
submodules: recursive
- name: Install doctl
uses: digitalocean/action-doctl@v2
with:
token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
- name: Add kubectl cert
run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.K8S_CLUSTER_ID }}
- name: Apply deployment changes
run: |
envsubst < $GITHUB_WORKSPACE/k8s/worker-deploy.yaml | kubectl apply -f -
- name: Deploy Rollout
run: kubectl rollout restart deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE
- name: Verify deployment
run: kubectl rollout status deployment/$CONTAINER_NAME --namespace $K8S_NAMESPACE