docker-compose.override.yml

version: '3.4'

x-airflow-env:
  &airflow-env
    - CIVICRM_API_KEY_FILE_PATH=/home/airflow/secrets/civi-key/civi_api_key.txt
    - CIVICRM_SITE_KEY_FILE_PATH=/home/airflow/secrets/civi-key/civi_site_key.txt
    - PATH_TO_TOGGL_API_TOKEN_FILE=/home/airflow/secrets/toggl.txt
    - GMAIL_E2E_TEST_ACCOUNT_SECRET_FILE=/home/airflow/secrets/gmail_end2end_test_credentials.json
    - SURVEYMONKEY_SECRET_FILE=/home/airflow/secrets/surveymonkey_credentials.json
    # Semantic Scholar API Key is optional
    - SEMANTIC_SCHOLAR_API_KEY_FILE_PATH=/home/airflow/secrets/semantic_scholar_api_key.txt
    - GITHUB_API_AUTHORIZATION_FILE_PATH=/home/airflow/secrets/github_api_authorization.txt
    - TWITTER_API_AUTHORIZATION_FILE_PATH=/home/airflow/secrets/twitter_api_authorization.txt
    - TWITTER_API_KEY_FILE_PATH=/home/airflow/secrets/twitter_api_key.txt
    - TWITTER_API_SECRET_FILE_PATH=/home/airflow/secrets/twitter_api_secret.txt
    - TWITTER_ACCESS_TOKEN_FILE_PATH=/home/airflow/secrets/twitter_access_token.txt
    - TWITTER_ACCESS_TOKEN_SECRET_FILE_PATH=/home/airflow/secrets/twitter_access_token_secret.txt

x-airflow-volumes:
  &airflow-volumes
    - ~/.config/gcloud/application_default_credentials.json:/tmp/credentials.json
    - ./dags:/opt/airflow/dags
    - ./data_pipeline:/opt/airflow/data_pipeline
    - ./tests:/opt/airflow/tests

x-airflow-defaults:
  &x-airflow-defaults
    volumes: *airflow-volumes
    environment: *airflow-env


services:
    data-hub-dags-dev:
        volumes: *airflow-volumes
        environment: *airflow-env

    webserver:
        ports:
            - "8080:8080"
        volumes: *airflow-volumes
        environment: *airflow-env

    scheduler:
        volumes: *airflow-volumes
        environment: *airflow-env


    worker:
        volumes: *airflow-volumes
        environment: *airflow-env

    test-client:
        volumes: *airflow-volumes
        environment: *airflow-env

    data-hub-pipelines:
        <<: *x-airflow-defaults
        environment:
            - DEPLOYMENT_ENV=${DATA_HUB_DEPLOYMENT_ENV}
            - GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials.json
            - PYTHONPATH=/home/airflow/.local/lib/python3.8/site-packages
        user: 0:0

    test-ftpserver:
        ports:
            - "8021:21"

    opensearch:
        ports:
            - 9200:9200
            - 9600:9600  # required for Performance Analyzer

    opensearch-dashboards:
        ports:
            - 5601:5601