crawl.docker-compose.yml

version: "3.7"

services:

  crawl_google_news:
    build: ./flow
    command: python3 /usr/app/crawl_google_news.py
    volumes:
      - /srv/docker/prefect/flows:/root/.prefect/flows
      - "./flow/scripts:/usr/app:ro"
      - type: bind
        source: ./flow/config.toml
        target: /root/.prefect/config.toml
        read_only: true
    env_file:
      - .env
    environment:
      MAX_ES_ROW_INJECT: ${MAX_ES_ROW_INJECT}
      ELASTIC_SCHEME: ${ELASTIC_SCHEME}
      ELASTIC_PORT: ${ELASTIC_PORT}
      ELASTIC_ENDPOINT: ${ELASTIC_ENDPOINT}
      ELASTIC_USER: ${ELASTIC_USER}
      ELASTIC_PWD: ${ELASTICSEARCH_PWD}

  crawl_tweets: 
    build: ./flow
    command: python3 /usr/app/crawl_tweets.py
    volumes:
      - /srv/docker/prefect/flows:/root/.prefect/flows
      - "./flow/scripts:/usr/app:ro"
      - type: bind
        source: ./flow/config.toml
        target: /root/.prefect/config.toml
        read_only: true
    env_file:
      - .env
    environment:
      MAX_ES_ROW_INJECT: ${MAX_ES_ROW_INJECT}
      ELASTIC_SCHEME: ${ELASTIC_SCHEME}
      ELASTIC_PORT: ${ELASTIC_PORT}
      ELASTIC_ENDPOINT: ${ELASTIC_ENDPOINT}
      ELASTIC_USER: ${ELASTIC_USER}
      ELASTIC_PWD: ${ELASTICSEARCH_PWD}