Feature/sendgrid airflow #528
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cornflow-server tests | |
| on: | |
| pull_request: | |
| types: [ opened, edited, synchronize, reopened ] | |
| paths: | |
| - 'cornflow-server/**' | |
| - '.github/workflows/test_cornflow_server.yml' | |
| - '!cornflow-server/docs/**' | |
| - '!cornflow-server/README.rst' | |
| push: | |
| branches: | |
| - master | |
| - development | |
| paths: | |
| - 'cornflow-server/**' | |
| - '!cornflow-server/docs/**' | |
| - '!cornflow-server/README.rst' | |
| jobs: | |
| testing: | |
| name: Run all test suites on cornflow-server | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: ./cornflow-server | |
| strategy: | |
| max-parallel: 21 | |
| matrix: | |
| python-version: [3.8, 3.9, '3.10', '3.11'] | |
| os: [ubuntu-latest] | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_DB: cornflow-test | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgresadmin | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Copy DAG files | |
| run: | | |
| cd .. | |
| cp -r cornflow-dags/DAG/* cornflow-server/airflow_config/dags/ | |
| cp cornflow-dags/requirements.txt cornflow-server/airflow_config/ | |
| - name: Install dependencies | |
| run: | | |
| docker run --volume $GITHUB_WORKSPACE/cornflow-server/airflow_config:/etc/ldap.dist/prepopulate \ | |
| -e SLAPD_DOMAIN=example.org \ | |
| -e SLAPD_PASSWORD=admin \ | |
| -d -p 389:389 dinkel/openldap | |
| CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" | |
| python -m pip install --upgrade pip | |
| python -m pip install -U -r requirements-dev.txt | |
| python -m pip uninstall cornflow-client -y | |
| python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" | |
| - name: Install airflow | |
| run: | | |
| python -m venv afvenv | |
| source afvenv/bin/activate | |
| AIRFLOW_VERSION=2.7.1 | |
| PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)" | |
| CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" | |
| CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" | |
| python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" | |
| python -m pip install -U orloge pulp | |
| python -m pip uninstall cornflow-client -y | |
| python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" | |
| python -m pip install -U -r airflow_config/requirements.txt | |
| airflow db init | |
| airflow users create \ | |
| --username admin \ | |
| --firstname admin \ | |
| --lastname admin \ | |
| --role Admin \ | |
| --password admin \ | |
| --email [email protected] | |
| airflow webserver -p 8080 & | |
| airflow scheduler & | |
| deactivate | |
| sleep 5s | |
| env: | |
| AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT: 0 | |
| AIRFLOW_HOME: "${GITHUB_WORKSPACE}/cornflow-server/airflow_config" | |
| AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 0 | |
| AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth | |
| AIRFLOW__WEBSERVER__SECRET_KEY: e9adafa751fd35adfc1fdd3285019be15eea0758f76e38e1e37a1154fb36 | |
| AIRFLOW__CORE__LOAD_EXAMPLES: 0 | |
| AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050 | |
| - name: Run unit tests | |
| run: | | |
| coverage run --source=./cornflow/ --omit="*/tests/data/*" -m unittest discover -s cornflow/tests/unit | |
| coverage report -m | |
| env: | |
| FLASK_ENV: testing | |
| OPEN_DEPLOYMENT: 1 | |
| DATABASE_URL: sqlite:///cornflow_test.db | |
| LOG_LEVEL: 30 | |
| AIRFLOW_USER: notadmin | |
| AIRFLOW_PWD: notadmin | |
| CF_ALARMS_ENDPOINT: 1 | |
| - name: Run ldap unit tests | |
| run: | | |
| coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/ldap | |
| coverage report -m | |
| env: | |
| FLASK_ENV: testing | |
| DATABASE_URL: sqlite:///cornflow_test.db | |
| AUTH_TYPE: 2 | |
| OPEN_DEPLOYMENT: 1 | |
| LDAP_HOST: ldap://localhost:389 | |
| LDAP_BIND_PASSWORD: admin | |
| LDAP_GROUP_BASE: dc=example,dc=org | |
| LDAP_GROUP_OBJECT_CLASS: groupOfNames | |
| LDAP_PROTOCOL_VERSION: 3 | |
| LDAP_BIND_DN: cn=admin,dc=example,dc=org | |
| LDAP_USE_TLS: False | |
| LDAP_USERNAME_ATTRIBUTE: cn | |
| LDAP_USER_BASE: ou=users,dc=example,dc=org | |
| LDAP_SERVICE_BASE: ou=service,dc=example,dc=org | |
| LDAP_EMAIL_ATTRIBUTE: mail | |
| LDAP_USER_OBJECT_CLASS: inetOrgPerson | |
| LDAP_GROUP_ATTRIBUTE: cn | |
| LDAP_GROUP_TO_ROLE_SERVICE: services | |
| LDAP_GROUP_TO_ROLE_ADMIN: administrators | |
| LDAP_GROUP_TO_ROLE_VIEWER: viewers | |
| LDAP_GROUP_TO_ROLE_PLANNER: planners | |
| LOG_LEVEL: 30 | |
| CORNFLOW_SERVICE_USER: cornflow | |
| - name: Run postgres unit tests | |
| run: | | |
| coverage run -a --source=./cornflow/ -m unittest cornflow/tests/unit/test_commands.py | |
| coverage report -m | |
| env: | |
| FLASK_ENV: testing | |
| OPEN_DEPLOYMENT: 1 | |
| DATABASE_URL: postgresql://postgres:[email protected]:5432/cornflow-test | |
| LOG_LEVEL: 30 | |
| CF_ALARMS_ENDPOINT: 1 | |
| - name: Run integration tests | |
| run: | | |
| coverage run -a --source=./cornflow/ -m unittest discover -s cornflow/tests/integration | |
| coverage report -m | |
| coverage xml | |
| env: | |
| FLASK_ENV: testing | |
| OPEN_DEPLOYMENT: 1 | |
| DATABASE_URL: sqlite:///cornflow_test.db | |
| LOG_LEVEL: 30 | |
| - name: Upload coverage to codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: server-tests |