Add a fetcher for AWS S3 Argo GDAC data #1487
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests Upstream | |
| # Daily tests with all un-pinned dependencies | |
| # Allows to be warned when argopy fail tests due to updates in dependencies | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: # allows you to trigger the workflow run manually | |
| pull_request: | |
| types: [synchronize] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| detect-ci-trigger: | |
| name: detect upstream ci trigger | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| triggered: ${{ steps.detect-trigger.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/ci-trigger@v1 | |
| id: detect-trigger | |
| with: | |
| keyword: "[test-upstream]" | |
| detect-ci-skip: | |
| name: detect upstream ci skip | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| skipped: ${{ steps.detect-skip.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/[email protected] | |
| id: detect-skip | |
| with: | |
| keyword: "[skip-ci]" | |
| core-free: | |
| # CI tests for environments with core requirements in free versions | |
| name: Core - Free - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: [detect-ci-trigger, detect-ci-skip] | |
| if: | | |
| always() | |
| && ( | |
| (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| || needs.detect-ci-trigger.outputs.triggered == 'true' | |
| || needs.detect-ci-skip.outputs.skipped == 'false' | |
| || contains(github.event.pull_request.labels.*.name, 'release') | |
| ) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| # timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS_UPSTREAM }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-free.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-Core-Free-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup core-free' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ | |
| --report-log output-${{ matrix.python-version }}-log.jsonl | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest core-free' | |
| continue-on-error: true | |
| - name: Generate and publish a report on failure | |
| if: | | |
| failure() | |
| && github.repository_owner == 'euroargodev' | |
| uses: xarray-contrib/issue-from-pytest-log@main | |
| with: | |
| log-path: output-${{ matrix.python-version }}-log.jsonl | |
| issue-title: "Upstream CI tests failed: Core - Free - Py${{matrix.python-version}} - ${{ matrix.os }}" | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-Core-${{matrix.python-version}}-${{matrix.os}} | |
| path: output-${{ matrix.python-version }}-log.jsonl | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true | |
| all-free: | |
| # CI tests for environments with all possible requirements in free versions | |
| name: All - Free - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| needs: [detect-ci-trigger, detect-ci-skip] | |
| if: | | |
| always() | |
| && ( | |
| (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| || needs.detect-ci-trigger.outputs.triggered == 'true' | |
| || needs.detect-ci-skip.outputs.skipped == 'false' | |
| || contains(github.event.pull_request.labels.*.name, 'release') | |
| ) | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| # timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS_UPSTREAM }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-free.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| pytest-reportlog | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup all-free' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 \ | |
| --report-log output-${{ matrix.python-version }}-log.jsonl | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest all-free' | |
| continue-on-error: true | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-All-${{matrix.python-version}}-${{matrix.os}} | |
| path: output-${{ matrix.python-version }}-log.jsonl | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-berlin/eco-ci-energy-estimation@v4 | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true |