pull-request.yml

name: Pull Request test
on:
  pull_request:
    branches:
      - main
  workflow_dispatch:
env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
jobs:
  get-changed-notebooks:
    runs-on: ubuntu-latest
    outputs:
      changed_files: ${{ steps.output.outputs.changed_files }}
    steps:
      - uses: actions/checkout@main
        with:
          fetch-depth: 0
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41
        with:
          json: true
          files: |
            **/*.ipynb
          files_ignore: |
            **/Create_delete_projects.ipynb
            **/sagemaker/**
            **/utils/**
      - name: Set output
        id: output
        if: steps.changed-files.outputs.any_changed == 'true'
        run: echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> "$GITHUB_OUTPUT"
  test-notebooks:
    needs: get-changed-notebooks
    if: ${{ needs.get-changed-notebooks.outputs.changed_files }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        notebooks: ${{ fromJSON(needs.get-changed-notebooks.outputs.changed_files) }}
        exclude:
          - python-version: ${{ 3.8 }}
            notebooks: integrations-and-supported-tools/mosaicml-composer/notebooks/Neptune_MosaicML_Composer.ipynb
    steps:
      - uses: actions/checkout@main
      - uses: actions/setup-python@main
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -U -r requirements.txt
      - name: Install setuptools for python>=3.12
        if: ${{ matrix.python-version }} >= ${{ 3.12 }}
        run: pip install -U setuptools
      - name: Test examples
        uses: nick-fields/retry@v3
        env:
          NEPTUNE_API_TOKEN: ${{ secrets.NEPTUNE_API_TOKEN }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        with:
          timeout_minutes: 60
          max_attempts: 2
          retry_on: timeout
          command: |
            echo "Running ${{ matrix.notebooks }}"
            ipython ${{ matrix.notebooks }}
  get-changed-scripts:
    runs-on: ubuntu-latest
    outputs:
      changed_dirs: ${{ steps.output.outputs.changed_dirs }}
    steps:
      - uses: actions/checkout@main
        with:
          fetch-depth: 0
      - name: Get changed directories
        id: changed-dirs
        uses: tj-actions/changed-files@v41
        with:
          dir_names: "true"
          json: true
          files: |
            **/scripts/**
            **/code/**
          files_ignore: |
            **/ddp-training/**
            **/neptune-docker/**
            **/utils/**
      - name: Set output
        id: output
        if: steps.changed-dirs.outputs.any_changed == 'true'
        run: echo "changed_dirs=${{ steps.changed-dirs.outputs.all_changed_files }}" >> "$GITHUB_OUTPUT"
  test-scripts:
    needs: get-changed-scripts
    if: ${{ needs.get-changed-scripts.outputs.changed_dirs }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        dir: ${{ fromJSON(needs.get-changed-scripts.outputs.changed_dirs) }}
        exclude:
          - python-version: ${{ 3.8 }}
            dir:
            - integrations-and-supported-tools/kedro/scripts
            - integrations-and-supported-tools/mosaicml-composer/scripts
    steps:
      - uses: actions/checkout@main
      - uses: actions/setup-python@main
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Upgrade pip
        run: python -m pip install --upgrade pip
      - name: Install setuptools for python>=3.12
        if: ${{ matrix.python-version }} >= ${{ 3.12 }}
        run: pip install -U setuptools
      - name: Test examples
        uses: nick-fields/retry@v3
        env:
          NEPTUNE_API_TOKEN: ${{ secrets.NEPTUNE_API_TOKEN }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        with:
          timeout_minutes: 60
          max_attempts: 2
          retry_on: timeout
          command: |
            echo "Running ${{ matrix.dir }}/run_examples.sh"
            cd ${{ matrix.dir }}
            bash run_examples.sh
  get-changed-docker:
    runs-on: ubuntu-latest
    outputs:
      changed_files: ${{ steps.output.outputs.changed_files }}
    steps:
      - uses: actions/checkout@main
        with:
          fetch-depth: 0
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41
        with:
          files: how-to-guides/neptune-docker/scripts/*
      - name: Set output
        id: output
        if: steps.changed-files.outputs.any_changed == 'true'
        run: echo "changed_files='${{ steps.changed-files.outputs.any_modified }}" >> "$GITHUB_OUTPUT"
  test-docker-example:
    needs: get-changed-docker
    if: ${{ needs.get-changed-docker.outputs.changed_files }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@main
      - name: build docker container from Dockerfile
        run: docker build -f how-to-guides/neptune-docker/scripts/Dockerfile --tag neptune-docker .
      - name: run docker container
        timeout-minutes: 20
        run: docker run -e NEPTUNE_API_TOKEN=${{ secrets.NEPTUNE_API_TOKEN }} neptune-docker