continuous-integration.yml

# Graphstorm CI
# Test execution order - [lint] -> [pytest] -> [end2end_cpu, end2end, end2end_mgpu]
name: continuous-integration

on:
  push:
    branches:
      - main
      - ci_dev
    paths-ignore:
      - '.github/workflow_scripts/gsprocessing_pytest.sh'
      - '.github/workflow_scripts/gsprocessing_lint.sh'
      - 'graphstorm-processing/**'
      - '.github/workflows/gsprocessing-workflow.yml'
      - 'docs/**'
  pull_request_target:
    types: [ labeled, opened, reopened, synchronize, ready_for_review ]
    paths-ignore:
      - '.github/workflow_scripts/gsprocessing_pytest.sh'
      - '.github/workflow_scripts/gsprocessing_lint.sh'
      - 'graphstorm-processing/**'
      - '.github/workflows/gsprocessing-workflow.yml' 
      - 'docs/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.number || github.event.pull_request.head.sha }}
  cancel-in-progress: true
  
permissions:
  id-token: write
  pull-requests: write
  contents: read

defaults:
  run:
    shell: bash

# All test scripts split in individual .sh files and moved to .github/workflow scripts
env:
  COMMAND-PYTEST: chmod +x ./pytest_check.sh && ./pytest_check.sh
  COMMAND-LINT: chmod +x ./lint_check.sh && ./lint_check.sh
  COMMAND-E2E-CPU-TEST: chmod +x ./e2e_cpu_check.sh && ./e2e_cpu_check.sh
  COMMAND-E2E: chmod +x ./e2e_check.sh && ./e2e_check.sh
  COMMAND-E2E-MGPU: chmod +x ./e2e_mgpu_check.sh && ./e2e_mgpu_check.sh
  COMMAND-E2E-MGPU-MULTITASK: chmod +x ./e2e_mgpu_multitask_check.sh && ./e2e_mgpu_multitask_check.sh
  COMMAND-E2E-GB: chmod +x ./e2e_gb_check.sh && ./e2e_gb_check.sh

jobs:

  lint_check:
    if: ${{ github.event_name == 'push' ||
      (github.event.label.name != 'draft'
      && github.event_name == 'pull_request_target'
      && contains(github.event.pull_request.labels.*.name, 'ready'))}}
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        role-duration-seconds: 14400
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-LINT }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-LINT-PUSH --name Graphstorm-LintTest-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-LINT }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait


  pytest_check:
    if: ${{ github.event.label.name != 'draft' }}
    needs: lint_check
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-CPU-PUSH --name Graphstorm-PyTest-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-PYTEST }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-CPU --name Graphstorm-PyTest-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-PYTEST }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait


  e2e_cpu_check:
    if: ${{ github.event.label.name != 'draft' }}
    needs: [pytest_check, lint_check]
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-CPU-CHECK-PUSH --name Graphstorm-E2ETest-CPU-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-E2E-CPU-TEST }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-CPU-CHECK --name Graphstorm-E2ETest-CPU-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-E2E-CPU-TEST }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait


  e2e_check:
    if: ${{ github.event.label.name != 'draft' }}
    needs: [pytest_check, lint_check]
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        role-duration-seconds: 14400
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-CPU-PUSH --name Graphstorm-E2ETest-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-E2E }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-CPU --name Graphstorm-E2ETest-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-E2E }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait


  e2e_mgpu_check:
    if: ${{ github.event.label.name != 'draft' }}
    needs: [pytest_check, lint_check]
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-GPU-PUSH --name Graphstorm-E2E-MGPUTest-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-E2E-MGPU }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-GPU --name Graphstorm-E2E-MGPUTest-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-E2E-MGPU }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait

  e2e_mgpu_check_multitask:
    if: ${{ github.event.label.name != 'draft' }}
    needs: [pytest_check, lint_check]
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-GPU-PUSH --name Graphstorm-E2E-MGPUTest-MultiTask-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-E2E-MGPU-MULTITASK }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-GPU --name Graphstorm-E2E-MGPUTest-MultiTask-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-E2E-MGPU-MULTITASK }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait
  e2e_graphbolt:
    if: ${{ github.event.label.name != 'draft' }}
    needs: [pytest_check, lint_check]
    runs-on: ubuntu-latest
    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        role-to-assume: arn:aws:iam::698571788627:role/github-oidc-role
        aws-region: us-east-1
    - name: Checkout repository
      uses: actions/checkout@v3
    - name: Set up Python 3.x
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      shell: bash
      run: |
        python3 -m pip install --upgrade pip
        python3 -m pip install pytest
        pip3 install boto3
    - name: Submit Job (for Push)
      if: ${{ github.event_name == 'push' }}
      shell: bash
      run: |
        echo "Start submitting job - Push"  
        python3 ./submitJob.py --job-type CI-GPU-DGL230 --name Graphstorm-E2E-Test-GB-Push-'${{ github.ref }}' \
                               --command "${{ env.COMMAND-E2E-GB }}" \
                               --remote https://github.com/'${{ github.repository }}' \
                               --source-ref '${{ github.ref }}' \
                               --wait
    - name: Submit Job (for Pull Request)
      if: ${{ github.event_name == 'pull_request_target' }}
      shell: bash
      run: |
        echo "Start submitting job - Pull Request"
        python3 ./submitJob.py --job-type CI-GPU-DGL230 --name Graphstorm-E2E-Test-GB-PR#-'${{ github.event.number }}' \
                               --command "${{ env.COMMAND-E2E-GB }}" \
                               --remote https://github.com/'${{ github.event.pull_request.head.repo.full_name }}' \
                               --source-ref '${{ github.event.pull_request.head.sha }}' \
                               --wait