local-ci.yml

name: GraphScope CI on Local

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  workflow_dispatch:
  push:
    branches:
      - main
    paths-ignore:
      - 'CONTRIBUTORS'
      - 'LICENSE'
      - 'NOTICE.txt'
      - '**.md'
      - '**.rst'
      - 'docs/**'
      - 'demo/**'
      - 'scripts/**'
      - 'tutorials/**'
  pull_request:
    branches:
      - main
    paths:
      - '.github/workflows/local-ci.yml'
      - 'Makefile'
      - 'analytical_engine/**'
      - 'charts/**'
      - 'coordinator/**'
      - 'interactive_engine/**'
      - 'k8s/**'
      - 'learning_engine/**'
      - 'proto/**'
      - 'python/**'
      - '!**.md'
      - '!**.rst'

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope

jobs:
  # JOB to run change detection
  changes:
    runs-on: ubuntu-20.04
    # Set job outputs to values from filter step
    outputs:
      gae-python: ${{ steps.filter.outputs.gae-python }}
      networkx: ${{ steps.filter.outputs.networkx }}
      gie-function-test: ${{ steps.filter.outputs.gie-function-test }}
    steps:
    # For push it's necessary to checkout the code
    - uses: actions/checkout@v4

    - name: Detect the tmate session
      run: |
        # hack here: get all field of build-wheels which run on self-hosted machine
        if sed -n '/self-hosted/,/mini-test/ p' .github/workflows/local-ci.yml | \
          grep -v "grep" - | grep "action-tmate"; then
          echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually'
          exit 1
        fi

    # For pull requests it's not necessary to checkout the code
    - uses: dorny/paths-filter@v3
      id: filter
      with:
        base: main  # Change detection against merge-base with main before push
        filters: |
          gae-python:
            - 'proto/**'
            - 'analytical_engine/**'
            - 'python/graphscope/analytical/**'
            - 'python/graphscope/client/**'
            - 'python/graphscope/dataset/**'
            - 'python/graphscope/deploy/**'
            - 'python/graphscope/framework/**'
            - 'python/graphscope/tests/unittest/**'
            - 'coordinator/gscoordinator/**'
            - '.github/workflows/local-ci.yml'
          networkx:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'analytical_engine/core/**'
            - 'python/graphscope/nx/**'
            - 'python/requirements.txt'
          gie-function-test:
            - 'interactive_engine/**'
            - 'python/graphscope/interactive/**'
            - '.github/workflows/ci.yml'
          gie-k8s-failover-test:
            - 'interactive_engine/**'
            - 'charts/ir-standalone/**'

    - name: Cpp Format and Lint Check
      run: |
        # install clang-format
        sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format
        sudo chmod +x /usr/bin/clang-format

        # collect the source files
        cd analytical_engine/
        files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \))

        # run format
        clang-format -i --style=file $(echo $files)

        # validate format
        function prepend() { while read line; do echo "${1}${line}"; done; }

        GIT_DIFF=$(git diff --ignore-submodules)
        if [[ -n $GIT_DIFF ]]; then
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echo "| clang-format failures found!"
            echo "|"
            echo "$GIT_DIFF" | prepend "| "
            echo "|"
            echo "| Run: "
            echo "|"
            echo "|    make gsa_clformat"
            echo "|"
            echo "| to fix this error."
            echo "|"
            echo "| Ensure you are working with clang-format-8, which can be obtained from"
            echo "|"
            echo "|    https://github.com/muttleyxd/clang-tools-static-binaries/releases"
            echo "|"
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            exit -1
        fi

        # validate cpplint
        function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; }

        # run cpplint
        ec ./misc/cpplint.py $(echo $files)
        if [[ "$ec" != "0" ]]; then
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echo "| cpplint failures found! Run: "
            echo "|"
            echo "|    make gsa_cpplint"
            echo "|"
            echo "| to fix this error."
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            exit -1
        fi

    - name: Python Format and Lint Check
      run: |
        echo "Checking formatting for $GITHUB_REPOSITORY"
        pip3 install -r coordinator/requirements-dev.txt
        pushd python
        python3 -m isort --check --diff .
        python3 -m black --check --diff .
        python3 -m flake8 .
        popd
        pushd coordinator
        python3 -m isort --check --diff .
        python3 -m black --check --diff .
        python3 -m flake8 .

    - name: Setup Java11
      uses: actions/setup-java@v4
      with:
         distribution: 'zulu'
         java-version: '11'

    - name: Java Format and Lint Check
      run: |
        wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar

        files_to_format=$(git ls-files *.java)

        # run formatter in-place
        java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format

        # validate format
        function prepend() { while read line; do echo "${1}${line}"; done; }

        GIT_DIFF=$(git diff --ignore-submodules)
        if [[ -n $GIT_DIFF ]]; then
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echo "| google-java-format failures found!"
            echo "|"
            echo "$GIT_DIFF" | prepend "| "
            echo "|"
            echo "| Run: "
            echo "|"
            echo '|    java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)'
            echo "|"
            echo "| to fix this error."
            echo "|"
            echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from"
            echo "|"
            echo "|    https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar"
            echo "|"
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            exit -1
        fi

  build-wheels:
    # Require the user id of the self-hosted is 1001, which may need to be
    # configured manually when a new self-hosted runner is added.
    runs-on: [self-hosted, ubuntu2004]
    needs: [changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - uses: actions/cache@v4
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ runner.os }}-maven-

    - uses: actions/cache@v4
      with:
        path: |
          ~/.cargo/bin/
          ~/.cargo/registry/index/
          ~/.cargo/registry/cache/
          ~/.cargo/git/db/
          ~/.cache/sccache
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    - uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Build Wheels
      run: |
        #sleep infinity
        git submodule update --init
        cd ${GITHUB_WORKSPACE}/learning_engine/graph-learn && git submodule update --init third_party/pybind11

        # build graphscope wheels
        cd ${GITHUB_WORKSPACE}/k8s/internal
        # sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope
        make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com

        # build client wheels
        # sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope
        make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com

        # package
        cd ${GITHUB_WORKSPACE}
        sudo chown $(id -u) -R .
        tar -zcf client.tar.gz python/dist/wheelhouse/*.whl
        tar -zcf graphscope.tar.gz coordinator/dist/

    - name: Upload Artifact
      uses: actions/upload-artifact@v4
      with:
        name: wheel-${{ github.sha }}
        path: |
          client.tar.gz
          graphscope.tar.gz
        retention-days: 5

  mini-test:
    runs-on: ubuntu-20.04
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    needs: [build-wheels]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          path: artifacts

      - name: Prepare Environment
        env:
          GS_TEST_DIR: ${{ github.workspace }}/gstest
        run: |
          # install graphscope-client
          cd artifacts
          tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
          pushd python/dist/wheelhouse
          for f in * ; do python3 -m pip install --no-cache-dir $f || true; done
          popd

          # install graphscope
          tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
          pushd coordinator/dist
          python3 -m pip install --no-cache-dir ./*.whl
          popd
          pushd coordinator/dist/wheelhouse
          python3 -m pip install --no-cache-dir ./*.whl
          popd

          # install tensorflow
          python3 -m pip install --no-cache-dir pytest pytest-xdist "pandas" --user
          # python3 -m pip install --no-cache-dir "tensorflow" --user
          # install pytorch
          # python3 -m pip install --no-cache-dir "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu
          # install java
          sudo apt update -y && sudo apt install openjdk-11-jdk -y

          # download dataset
          git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: false

      - name: Run Minimum Test
        env:
          GS_TEST_DIR: ${{ github.workspace }}/gstest
        run: |
          python3 -m pytest -d --tx popen//python=python3 \
                            -s -v \
                            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest

      - name: Upload GIE log
        if: failure()
        uses: actions/upload-artifact@v4
        continue-on-error: true
        with:
          name: gie-log
          path: ~/.local/log/graphscope

  python-unittest:
    runs-on: ubuntu-20.04
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    needs: [build-wheels, changes]
    defaults:
      run:
        shell: bash --noprofile --norc -eo pipefail {0}
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]

    steps:
    - uses: actions/checkout@v4
      if: ${{ needs.changes.outputs.gae-python == 'true' }}

    - uses: actions/download-artifact@v4
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install --no-cache-dir $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install --no-cache-dir ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install --no-cache-dir ./*.whl
        popd

        # install pytest
        python3 -m pip install --no-cache-dir pytest pytest-cov pytest-timeout pytest-xdist

    - name: Setup tmate session
      uses: mxschmitt/action-tmate@v3
      if: false

    - name: Run Python Test
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        DEPLOYMENT: ${{ matrix.deployment }}
      run: |
        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
        export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

        python3 -m pytest -d --tx popen//python=python3 \
                          -s -v \
                          --cov=graphscope --cov-config=${GITHUB_WORKSPACE}/python/.coveragerc --cov-report=xml --cov-report=term \
                          --exitfirst \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/unittest

    - name: Upload Coverage
      uses: codecov/codecov-action@v4
      continue-on-error: true
      with:
        files: ./coverage.xml
        token: ${{ secrets.CODECOV_TOKEN }}
        verbose: true

  networkx-basic-test:
    runs-on: ubuntu-20.04
    needs: [build-wheels, changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]
    steps:
    - uses: actions/checkout@v4
      if: ${{ needs.changes.outputs.networkx == 'true' }}

    - uses: dorny/paths-filter@v3
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      id: nx-filter
      with:
        filters: |
          basic:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'analytical_engine/core/**'
            - 'python/graphscope/nx/__init__.py'
            - 'python/graphscope/nx/classes/**'
            - 'python/graphscope/nx/!(tests)'
            - 'python/graphscope/nx/tests/!(convert)'
            - 'python/requirements.txt'
          convert:
            - 'python/graphscope/nx/convert.py'
            - 'python/graphscope/nx/convert_matrix.py'
            - 'python/graphscope/nx/tests/convert/**'

    - uses: actions/download-artifact@v4
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install --no-cache-dir $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install --no-cache-dir ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install --no-cache-dir ./*.whl
        popd

        # install pytest
        python3 -m pip install --no-cache-dir pytest pytest-xdist

        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

    - name: Setup tmate session
      uses: mxschmitt/action-tmate@v3
      if: false

    - name: Basic Test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.basic == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest -d --tx popen//python=python3 \
                          --exitfirst -s -vvv \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests \
                          --ignore=$(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert

    - name: Convert Test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.convert == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest -d --tx popen//python=python3 \
                          --exitfirst -s -vvv \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert

  networkx-algo-and-generator-test:
    runs-on: ubuntu-20.04
    needs: [build-wheels, changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]
    steps:
    - uses: actions/checkout@v4
      if: ${{ needs.changes.outputs.networkx == 'true' }}

    - uses: dorny/paths-filter@v3
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      id: nx-filter
      with:
        filters: |
          algorithm:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'python/graphscope/nx/algorithms/builtin.py'
            - 'python/graphscope/nx/algorithms/tests/builtin/**'
          generator:
            - 'python/graphscope/nx/classes/**'
            - 'python/graphscope/nx/utils/**'
            - 'python/graphscope/nx/generators/**'
          io:
            - 'python/graphscope/nx/readwrite/**'

    - uses: actions/download-artifact@v4
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install --no-cache-dir $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install --no-cache-dir ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install --no-cache-dir ./*.whl
        popd

        # install pytest
        python3 -m pip install --no-cache-dir pytest pytest-xdist

        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

    - name: Algorithm test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.algorithm == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest -d --tx popen//python=python3 \
                          --exitfirst -s -v \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/algorithms/tests/builtin

    - name: Generator test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.generator == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest -d --tx popen//python=python3 \
                          --exitfirst -s -v \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/generators/tests

    - name: Readwrite test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.io == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
      run: |
        python3 -m pytest -d --tx popen//python=python3 \
                          --exitfirst -s -v -m "not slow" \
                          $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/readwrite/tests