AZP: Add header check for copyrights scan

openucx · Jan 14, 2025 · 8629f0c · 8629f0c
1 parent 63be496
commit 8629f0c
Show file tree

Hide file tree

Showing 4 changed files with 223 additions and 104 deletions.
diff --git a/buildlib/pr/codestyle.yml b/buildlib/pr/codestyle.yml
@@ -1,125 +1,143 @@
 jobs:
   # Check that commit title matches code style guidelines
-  - job: commit_title
-    displayName: commit title
-    pool:
-      name: MLNX
-      demands:
-      - ucx_docker -equals yes
-    steps:
-      - checkout: self
-        clean: true
-        fetchDepth: 100
-        retryCountOnTaskFailure: 5
+  # - job: commit_title
+  #   displayName: commit title
+  #   pool:
+  #     name: MLNX
+  #     demands:
+  #     - ucx_docker -equals yes
+  #   steps:
+  #     - checkout: self
+  #       clean: true
+  #       fetchDepth: 100
+  #       retryCountOnTaskFailure: 5
 
-      - bash: |
-          set -eE
-          source ./buildlib/tools/codestyle.sh
-
-          BASE_SOURCEVERSION=$(git rev-parse HEAD^)
-          range="$BASE_SOURCEVERSION..$(Build.SourceVersion)"
-
-          codestyle_check_commit_title "$range"
-          if [[ $? -ne 0 ]]
-          then
-              url="https://github.com/openucx/ucx/wiki/Guidance-for-contributors#general-guidelines"
-              echo "##vso[task.logissue type=error]Bad commit title(s), see $url for more info."
-              echo "##vso[task.complete result=Failed;]"
-          fi
-        condition: eq(variables['Build.Reason'], 'PullRequest')
-
-  # Check that the code is formatted according to the code style guidelines
-  - job: format
-    displayName: format code
-    pool:
-      name: MLNX
-      demands:
-      - ucx_docker -equals yes
-    container: fedora
-    steps:
-      - checkout: self
-        clean: true
-        fetchDepth: 100
-        retryCountOnTaskFailure: 5
+  #     - bash: |
+  #         set -eE
+  #         source ./buildlib/tools/codestyle.sh
 
-      - bash: |
-          source ./buildlib/az-helpers.sh
-          set -x
-          git log -1 HEAD
-          git log -1 HEAD^
-          BASE_SOURCEVERSION=$(git rev-parse HEAD^)
-          echo "Checking code format on diff ${BASE_SOURCEVERSION}..${BUILD_SOURCEVERSION}"
-          git-clang-format --diff ${BASE_SOURCEVERSION} ${BUILD_SOURCEVERSION} > format.patch
-          echo "Generated patch file:"
-          cat format.patch
-          if [ "`cat format.patch`" = "no modified files to format" ]; then
-              exit
-          fi
-          git apply format.patch
-          if ! git diff --quiet --exit-code
-          then
-              url="https://github.com/openucx/ucx/wiki/Code-style-checking"
-              azure_complete_with_issues "Code is not formatted according to the code style, see $url for more info."
-          fi
-        condition: eq(variables['Build.Reason'], 'PullRequest')
-
-  - job: author
-    displayName: AUTHORS file update check
-    pool:
-      name: MLNX
-      demands:
-      - ucx_docker -equals yes
-    container: fedora
-    steps:
-      - checkout: self
-        clean: true
-        fetchDepth: 100
-        retryCountOnTaskFailure: 5
+  #         BASE_SOURCEVERSION=$(git rev-parse HEAD^)
+  #         range="$BASE_SOURCEVERSION..$(Build.SourceVersion)"
 
-      - bash: |
-          set -eEx
+  #         codestyle_check_commit_title "$range"
+  #         if [[ $? -ne 0 ]]
+  #         then
+  #             url="https://github.com/openucx/ucx/wiki/Guidance-for-contributors#general-guidelines"
+  #             echo "##vso[task.logissue type=error]Bad commit title(s), see $url for more info."
+  #             echo "##vso[task.complete result=Failed;]"
+  #         fi
+  #       condition: eq(variables['Build.Reason'], 'PullRequest')
 
-          BASE_SOURCEVERSION=$(git rev-parse HEAD^)
-          range="$BASE_SOURCEVERSION..$(Build.SourceVersion)"
+  # # Check that the code is formatted according to the code style guidelines
+  # - job: format
+  #   displayName: format code
+  #   pool:
+  #     name: MLNX
+  #     demands:
+  #     - ucx_docker -equals yes
+  #   container: fedora
+  #   steps:
+  #     - checkout: self
+  #       clean: true
+  #       fetchDepth: 100
+  #       retryCountOnTaskFailure: 5
 
-          echo "Looking for missing AUTHORS on commit range ${range}"
-          ./contrib/authors_update.sh "$range"
-          git diff --exit-code
-        displayName: AUTHORS file check
+  #     - bash: |
+  #         source ./buildlib/az-helpers.sh
+  #         set -x
+  #         git log -1 HEAD
+  #         git log -1 HEAD^
+  #         BASE_SOURCEVERSION=$(git rev-parse HEAD^)
+  #         echo "Checking code format on diff ${BASE_SOURCEVERSION}..${BUILD_SOURCEVERSION}"
+  #         git-clang-format --diff ${BASE_SOURCEVERSION} ${BUILD_SOURCEVERSION} > format.patch
+  #         echo "Generated patch file:"
+  #         cat format.patch
+  #         if [ "`cat format.patch`" = "no modified files to format" ]; then
+  #             exit
+  #         fi
+  #         git apply format.patch
+  #         if ! git diff --quiet --exit-code
+  #         then
+  #             url="https://github.com/openucx/ucx/wiki/Code-style-checking"
+  #             azure_complete_with_issues "Code is not formatted according to the code style, see $url for more info."
+  #         fi
+  #       condition: eq(variables['Build.Reason'], 'PullRequest')
 
+  # - job: author
+  #   displayName: AUTHORS file update check
+  #   pool:
+  #     name: MLNX
+  #     demands:
+  #     - ucx_docker -equals yes
+  #   container: fedora
+  #   steps:
+  #     - checkout: self
+  #       clean: true
+  #       fetchDepth: 100
+  #       retryCountOnTaskFailure: 5
 
-  - job: codespell
-    displayName: codespell check
-    pool:
-      name: MLNX
-      demands:
-      - ucx_docker -equals yes
-    container: fedora
-    steps:
-      - checkout: self
-        clean: true
-        fetchDepth: 100
-        retryCountOnTaskFailure: 5
+  #     - bash: |
+  #         set -eEx
 
-      - bash: |
-          set -eE
-          source ./buildlib/tools/codestyle.sh
-          codestyle_check_spell
-        displayName: codespell test
+  #         BASE_SOURCEVERSION=$(git rev-parse HEAD^)
+  #         range="$BASE_SOURCEVERSION..$(Build.SourceVersion)"
+
+  #         echo "Looking for missing AUTHORS on commit range ${range}"
+  #         ./contrib/authors_update.sh "$range"
+  #         git diff --exit-code
+  #       displayName: AUTHORS file check
+
+
+  # - job: codespell
+  #   displayName: codespell check
+  #   pool:
+  #     name: MLNX
+  #     demands:
+  #     - ucx_docker -equals yes
+  #   container: fedora
+  #   steps:
+  #     - checkout: self
+  #       clean: true
+  #       fetchDepth: 100
+  #       retryCountOnTaskFailure: 5
+
+  #     - bash: |
+  #         set -eE
+  #         source ./buildlib/tools/codestyle.sh
+  #         codestyle_check_spell
+  #       displayName: codespell test
+
+  # - job: ctags_generation
+  #   displayName: ctags check
+  #   pool:
+  #     name: MLNX
+  #     demands:
+  #     - ucx_docker -equals yes
+  #   container: fedora
+  #   steps:
+  #     - checkout: self
+  #       clean: true
+  #       fetchDepth: 100
+  #       retryCountOnTaskFailure: 5
+
+  #     - bash: |
+  #         ./buildlib/tools/test_ctags.sh
+  #       displayName: ctags generation test
 
-  - job: ctags_generation
-    displayName: ctags check
+  - job: copyright_check
+    displayName: Copyright Check
     pool:
       name: MLNX
       demands:
-      - ucx_docker -equals yes
-    container: fedora
+      - ucx_docker
     steps:
       - checkout: self
         clean: true
         fetchDepth: 100
         retryCountOnTaskFailure: 5
 
       - bash: |
-          ./buildlib/tools/test_ctags.sh
-        displayName: ctags generation test
+          ./buildlib/tools/copyright_check.sh
+        displayName: Copyright Check
+        env:
+          WORKSPACE: $(workspace)
diff --git a/buildlib/pr/main.yml b/buildlib/pr/main.yml
@@ -180,6 +180,9 @@ resources:
     - container: ubuntu2204_rocm_6_0_0
       image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu2204:rocm-6.0.0
       options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
+    - container: header_check
+      image: rdmz-harbor.rdmz.labs.mlnx/toolbox/header_check
+      options: --config copyright-check-map.yaml --git-repo ${WORKSPACE} --path ${WORKSPACE}
 
 stages:
   - stage: Codestyle

diff --git a/buildlib/tools/copyright-check-map.yaml b/buildlib/tools/copyright-check-map.yaml
@@ -0,0 +1,30 @@
+general:
+  exclude:
+    - "\\.git.*"
+    - "\\.(yml|md|txt)"
+    - "^\\.ci.*"
+    - "NEWS"
+    - "\\.(m4|ac)"
+    - "\\.conf"
+    - "src/components/tl/nvat/*"
+    - "\\.cu"
+    - "AUTHORS"
+    - "LICENSE"
+    - "docs/*"
+    - "\\.Dockerfile"      # might need an update from this point on
+    - "perftest_msg_pow2"
+    - "contrib/*"
+    - "debian/*"
+    - "src/\\*/Makefile"
+    - "src/ucg/Makefile"
+    - "src/ucg/base/Makefile"
+    - "src/ucg/builtin/Makefile"
+    - "src/ucs/arch/aarch64/memcpy_thunderx2.S"
+    - "\\.mod"
+    - "\\.supp"
+    - "\\.log"
+
+bsd:
+  validate-spdx-license: true
+  include:
+    - ".*\\.(am|in|hpp|cpp|py|cc|h|c|sh|go|xml|java|yaml|inl)$"
diff --git a/buildlib/tools/copyright_check.sh b/buildlib/tools/copyright_check.sh
@@ -0,0 +1,68 @@
+#!/bin/bash -x
+
+if [ -z "$WORKSPACE" ]; then
+    echo "ERROR: WORKSPACE variable is empty"
+    exit 1
+fi
+
+if [ ! -d "$WORKSPACE" ]; then
+    echo "ERROR: ""$WORKSPACE"" does not exist"
+    exit 1
+fi
+
+# this variable is mandatory for header_check.py
+if [[ -z $GITHUB_TOKEN ]]; then
+    echo "ERROR: GITHUB_TOKEN variable is empty"
+    exit 1
+fi
+
+IMAGE="rdmz-harbor.rdmz.labs.mlnx/toolbox/header_check"
+IMAGE="harbor.mellanox.com/toolbox/header_check"
+HEADER_CHECK_TOOL="docker run --rm \
+    --user $(id -u):$(id -g) \
+    -v $PWD:$PWD -w $PWD \
+    $IMAGE"
+
+$HEADER_CHECK_TOOL \
+    --config "${WORKSPACE}"/buildlib/tools/copyright-check-map.yaml \
+    --path "$WORKSPACE" \
+    --git-repo "$WORKSPACE" | tee copyrights.log
+
+exit_code=$?
+echo "exit_code=${exit_code}"
+# Correct error code is not returned by the script, need to check output file if its empty it failed
+if [[ ! -s copyrights.log ]]; then
+    echo "copyrights.log is empty which means the script failed internally"
+    exit 1
+fi
+set +eE
+grep -rn ERROR copyrights.log
+exit_code=$?
+set -eE
+if [ ${exit_code} -eq 0 ]; then
+    echo "Please refer to https://confluence.nvidia.com/pages/viewpage.action?pageId=788418816"
+    ${HEADER_CHECK_TOOL} \
+      --config "${WORKSPACE}"/buildlib/tools/copyright-check-map.yaml \
+      --path "${WORKSPACE}" \
+      --repair \
+      --git-repo "${WORKSPACE}" | tee copyrights_repair.log
+    # create list of modified files
+    # needed for new git versions (from the check header docker image)
+    git config --global --add safe.directory "$WORKSPACE"
+    files=$(git status | grep 'modified:' | awk '{print $NF}'  )
+    mkdir "$WORKSPACE"/repaired_files/
+    cp --parents "$files" "$WORKSPACE"/repaired_files/
+    cd "$WORKSPACE"/repaired_files/
+    tar -czf "$WORKSPACE"/copyright_repaired_files.tar.gz .
+    exit 1
+fi
+if [ "${do_release}" == "true" ] && [ ! -z "${RMAX_SDK_VERSION}" ]; then
+    REPORTS_DIR="${release_dir}/${RMAX_SDK_VERSION}/reports"
+    sudo -E -u swx-jenkins mkdir -p "${REPORTS_DIR}"
+    sudo -E -u swx-jenkins cp -v copyrights.log "${REPORTS_DIR}"
+
+    # upload to artifactory - experimental does not fail build
+    TARGET_PATH="$ARTIFACTORY_URL/$ARTIFACTORY_REPO/release/$RMAX_SDK_VERSION/linux/reports/copyright/"
+    curl --silent --show-error --fail -u "$ARTIFACTORY_USER:$ARTIFACTORY_KEY" -T copyrights.log \
+        -X PUT "$TARGET_PATH" || true
+fi