Enable pre-commit for formatting and linting (#267)

* Enable pre-commit (lint, format, line endings, and remove trailing whitespace) and apply to code * Remove comments from lint tests * Update workflow and remove formatting * Remove auto-formatter.sh as no longer needed * Add action to run precommit on pushes * Insert change that should be fixed by precommit * link precommit and CI workflows * Fix inserted issue to run CI * Remove concurrency to see if CI is triggered * try to merge workflows as ci isnt triggering * Insert change that should be fixed by precommit * Update CI * fix trigger * fix precommit trigger * fix precommit permissions * Fix update trigger * Fix inserted issue * Clean workflow scripts * Insert whitespace to verify CI * [pre-commit.ci lite] apply automatic fixes * Update error message if precommit fails * Covert all code to Google coding style * Update push config in workflow * clear cache command * Remove comments in update CI and add reference to Developer Guide to point to pre-commit instructions * Remove check push failure workflow * Update commit cmd. * fix permission * Update commit cmd. * Remove token from update workflow * Automated coverage update --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
IntelLabs · Feb 21, 2025 · cf1f51d · cf1f51d
1 parent 195b3c6
commit cf1f51d
Show file tree

Hide file tree

Showing 122 changed files with 736 additions and 792 deletions.
diff --git a/.clang-format-ignore b/.clang-format-ignore
@@ -0,0 +1,2 @@
+./build/*
+./src/pmgd/*
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -0,0 +1,5 @@
+# Contributing to VDMS
+Thank you for even being interested in contributing to VDMS.
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether they involve new features, improved infrastructure, better documentation, or bug fixes.
+
+To learn how to contribute to VDMS, please follow the [Developer Guide](https://github.com/IntelLabs/vdms/wiki/Developer-Guide).
diff --git a/.github/coverage/python.develop.coverage_report.txt b/.github/coverage/python.develop.coverage_report.txt
@@ -1,6 +1,6 @@
 Name                                   Stmts   Miss  Cover   Missing
 --------------------------------------------------------------------
 /vdms/client/python/vdms/__init__.py       2      0   100%
-/vdms/client/python/vdms/vdms.py          99      2    98%   152, 167
+/vdms/client/python/vdms/vdms.py          95      2    98%   148, 163
 --------------------------------------------------------------------
-TOTAL                                    101      2    98%
+TOTAL                                     97      2    98%
diff --git a/.github/coverage/python.develop.coverage_value.txt b/.github/coverage/python.develop.coverage_value.txt
@@ -1 +1 @@
-98.02
+97.94
diff --git a/.github/scripts/auto-formatter.sh b/.github/scripts/auto-formatter.sh
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -19,8 +19,18 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  PreCommit:
+    # Apply formatting and linting
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+    uses: ./.github/workflows/_precommit.yml
+    secrets: inherit
+
   Get-Envs:
     name: Get Environment vars
+    needs: [PreCommit]
     runs-on: ubuntu-latest
     outputs:
       CHECKIN_DOCKERFILE: .github/scripts/docker-compose.yml
@@ -60,9 +70,9 @@ jobs:
   Update:
     permissions:
         contents: write
+        pull-requests: write
     # name: Lint & Update Reports
     needs: [Testing, Results]
-    if: ${{ always() }} 
     uses: ./.github/workflows/_CI_update.yml
     with:
         coverage_value_updated: ${{ needs.Testing.outputs.coverage_value_updated }}

diff --git a/.github/workflows/_CI_coverage.yml b/.github/workflows/_CI_coverage.yml
@@ -73,7 +73,7 @@ jobs:
       target_cpp_value: ${{ steps.target_values.outputs.cpp }}
       target_py_value: ${{ steps.target_values.outputs.py }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.1.7
         with:
           ref: ${{ env.TARGET_BRANCH_REF }}
           fetch-depth: 0
@@ -105,7 +105,7 @@ jobs:
       coverage_value_updated: ${{ steps.report_coverage.outputs.cov_changed }}
       coverage_test_status: ${{ steps.report_coverage.outputs.coverage_test_status }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.1.7
         with:
           submodules: true
 
@@ -273,7 +273,7 @@ jobs:
           echo "coverage_test_status=${test_status}" >> $GITHUB_OUTPUT
 
       - name: Upload New coverage results
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b  # v4.3.4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08  # v4.6.0
         with:
           name: coverage_artifact
           path: .github/coverage/*.new.*.txt

diff --git a/.github/workflows/_CI_update.yml b/.github/workflows/_CI_update.yml
@@ -16,34 +16,30 @@ jobs:
     commit_job:
         permissions:
           contents: write
-        name: Commit Code Updates 
+          pull-requests: write
+        name: Commit Code Updates
         env:
-          COMMIT_MSG: "Automated updates: Format and/or coverage"
+          COMMIT_MSG: "Automated coverage update"
           DOCKER_ARTIFACT_DIR: "Docker_artifacts"
         runs-on: ubuntu-latest
         steps:
           # Checkout code doesn't persist across jobs
           - name: Checkout Source Branch
-            uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+            uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
             with:
               fetch-depth: 0
               ref: ${{ github.event.pull_request.head.ref }}
               repository: ${{ github.event.pull_request.head.repo.full_name }}
-              token: ${{ secrets.FACELESS_TOKEN || github.token }}
 
           - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }}
 
           - name: Retrieve Current Coverage Files
             if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed'
-            uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+            uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16  # v4.1.8
             with:
               name: coverage_artifact
               path: .github/coverage/
 
-          - name: Format C++ Code (clang-format), Python (black code), and apply dos2unix
-            run: |
-              ./.github/scripts/auto-formatter.sh
-
           - name: Update coverage reports with latest coverage
             # Change latest coverage as develop (future target)
             if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed'
@@ -67,9 +63,8 @@ jobs:
               echo "changes=$changes" >> $GITHUB_OUTPUT
 
               if [ "$changes" != "Empty" ]; then
-                git config user.name ${{ secrets.FACELESS_NAME }}
-                git config user.email ${{ secrets.FACELESS_NAME }}@intel.com
-                git remote set-url origin https://x-access-token:${{ secrets.FACELESS_TOKEN }}@github.com/${{ github.event.pull_request.head.repo.full_name }}
+                git config user.name 'github-actions[bot]'
+                git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
                 git add .github/coverage/*
                 git commit -am "${{ env.COMMIT_MSG }}"
                 git push
@@ -81,6 +76,6 @@ jobs:
               FILE_CHANGES: ${{ steps.update_commit.outputs.changes }}
             run: |
               if [ "${{ env.FILE_CHANGES }}" != "Empty" ]; then
-                echo "Please provide sys-vdms write access to fork (if applicable)."
+                echo "Error issuing commit. Coverage reports were NOT updated"
                 exit 1
               fi
diff --git a/.github/workflows/_precommit.yml b/.github/workflows/_precommit.yml
@@ -0,0 +1,32 @@
+name: Pre-commit
+
+on:
+  workflow_call:
+
+
+permissions: {}
+
+jobs:
+  Pre-Commit:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.1.7
+      with:
+        submodules: false
+    - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38  # v5.4.0
+      with:
+        python-version: 3.x
+    - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1
+    - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43  # v1.1.0
+      if: always()
+    - if: failure()
+      run: |
+        echo 'Pre-commit failed. CI will restart or please fix requested issues.'
+        pre-commit clean
+        exit 1
+    - run: pre-commit clean
+
diff --git a/.gitmodules b/.gitmodules
@@ -2,4 +2,3 @@
 	path = src/pmgd
 	url = https://github.com/omp87/pmgd.git
 	branch = develop
-
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,32 @@
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.6
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [
+          "--fix",
+          "--ignore=E402",
+          "--extend-exclude=client/python/vdms/queryMessage_pb2.py"
+        ]
+      # Run the formatter.
+      - id: ruff-format
+        args: [
+          "--exclude=client/python/vdms/queryMessage_pb2.py"
+        ]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: mixed-line-ending
+        args: ["--fix=auto"]
+      - id: trailing-whitespace
+  -   repo: https://github.com/pre-commit/mirrors-clang-format
+      rev: v19.1.7
+      hooks:
+      -   id: clang-format
+          types: [c, c++]
+          args: [
+            "--style=Google",
+            "-i"
+          ]
diff --git a/README.md b/README.md
@@ -57,3 +57,11 @@ Industrial and Applications @ VLDB 2021 | [Paper](http://vldb.org/pvldb/vol14/p3
 2nd USENIX Workshop @ HotEdge 2019 | [Paper](https://www.usenix.org/system/files/hotedge19-paper-altarawneh.pdf), [Presentation](https://www.usenix.org/sites/default/files/conference/protected-files/hotedge19_slides_altarawneh.pdf) | VDMS in Edge-to-cloud architecture for video streaming application
 Learning Systems @ NIPS 2018 | [Paper](https://export.arxiv.org/abs/1810.11832), [Cite](https://dblp.uni-trier.de/rec/bibtex/journals/corr/abs-1810-11832) | Systems for Machine Learning [Workshop](http://learningsys.org/nips18/cfp.html) @ NIPS
 HotStorage @ ATC 2017 | [Paper](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Presentation](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Cite](https://www.usenix.org/biblio/export/bibtex/203374)| Positioning Paper at USENIX ATC 2017 Workshop
+
+
+## Contributing
+
+Thank you for your interest in contributing to VDMS.
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether they involve new features, improved infrastructure, better documentation, or bug fixes.
+
+To learn how to contribute to VDMS, please follow the [Developer Guide](https://github.com/IntelLabs/vdms/wiki/Developer-Guide).
diff --git a/client/cpp/BoundingBoxQueryParser.h b/client/cpp/BoundingBoxQueryParser.h
@@ -1,20 +1,19 @@
 #include "CSVParserUtil.h"
 namespace VDMS {
 class BoundingBoxQueryParser : public CSVParserUtil {
-private:
+ private:
   vector<string> rectangleKeys{"x", "y", "w", "h"};
   void parseRectangle(string row, string queryType, Json::Value &aquery);
 
-public:
+ public:
   VDMS::Response ParseAddBoundingBox(vector<string> row, vector<string> cols);
   // VDMS::Response ParseUpdateBoundingBox(vector<string> row, vector<string>&
   // cols);
 };
-}; // namespace VDMS
+};  // namespace VDMS
 
-VDMS::Response
-VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector<string> row,
-                                                  vector<string> columnNames) {
+VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(
+    vector<string> row, vector<string> columnNames) {
   if (row.empty() || row[0].empty()) {
     throw "please provide rectangle details";
   }
@@ -49,8 +48,7 @@ VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector<string> row,
     }
   }
 
-  if (cons)
-    allquery.append(aqueryf);
+  if (cons) allquery.append(aqueryf);
 
   allquery.append(aquery);
   // std::cout<<allquery<<std::endl;

diff --git a/client/cpp/CSVParser.h b/client/cpp/CSVParser.h
@@ -1,26 +1,27 @@
-#include "CSVParserUtil.h"
-#include "rapidcsv.h"
 #include <fstream>
 #include <iostream>
 #include <mutex>
 #include <string>
 #include <thread>
 #include <vector>
 
+#include "CSVParserUtil.h"
+#include "rapidcsv.h"
+
 namespace VDMS {
 class CSVParser {
-public:
+ public:
   CSVParser(std::string filename, size_t num_threads, std::string server,
             int port)
-      : m_filename(filename), m_num_threads(num_threads), vdms_server(server),
+      : m_filename(filename),
+        m_num_threads(num_threads),
+        vdms_server(server),
         vdms_port(port) {}
   ~CSVParser() {}
 
-  std::vector<VDMS::Response>
-  parse_csv_lines(const std::string &filename, int start_line, int end_line,
-                  std::vector<VDMS::Response> &local_results,
-                  const size_t thread_id) {
-
+  std::vector<VDMS::Response> parse_csv_lines(
+      const std::string &filename, int start_line, int end_line,
+      std::vector<VDMS::Response> &local_results, const size_t thread_id) {
     rapidcsv::Document csv(filename);
     std::vector<std::string> columnNames = csv.GetColumnNames();
     VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames,
@@ -58,7 +59,7 @@ class CSVParser {
     std::mutex mutex;
     std::vector<std::thread> threads;
     std::vector<std::vector<VDMS::Response>> all_local_results(m_num_threads);
-    std::vector<VDMS::Response> all_results; // Local vector for each thread
+    std::vector<VDMS::Response> all_results;  // Local vector for each thread
     all_results.reserve(num_lines);
 
     for (size_t i = 0; i < m_num_threads; i++) {
@@ -76,7 +77,6 @@ class CSVParser {
     }
     size_t allResultsSizeBefore = all_results.size();
     for (const auto &local_results : all_local_results) {
-
       // Extend the size of all_results to accommodate local_results
       all_results.resize(all_results.size() + local_results.size());
 
@@ -89,10 +89,10 @@ class CSVParser {
     return all_results;
   }
 
-private:
+ private:
   std::string m_filename;
   size_t m_num_threads;
   std::string vdms_server;
   int vdms_port;
 };
-}; // namespace VDMS
+};  // namespace VDMS
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,3 @@
		path = src/pmgd
		url = https://github.com/omp87/pmgd.git
		branch = develop