Merge pull request #19 from onuralpszr/pre-commit

feat: ✨ introduce pyproject toml file and pre commit with CI

.github/workflows/main.yml

@@ -23,16 +23,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install ruff pytest
+          pip install pytest
           pip install git+https://github.com/ultralytics/ultralytics.git@embeddings
           pip install -e .
-          
-      - name: Lint with ruff
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          ruff --format=github --select=E9,F63,F7,F82 --target-version=py37  --line-length=120 .
-          # default set of ruff rules with GitHub Annotations
-          ruff --format=github --target-version=py37 --line-length=120 .
+
       - name: Test with pytest
         run: |
           pytest tests

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+default_language_version:
+    python: python3.8
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-added-large-files
+    -   id: check-toml
+    -   id: check-yaml
+        args:
+        -   --unsafe
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.7.0
+    hooks:
+    -   id: pyupgrade
+        args:
+        - --py3-plus
+        - --keep-runtime-typing
+-   repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.275
+    hooks:
+    -   id: ruff
+        args:
+        - --fix
+-   repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+    -   id: black
+ci:
+    autofix_commit_msg: "fix(pre_commit): 🎨 auto format pre-commit hooks"
+    autoupdate_commit_msg: "fix(pre_commit): ⬆ pre_commit autoupdate"

MANIFEST.in

@@ -1 +1 @@
-include requirements.txt
+include requirements.txt

README.md

@@ -1,6 +1,6 @@
 # YOLOExplorer
 
-Explore, manipulate and iterate on Computer Vision datasets with precision using simple APIs. 
+Explore, manipulate and iterate on Computer Vision datasets with precision using simple APIs.
 Supports SQL filters, vector similarity search, native interface with Pandas and more.
 
 
@@ -111,12 +111,12 @@ coco_exp.remove_imgs([100,120,300..n]) # Removes images at the given ids.
 <b>Adding data</b><br/>
 For adding data from another dataset, you need an explorer object of that dataset with embeddings built. You can then pass that object along with the ids of the imgs that you'd like to add from that dataset.
 ```
-coco_exp.add_imgs(exp, idxs) # 
+coco_exp.add_imgs(exp, idxs) #
 ```
 Note: You can use SQL querying and/or similarity searches to get the desired ids from the datasets.
 
 <b>Persisting the Table: Create new dataset and start training</b><br/>
-After making the desired changes, you can persist the table to create the new dataset. 
+After making the desired changes, you can persist the table to create the new dataset.
 ```
 coco_exp.persist()
 ```
@@ -175,4 +175,3 @@ Pre-filtering will enable powerful queries like - "Show me images similar to <IM
 Notes:
 * The API will have some minor changes going from dev to minor release
 * For all practical purposes the ids are same as row number and is reset after every addition or removal
-

frontend/dashboard.py

@@ -9,20 +9,20 @@
     default_data_yaml = "coco128.yaml"
     st.text(f"Dataset: {default_data_yaml}")
     if default_data_yaml:
-        if 'data_yaml' not in st.session_state:
-            st.session_state['data_yaml'] = default_data_yaml
-        if 'session' not in st.session_state:
-            st.session_state['session'] = Explorer(default_data_yaml, model=None)
-            dataset_info = st.session_state['session'].dataset_info
+        if "data_yaml" not in st.session_state:
+            st.session_state["data_yaml"] = default_data_yaml
+        if "session" not in st.session_state:
+            st.session_state["session"] = Explorer(default_data_yaml, model=None)
+            dataset_info = st.session_state["session"].dataset_info
 
             st.write("**Dataset Information :**")
 
             for dataset_type in ["train", "val", "tets"]:
                 st.text(f"{dataset_type.capitalize()} paths :")
-                if isinstance(dataset_info['train'], str):
-                    st.text(dataset_info['train'])
-                elif isinstance(dataset_info['train', List]):
-                    for p in dataset_info['train']:
+                if isinstance(dataset_info["train"], str):
+                    st.text(dataset_info["train"])
+                elif isinstance(dataset_info["train", List]):
+                    for p in dataset_info["train"]:
                         st.text(p)
 
 st.write("**Dataset Visualization**")
@@ -45,5 +45,3 @@
         train_image_paths = [img_url] * int(num_samples_show)
         # st.image(train_image_paths, use_column_width="always")
         clicked = image_select("Training Samples", images=train_image_paths)
-
-

pyproject.toml

@@ -0,0 +1,60 @@
+[tool.ruff]
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E9", "F63", "F7", "F82"]
+ignore = []
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+per-file-ignores = {}
+line-length = 120
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.8
+target-version = "py38"
+
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''

requirements.txt

@@ -3,4 +3,4 @@ duckdb
 scikit-learn
 ultralytics@git+https://github.com/ultralytics/ultralytics.git@embeddings
 # streamlit # To run dashboard
-# streamlit-image-select # For image gallery
+# streamlit-image-select # For image gallery

setup.py

@@ -6,14 +6,15 @@
 # Settings
 FILE = Path(__file__).resolve()
 PARENT = FILE.parent  # root directory
-README = (PARENT / 'README.md').read_text(encoding='utf-8')
+README = (PARENT / "README.md").read_text(encoding="utf-8")
 REQUIREMENTS = [
-"lancedb",
-"duckdb",
-"scikit-learn",
-"ultralytics@git+https://github.com/ultralytics/ultralytics.git@embeddings"
+    "lancedb",
+    "duckdb",
+    "scikit-learn",
+    "ultralytics@git+https://github.com/ultralytics/ultralytics.git@embeddings",
 ]
 
+
 def get_version():
     return "0.0.1.dev1"
 

tests/test_explorer.py

@@ -5,12 +5,8 @@ class TestExplorer:
     def test_embeddings_creation(self):
         coco_exp = Explorer("coco8.yaml")
         coco_exp.build_embeddings(force=True)
-        assert (
-            coco_exp.table_name == "coco8.yaml"
-        ), "the table name should be coco8.yaml"
-        assert (
-            len(coco_exp.table) == 4
-        ), "the length of the embeddings table should be 8"
+        assert coco_exp.table_name == "coco8.yaml", "the table name should be coco8.yaml"
+        assert len(coco_exp.table) == 4, "the length of the embeddings table should be 8"
 
     def test_sim_idx(self):
         coco_exp = Explorer("coco8.yaml")
@@ -32,13 +28,9 @@ def test_operations(self):
         coco_exp.log_status()
         coco_exp.remove_imgs([0, 1])
         coco_exp.remove_imgs([0])
-        assert (
-            len(coco_exp.table.to_arrow()) == 1
-        ), "the length of the embeddings table should be 1"
+        assert len(coco_exp.table.to_arrow()) == 1, "the length of the embeddings table should be 1"
         coco_exp.persist()
-        assert (
-            len(coco_exp.table.to_arrow()) == 1
-        ), "the length of the embeddings table should be 1"
+        assert len(coco_exp.table.to_arrow()) == 1, "the length of the embeddings table should be 1"
 
     def test_add_imgs(self):
         coco_exp = Explorer("coco8.yaml")
@@ -47,9 +39,7 @@ def test_add_imgs(self):
         coco128_exp.build_embeddings()
 
         coco_exp.add_imgs(coco128_exp, [i for i in range(4)])
-        assert (
-            len(coco_exp.table) == 8
-        ), "the length of the embeddings table should be 8"
+        assert len(coco_exp.table) == 8, "the length of the embeddings table should be 8"
 
     def test_sql(self):
         coco_exp = Explorer("coco8.yaml")

yoloexplorer/__init__.py

@@ -1,3 +1,3 @@
-from .explorer import Explorer 
+from .explorer import Explorer
 
 __all__ = ["Explorer"]

yoloexplorer/dataset.py

@@ -32,9 +32,7 @@ def get_relative_path(path1, path2):
 
 class Dataset(YOLODataset):
     def __init__(self, *args, data=None, **kwargs):
-        super().__init__(
-            *args, data=data, use_segments=False, use_keypoints=False, **kwargs
-        )
+        super().__init__(*args, data=data, use_segments=False, use_keypoints=False, **kwargs)
 
     # NOTE: Load the image directly without any resize operations.
     def load_image(self, i):