feat: add autoflake to pre commit config

.pre-commit-config.yaml

@@ -9,6 +9,11 @@ repos:
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-ast
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+      - id: autoflake
+        args: [--remove-all-unused-imports, --in-place]
   - repo: 'https://github.com/psf/black'
     rev: 23.3.0
     hooks:

backend/indexer/indexer.py

@@ -1,4 +1,3 @@
-import os
 import tempfile
 from concurrent.futures import Executor
 from typing import Dict, List, Optional

backend/modules/dataloaders/loader.py

@@ -105,7 +105,6 @@ def load_filtered_data(
         Returns:
             Iterator[List[LoadedDataPoint]]: An iterator of list of loaded data points.
         """
-        pass
 
 
 def get_loader_for_data_source(type, *args, **kwargs) -> BaseDataLoader:

backend/modules/metadata_store/prisma_store.py

@@ -1,18 +1,14 @@
 import asyncio
 import json
-import os
 import random
-import shutil
 import string
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from fastapi import HTTPException
 from prisma import Prisma
-from prisma.errors import RecordNotFoundError, UniqueViolationError
 
 from backend.logger import logger
 from backend.modules.metadata_store.base import BaseMetadataStore
-from backend.settings import settings
 from backend.types import (
     AssociateDataSourceWithCollection,
     AssociatedDataSources,
@@ -25,7 +21,6 @@
     DataSource,
     RagApplication,
 )
-from backend.utils import TRUEFOUNDRY_CLIENT
 
 if TYPE_CHECKING:
     # TODO (chiragjn): Can we import these safely even if the prisma client might not be generated yet?

backend/modules/parsers/multi_modal_parser.py

@@ -10,7 +10,6 @@
 from langchain.docstore.document import Document
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.messages import HumanMessage
-from PIL import Image
 
 from backend.logger import logger
 from backend.modules.model_gateway.model_gateway import model_gateway

backend/modules/parsers/parser.py

@@ -5,7 +5,6 @@
 from langchain.docstore.document import Document
 
 from backend.logger import logger
-from backend.types import LoadedDataPoint
 
 PARSER_REGISTRY = {}
 PARSER_REGISTRY_EXTENSIONS = defaultdict(list)
@@ -51,7 +50,6 @@ async def get_chunks(
         Returns:
             typing.List[Document]: A list of Document objects, each representing a chunk of the file.
         """
-        pass
 
 
 def get_parser_for_extension(

backend/modules/parsers/video_parser.py

@@ -1,4 +1,3 @@
-import json
 import os
 import tempfile
 from typing import Any, Dict, List
@@ -7,7 +6,6 @@
 from moviepy.video.io.VideoFileClip import VideoFileClip
 
 from backend.logger import logger
-from backend.modules.model_gateway.model_gateway import model_gateway
 from backend.modules.parsers.audio_parser import AudioParser
 from backend.modules.parsers.multi_modal_parser import MultiModalParser
 from backend.modules.parsers.parser import BaseParser

backend/requirements.txt

@@ -24,6 +24,7 @@ python-multipart==0.0.9
 opencv-python==4.9.0.80
 pillow==10.3.0
 aiofiles==24.1.0
+autoflake==2.3.1
 
 ### prisma
 prisma==0.13.1

backend/server/routers/data_source.py

@@ -1,9 +1,8 @@
 import os
 import shutil
-from contextlib import asynccontextmanager
 from urllib.parse import unquote
 
-from fastapi import APIRouter, Depends, FastAPI, HTTPException
+from fastapi import APIRouter
 from fastapi.responses import JSONResponse
 
 from backend.logger import logger

backend/server/routers/internal.py

@@ -5,7 +5,6 @@
 
 from fastapi import APIRouter, File, Form, HTTPException, Query, UploadFile
 from fastapi.responses import JSONResponse
-from truefoundry import ml
 from truefoundry.ml import DataDirectory
 
 from backend.logger import logger

backend/types.py

@@ -1,7 +1,7 @@
 import enum
 import uuid
 from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional
 
 from pydantic import (
     BaseModel,

deployment/indexer.py

@@ -1,7 +1,6 @@
 from truefoundry.deploy import (
     Build,
     DockerFileBuild,
-    Helm,
     Job,
     LocalSource,
     Manual,

deployment/unstructured_io.py

@@ -1,7 +1,3 @@
-import argparse
-import json
-import logging
-
 from truefoundry.deploy import Image, NodeSelector, Port, Resources, Service
 
 from deployment.config import UNSTRUCTURED_IO_SERVICE_NAME