Fix gitlab integration issues (#859)

* fix

* bump

* fix repo_slug parsing

* update lock

* undo editing test file

* add code2prompt gitignore

* fix code2prompt

patchwork/common/client/llm/google.py

@@ -21,7 +21,7 @@
 from typing_extensions import Any, Dict, Iterable, List, Optional, Union
 
 from patchwork.common.client.llm.protocol import NOT_GIVEN, LlmClient, NotGiven
-from patchwork.common.client.llm.utils import base_model_to_schema, json_schema_to_model
+from patchwork.common.client.llm.utils import json_schema_to_model
 
 
 @functools.lru_cache
@@ -52,20 +52,20 @@ def is_model_supported(self, model: str) -> bool:
         return model in self.get_models()
 
     def chat_completion(
-            self,
-            messages: Iterable[ChatCompletionMessageParam],
-            model: str,
-            frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN,
-            logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN,
-            logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
-            max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
-            n: Optional[int] | NotGiven = NOT_GIVEN,
-            presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
-            response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
-            stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
-            temperature: Optional[float] | NotGiven = NOT_GIVEN,
-            top_logprobs: Optional[int] | NotGiven = NOT_GIVEN,
-            top_p: Optional[float] | NotGiven = NOT_GIVEN,
+        self,
+        messages: Iterable[ChatCompletionMessageParam],
+        model: str,
+        frequency_penalty: Optional[float] | NotGiven = NOT_GIVEN,
+        logit_bias: Optional[Dict[str, int]] | NotGiven = NOT_GIVEN,
+        logprobs: Optional[bool] | NotGiven = NOT_GIVEN,
+        max_tokens: Optional[int] | NotGiven = NOT_GIVEN,
+        n: Optional[int] | NotGiven = NOT_GIVEN,
+        presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
+        response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
+        stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
+        temperature: Optional[float] | NotGiven = NOT_GIVEN,
+        top_logprobs: Optional[int] | NotGiven = NOT_GIVEN,
+        top_p: Optional[float] | NotGiven = NOT_GIVEN,
     ) -> ChatCompletion:
         generation_dict = dict(
             stop_sequences=[stop] if isinstance(stop, str) else stop,

patchwork/common/client/scm.py

@@ -12,23 +12,17 @@
 from github import Auth, Consts, Github, GithubException, PullRequest
 from gitlab import Gitlab, GitlabAuthenticationError, GitlabError
 from gitlab.v4.objects import ProjectMergeRequest
+from giturlparse import GitUrlParsed, parse
 from typing_extensions import Protocol, TypedDict
 
 from patchwork.logger import logger
 
 
 def get_slug_from_remote_url(remote_url: str) -> str:
-    # TODO: consider using https://github.com/nephila/giturlparse instead
-    if remote_url.startswith("git@"):
-        # ssh
-        _, _, potential_slug = remote_url.partition(":")
-    else:
-        potential_slug = "/".join(remote_url.split("/")[-2:])
-
-    if potential_slug.endswith(".git"):
-        potential_slug = potential_slug[:-4]
-
-    return potential_slug
+    parsed_repo: GitUrlParsed = parse(remote_url)
+    parts = [parsed_repo.owner, *parsed_repo.groups, parsed_repo.name]
+    slug = "/".join(parts)
+    return slug
 
 
 @define
@@ -567,10 +561,10 @@ def find_prs(
         for instance in itertools.product(*kwargs_list.values()):
             kwargs = dict(((key, value) for key, value in zip(keys, instance) if value is not None))
             mrs_instance = project.mergerequests.list(**kwargs)
-            page_list.append(mrs_instance)
+            page_list.append(list(mrs_instance))
 
         rv_list = []
-        for mr in itertools.islice(itertools.chain(mrs), limit):
+        for mr in itertools.islice(itertools.chain(*page_list), limit):
             rv_list.append(GitlabMergeRequest(mr))
 
         return rv_list

patchwork/patchflows/PRReview/PRReview.py

@@ -100,7 +100,9 @@ def run(self) -> dict:
 
         header = ""
         if self.verbosity > _SUMMARY_LEVEL[_SHORT]:
-            filtered_summaries = [str(summary["commit_message"]) for summary in summaries if summary.get("commit_message")]
+            filtered_summaries = [
+                str(summary["commit_message"]) for summary in summaries if summary.get("commit_message")
+            ]
             self.inputs["prompt_id"] = "diffreview_summary"
             self.inputs["prompt_values"] = [{"diffreviews": "\n".join(filtered_summaries)}]
 

patchwork/step.py

@@ -13,6 +13,7 @@ class StepStatus(Flag):
     COMPLETED = auto()
     FAILED = auto()
     SKIPPED = auto()
+    WARNING = auto()
 
     def __str__(self):
         return self.name.lower()

patchwork/steps/CallCode2Prompt/CallCode2Prompt.py

@@ -1,14 +1,72 @@
+from __future__ import annotations
 import os
 import subprocess
+import tempfile
 from pathlib import Path
+from typing_extensions import Buffer
 
 from patchwork.logger import logger
 from patchwork.step import Step, StepStatus
 
 FOLDER_PATH = "folder_path"
 
 
+
 class CallCode2Prompt(Step):
+    __CUSTOM_GIT_IGNORE = [
+        # Compiled source
+        "*.com",
+        "*.class",
+        "*.dll",
+        "*.exe",
+        "*.o",
+        "*.so",
+        # Archives
+        "*.7z",
+        "*.dmg",
+        "*.gz",
+        "*.iso",
+        "*.jar",
+        "*.rar",
+        "*.tar",
+        "*.zip",
+        # data stores
+        "*.log",
+        "*.sql",
+        "*.sqlite",
+        "*.yml",
+        "*.json",
+        # OS generated files
+        ".DS_Store",
+        ".DS_Store?",
+        "._*",
+        ".Spotlight-V100",
+        ".Trashes",
+        "ehthumbs.db",
+        "Thumbs.db",
+        # Test files
+        "test",
+        "tests",
+        "Test",
+        "Tests",
+        "testing",
+        "Testing",
+        "*.test.js",
+        "*.test.ts",
+        "*.test.jsx",
+        "*.test.tsx",
+        "*.spec.js",
+        "*.spec.ts",
+        "*.spec.jsx",
+        "*.spec.tsx",
+        # IDE files
+        ".idea",
+        ".vscode",
+        "*.suo",
+        "*.ntvs*",
+        "*.njsproj",
+        "*.sln",
+    ]
     required_keys = {FOLDER_PATH}
 
     def __init__(self, inputs: dict):
@@ -29,21 +87,33 @@ def __init__(self, inputs: dict):
             with open(self.code_file_path, "a") as file:
                 pass  # No need to write anything, just create the file if it doesn't exist
 
-    def run(self) -> dict:
+    def __get_cmd_args(self, git_ignore_temp_fp) -> list[str]:
         cmd = [
             "code2prompt",
             "--path",
             self.folder_path,
         ]
 
+        repo_gitignore = self.folder_path / ".gitignore"
+        custom_gitignore_text = "\n".join(self.__CUSTOM_GIT_IGNORE)
+        if repo_gitignore.is_file():
+            custom_gitignore_text = custom_gitignore_text + "\n" + repo_gitignore.read_text()
+        git_ignore_temp_fp.write(custom_gitignore_text)
+        cmd.extend(["--gitignore", git_ignore_temp_fp.name])
+
         if self.filter is not None:
             cmd.extend(["--filter", self.filter])
 
         if self.suppress_comments:
             cmd.append("--suppress-comments")
 
+        return cmd
+
+    def run(self) -> dict:
         try:
-            p = subprocess.run(cmd, capture_output=True, text=True, check=True)
+            with tempfile.NamedTemporaryFile(mode="w+") as temp_file:
+                cmd = self.__get_cmd_args(temp_file)
+                p = subprocess.run(cmd, capture_output=True, text=True, check=True)
             prompt_content_md = p.stdout
         except subprocess.CalledProcessError as e:
             self.set_status(StepStatus.FAILED, f"Subprocess failed: {e}")

patchwork/steps/GetTypescriptTypeInfo/GetTypescriptTypeInfo.py

@@ -3,8 +3,10 @@
 from pathlib import Path
 
 from patchwork.step import Step
-from patchwork.steps.GetTypescriptTypeInfo.typed import GetTypescriptTypeInfoInputs, GetTypescriptTypeInfoOutputs
-
+from patchwork.steps.GetTypescriptTypeInfo.typed import (
+    GetTypescriptTypeInfoInputs,
+    GetTypescriptTypeInfoOutputs,
+)
 
 _DEFAULT_TS_FILE = Path(__file__).parent / "get_type_info.ts"
 

patchwork/steps/__init__.py

@@ -25,6 +25,9 @@
     GenerateCodeRepositoryEmbeddings,
 )
 from patchwork.steps.GenerateEmbeddings.GenerateEmbeddings import GenerateEmbeddings
+from patchwork.steps.GetTypescriptTypeInfo.GetTypescriptTypeInfo import (
+    GetTypescriptTypeInfo,
+)
 from patchwork.steps.JoinList.JoinList import JoinList
 from patchwork.steps.JoinListPB.JoinListPB import JoinListPB
 from patchwork.steps.LLM.LLM import LLM
@@ -46,7 +49,6 @@
 from patchwork.steps.SimplifiedLLMOnce.SimplifiedLLMOnce import SimplifiedLLMOnce
 from patchwork.steps.SimplifiedLLMOncePB.SimplifiedLLMOncePB import SimplifiedLLMOncePB
 from patchwork.steps.SlackMessage.SlackMessage import SlackMessage
-from patchwork.steps.GetTypescriptTypeInfo.GetTypescriptTypeInfo import GetTypescriptTypeInfo
 
 __all__ = [
     "AnalyzeImpact",