Merge branch 'main' into kevin

SmartManoj · Sep 6, 2024 · 3543e1c · 3543e1c
2 parents f8d9b1a + 0f118df
commit 3543e1c
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 20 deletions.
diff --git a/...dules/usage/architecture/architecture.mdx → docs/modules/usage/architecture/backend.mdx b/...dules/usage/architecture/architecture.mdx → docs/modules/usage/architecture/backend.mdx
diff --git a/docs/sidebars.ts b/docs/sidebars.ts
@@ -17,30 +17,58 @@ const sidebars: SidebarsConfig = {
   }, {
     type: 'category',
     label: 'How-to Guides',
-      items: [{
-        type: 'doc',
-        id: 'usage/how-to/cli-mode',
-      }, {
-        type: 'doc',
-        id: 'usage/how-to/headless-mode',
-      }, {
-        type: 'doc',
-        id: 'usage/how-to/custom-sandbox-guide',
-      }, {
-        type: 'doc',
-        id: 'usage/how-to/evaluation-harness',
-      }, {
-        type: 'doc',
-        id: 'usage/how-to/openshift-example',
-      }]
+    items: [{
+      type: 'doc',
+      id: 'usage/how-to/cli-mode',
+    }, {
+      type: 'doc',
+      id: 'usage/how-to/headless-mode',
+    }, {
+      type: 'doc',
+      id: 'usage/how-to/custom-sandbox-guide',
+    }, {
+      type: 'doc',
+      id: 'usage/how-to/evaluation-harness',
+    }, {
+      type: 'doc',
+      id: 'usage/how-to/openshift-example',
+    }]
   }, {
-    type: 'doc',
+    type: 'category',
     label: 'LLMs',
-    id: 'usage/llms/llms',
+    items: [{
+      type: 'doc',
+      label: 'Overview',
+      id: 'usage/llms/llms',
+    }, {
+      type: 'doc',
+      label: 'OpenAI',
+      id: 'usage/llms/openai-llms',
+    }, {
+      type: 'doc',
+      label: 'Azure',
+      id: 'usage/llms/azure-llms',
+    }, {
+      type: 'doc',
+      label: 'Google',
+      id: 'usage/llms/google-llms',
+    }, {
+      type: 'doc',
+      label: 'Local/ollama',
+      id: 'usage/llms/local-llms',
+    }],
   }, {
-    type: 'doc',
+    type: 'category',
     label: 'Architecture',
-    id: 'usage/architecture/architecture',
+    items: [{
+      type: 'doc',
+      label: 'Backend',
+      id: 'usage/architecture/backend',
+    }, {
+      type: 'doc',
+      label: 'Runtime',
+      id: 'usage/architecture/runtime',
+    }],
   }, {
     type: 'doc',
     label: 'About',

diff --git a/evaluation/swe_bench/run_infer.py b/evaluation/swe_bench/run_infer.py
@@ -456,6 +456,12 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
     output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl')
     instances = prepare_dataset(swe_bench_tests, output_file, args.eval_n_limit)
 
+    if not isinstance(
+        instances['PASS_TO_PASS'][instances['PASS_TO_PASS'].index[0]], str
+    ):
+        for col in ['PASS_TO_PASS', 'FAIL_TO_PASS']:
+            instances[col] = instances[col].apply(lambda x: str(list(x)))
+
     run_evaluation(
         instances, metadata, output_file, args.eval_num_workers, process_instance
     )