Add gpt-4o-mini to smoke test github workflow and make smoke test judge more reliable

.github/workflows/smoke.yaml

@@ -59,7 +59,7 @@ jobs:
 
           echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
 
-  gpt-4o-2024-05-13:
+  gpt-4o-2024-08-06:
     needs: check-label
     if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
     runs-on: ubuntu-22.04
@@ -81,14 +81,14 @@ jobs:
           go-version: "1.21"
       - env:
           OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
-          GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
-        name: Run smoke test for gpt-4o-2024-05-13
+          GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-08-06
+        name: Run smoke test for gpt-4o-2024-08-06
         run: |
-          echo "Running smoke test for model gpt-4o-2024-05-13"
+          echo "Running smoke test for model gpt-4o-2024-08-06"
           export PATH="$(pwd)/bin:${PATH}"
           make smoke
 
-  gpt-4-turbo-2024-04-09:
+  gpt-4o-mini-2024-07-18:
     needs: check-label
     if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
     runs-on: ubuntu-22.04
@@ -110,10 +110,10 @@ jobs:
           go-version: "1.21"
       - env:
           OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
-          GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
-        name: Run smoke test for gpt-4-turbo-2024-04-09
+          GPTSCRIPT_DEFAULT_MODEL: gpt-4o-mini-2024-07-18
+        name: Run smoke test for gpt-4o-mini-2024-07-18
         run: |
-          echo "Running smoke test for model gpt-4-turbo-2024-04-09"
+          echo "Running smoke test for model gpt-4o-mini-2024-07-18"
           export PATH="$(pwd)/bin:${PATH}"
           make smoke
 

pkg/tests/judge/judge.go

@@ -40,6 +40,8 @@ After making a determination, respond with a JSON object that conforms to the fo
   ]
 }
 
+If you determine actual and expected are not equivalent, include a diff of the parts of actual and expected that are not equivalent in the reasoning field of your response.
+
 Your responses are concise and include only the json object described above.
 `
 
@@ -84,10 +86,10 @@ func New[T any](client *openai.Client) (*Judge[T], error) {
 }
 
 func (j *Judge[T]) Equal(ctx context.Context, expected, actual T, criteria string) (equal bool, reasoning string, err error) {
-	comparisonJSON, err := json.MarshalIndent(&comparison[T]{
+	comparisonJSON, err := json.Marshal(&comparison[T]{
 		Expected: expected,
 		Actual:   actual,
-	}, "", "    ")
+	})
 	if err != nil {
 		return false, "", fmt.Errorf("failed to marshal judge testcase JSON: %w", err)
 	}

pkg/tests/smoke/smoke_test.go

@@ -82,8 +82,8 @@ func TestSmoke(t *testing.T) {
 				expectedEvents,
 				actualEvents,
 				`
-- disregard differences in timestamps, generated IDs, natural language verbiage, and event order
-- omit callProgress events from the comparison
+- disregard differences in event order, timestamps, generated IDs, and natural language verbiage, grammar, and punctuation
+- compare events with matching event types
 - the overall stream of events and set of tools called should roughly match
 - arguments passed in tool calls should be roughly the same
 - the final callFinish event should be semantically similar
@@ -175,6 +175,11 @@ func getActualEvents(t *testing.T, eventsFile string) []event {
 
 		var e event
 		require.NoError(t, json.Unmarshal([]byte(line), &e))
+
+		if e.Type == runner.EventTypeCallProgress {
+			continue
+		}
+
 		events = append(events, e)
 	}