From b70f8643038923acb012793b5f12d41f3ff9d4d7 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 11 Oct 2024 15:56:08 -0400 Subject: [PATCH 1/3] Update `failure-finish-reason-safety` with Vertex responses --- ...streaming-failure-finish-reason-safety.txt | 2 +- ...ilure-finish-reason-safety-no-content.json | 79 +++++++++---------- .../unary-failure-finish-reason-safety.json | 53 ++++++------- 3 files changed, 64 insertions(+), 70 deletions(-) diff --git a/mock-responses/streaming-failure-finish-reason-safety.txt b/mock-responses/streaming-failure-finish-reason-safety.txt index 05e0936..b318cba 100644 --- a/mock-responses/streaming-failure-finish-reason-safety.txt +++ b/mock-responses/streaming-failure-finish-reason-safety.txt @@ -1,2 +1,2 @@ -data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "HIGH"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}],"promptFeedback": {"safetyRatings": [{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_HARASSMENT","probability": "NEGLIGIBLE"},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE"}]}} +data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.40429688,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.18554688},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.15820313,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.030639648},{"category": "HARM_CATEGORY_HARASSMENT","probability": "LOW","blocked": true,"probabilityScore": 0.55078125,"severity": "HARM_SEVERITY_LOW","severityScore": 0.24804688},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.17675781,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.17871094}]}],"usageMetadata": {"promptTokenCount": 10,"candidatesTokenCount": 66,"totalTokenCount": 76},"modelVersion": "gemini-1.5-flash"} diff --git a/mock-responses/unary-failure-finish-reason-safety-no-content.json b/mock-responses/unary-failure-finish-reason-safety-no-content.json index 03958d4..4d0e1f7 100644 --- a/mock-responses/unary-failure-finish-reason-safety-no-content.json +++ b/mock-responses/unary-failure-finish-reason-safety-no-content.json @@ -1,46 +1,43 @@ { - "candidates": [ + "candidates": [ + { + "finishReason": "SAFETY", + "safetyRatings": [ { - "finishReason": "SAFETY", - "index": 0, - "safetyRatings": [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "HIGH" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE" - } - ] + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.3984375, + "severity": "HARM_SEVERITY_LOW", + "severityScore": 0.21582031 + }, + { + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.14941406, + "severity": "HARM_SEVERITY_NEGLIGIBLE", + "severityScore": 0.02331543 + }, + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "LOW", + "blocked": true, + "probabilityScore": 0.61328125, + "severity": "HARM_SEVERITY_LOW", + "severityScore": 0.31835938 + }, + { + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.13476563, + "severity": "HARM_SEVERITY_NEGLIGIBLE", + "severityScore": 0.12109375 } - ], - "promptFeedback": { - "safetyRatings": [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE" - } - ] + ] } + ], + "usageMetadata": { + "promptTokenCount": 8, + "totalTokenCount": 8 + }, + "modelVersion": "gemini-1.5-flash" } diff --git a/mock-responses/unary-failure-finish-reason-safety.json b/mock-responses/unary-failure-finish-reason-safety.json index 111e33d..ca27481 100644 --- a/mock-responses/unary-failure-finish-reason-safety.json +++ b/mock-responses/unary-failure-finish-reason-safety.json @@ -10,45 +10,42 @@ "role": "model" }, "finishReason": "SAFETY", - "index": 0, "safetyRatings": [ { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE" + "category": "HARM_CATEGORY_HATE_SPEECH", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.3984375, + "severity": "HARM_SEVERITY_LOW", + "severityScore": 0.21582031 }, { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "HIGH" + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.14941406, + "severity": "HARM_SEVERITY_NEGLIGIBLE", + "severityScore": 0.02331543 }, { "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE" + "probability": "LOW", + "blocked": true, + "probabilityScore": 0.61328125, + "severity": "HARM_SEVERITY_LOW", + "severityScore": 0.31835938 }, { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE" + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "probability": "NEGLIGIBLE", + "probabilityScore": 0.13476563, + "severity": "HARM_SEVERITY_NEGLIGIBLE", + "severityScore": 0.12109375 } ] } ], - "promptFeedback": { - "safetyRatings": [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE" - } - ] - } + "usageMetadata": { + "promptTokenCount": 8, + "totalTokenCount": 8 + }, + "modelVersion": "gemini-1.5-flash" } From 7c5c51975c50be2183e87f40d3f209c7d75f29dc Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 11 Oct 2024 16:54:02 -0400 Subject: [PATCH 2/3] Temporarily add `"index": 0` to see if it fixes JS test --- mock-responses/streaming-failure-finish-reason-safety.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mock-responses/streaming-failure-finish-reason-safety.txt b/mock-responses/streaming-failure-finish-reason-safety.txt index b318cba..f8a36e7 100644 --- a/mock-responses/streaming-failure-finish-reason-safety.txt +++ b/mock-responses/streaming-failure-finish-reason-safety.txt @@ -1,2 +1,2 @@ -data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.40429688,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.18554688},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.15820313,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.030639648},{"category": "HARM_CATEGORY_HARASSMENT","probability": "LOW","blocked": true,"probabilityScore": 0.55078125,"severity": "HARM_SEVERITY_LOW","severityScore": 0.24804688},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.17675781,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.17871094}]}],"usageMetadata": {"promptTokenCount": 10,"candidatesTokenCount": 66,"totalTokenCount": 76},"modelVersion": "gemini-1.5-flash"} +data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.40429688,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.18554688},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.15820313,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.030639648},{"category": "HARM_CATEGORY_HARASSMENT","probability": "LOW","blocked": true,"probabilityScore": 0.55078125,"severity": "HARM_SEVERITY_LOW","severityScore": 0.24804688},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.17675781,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.17871094}]}],"usageMetadata": {"promptTokenCount": 10,"candidatesTokenCount": 66,"totalTokenCount": 76},"modelVersion": "gemini-1.5-flash"} From e5c85be4812ac218ff9456e78fa896fbf2363ae6 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 11 Oct 2024 17:10:16 -0400 Subject: [PATCH 3/3] Revert "Temporarily add `"index": 0` to see if it fixes JS test" This reverts commit 7c5c51975c50be2183e87f40d3f209c7d75f29dc. --- mock-responses/streaming-failure-finish-reason-safety.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mock-responses/streaming-failure-finish-reason-safety.txt b/mock-responses/streaming-failure-finish-reason-safety.txt index f8a36e7..b318cba 100644 --- a/mock-responses/streaming-failure-finish-reason-safety.txt +++ b/mock-responses/streaming-failure-finish-reason-safety.txt @@ -1,2 +1,2 @@ -data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","index": 0,"safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.40429688,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.18554688},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.15820313,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.030639648},{"category": "HARM_CATEGORY_HARASSMENT","probability": "LOW","blocked": true,"probabilityScore": 0.55078125,"severity": "HARM_SEVERITY_LOW","severityScore": 0.24804688},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.17675781,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.17871094}]}],"usageMetadata": {"promptTokenCount": 10,"candidatesTokenCount": 66,"totalTokenCount": 76},"modelVersion": "gemini-1.5-flash"} +data: {"candidates": [{"content": {"parts": [{"text": ""}],"role": "model"},"finishReason": "SAFETY","safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH","probability": "NEGLIGIBLE","probabilityScore": 0.40429688,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.18554688},{"category": "HARM_CATEGORY_DANGEROUS_CONTENT","probability": "NEGLIGIBLE","probabilityScore": 0.15820313,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.030639648},{"category": "HARM_CATEGORY_HARASSMENT","probability": "LOW","blocked": true,"probabilityScore": 0.55078125,"severity": "HARM_SEVERITY_LOW","severityScore": 0.24804688},{"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT","probability": "NEGLIGIBLE","probabilityScore": 0.17675781,"severity": "HARM_SEVERITY_NEGLIGIBLE","severityScore": 0.17871094}]}],"usageMetadata": {"promptTokenCount": 10,"candidatesTokenCount": 66,"totalTokenCount": 76},"modelVersion": "gemini-1.5-flash"}