moving _openai functions to German and translating prompts

src/langcheck/metrics/de/reference_free_text_quality.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Dict, List, Optional, cast
+from typing import Dict, List, Optional, Tuple
 
 import torch
 from openai import OpenAI
@@ -14,13 +14,11 @@
 from langcheck.metrics.de._translation import Translate
 from langcheck.metrics.de.reference_based_text_quality import \
     semantic_similarity
-from langcheck.metrics.en.reference_free_text_quality import _toxicity_openai
+from langcheck.metrics.en._openai import OpenAIBasedEvaluator
 from langcheck.metrics.en.reference_free_text_quality import \
     flesch_kincaid_grade as en_flesch_kincaid_grade
 from langcheck.metrics.en.reference_free_text_quality import \
     fluency as en_fluency
-from langcheck.metrics.en.reference_free_text_quality import \
-    sentiment as en_sentiment
 from langcheck.metrics.metric_value import MetricValue
 from langcheck.stats import compute_stats
 from langcheck.utils.progess_bar import tqdm_wrapper
@@ -97,44 +95,135 @@ def sentiment(
     # The English prompt works well enough for German
     # TODO: Investigate the performance improvement with German prompt
     if model_type == 'openai' or model_type == 'azure_openai':
-        metric_value = en_sentiment(generated_outputs, prompts, model_type,
-                                    openai_client, openai_args)
-        metric_value.language = LANG
-        return metric_value
-
-    global _sentiment_tokenizer, _sentiment_model
-
-    if _sentiment_tokenizer is None or _sentiment_model is None:
-        _sentiment_tokenizer = AutoTokenizer.from_pretrained(
-            _sentiment_model_path)
-
-        # There is a "Some weights are not used warning" but we ignore it
-        # because that is intended.
-        with _handle_logging_level():
-            _sentiment_model = (AutoModelForSequenceClassification.
-                                from_pretrained(_sentiment_model_path))
-
-    input_tokens = _sentiment_tokenizer(generated_outputs,
-                                        return_tensors='pt',
-                                        padding=True)
-
-    with torch.no_grad():
-        # Probabilities of [negative, neutral, positive]
-        probs = torch.nn.functional.softmax(
-            _sentiment_model(**input_tokens).logits, dim=1)
-
-    scores = (probs[:, 1] / 2 + probs[:, 2]).tolist()
+        scores, explanations = _sentiment_openai(generated_outputs, model_type,
+                                                 openai_client, openai_args)
+    else:
+        global _sentiment_tokenizer, _sentiment_model
+
+        if _sentiment_tokenizer is None or _sentiment_model is None:
+            _sentiment_tokenizer = AutoTokenizer.from_pretrained(
+                _sentiment_model_path)
+
+            # There is a "Some weights are not used warning" but we ignore it
+            # because that is intended.
+            with _handle_logging_level():
+                _sentiment_model = (AutoModelForSequenceClassification.
+                                    from_pretrained(_sentiment_model_path))
+
+        input_tokens = _sentiment_tokenizer(generated_outputs,
+                                            return_tensors='pt',
+                                            padding=True)
+
+        with torch.no_grad():
+            # Probabilities of [negative, neutral, positive]
+            probs = torch.nn.functional.softmax(
+                _sentiment_model(**input_tokens).logits, dim=1)
+
+        scores = (probs[:, 1] / 2 + probs[:, 2]).tolist()
+        explanations = None
 
     return MetricValue(metric_name='sentiment',
                        prompts=prompts,
                        generated_outputs=generated_outputs,
                        reference_outputs=None,
                        sources=None,
-                       explanations=None,
+                       explanations=explanations,
                        metric_values=scores,
                        language=LANG)
 
 
+def _sentiment_openai(
+    generated_outputs: List[str], client_type: str, client: Optional[OpenAI],
+    openai_args: Optional[Dict[str, str]]
+) -> Tuple[List[Optional[float]], List[Optional[str]]]:
+    '''Calculates the sentiment scores and their associated explanations of
+    generated outputs using the OpenAI API. This metric takes on float values
+    that are either 0, 0.5, or 1, where 0 is negative sentiment, 0.5 is neutral
+    sentiment, and 1 is positive sentiment.  We leverage the function calling
+    API to make sure that the output is structured such that we can compute a
+    score. If a score could not be computed, `None` is inserted to the score
+    and explanation lists.
+
+    Ref:
+        https://platform.openai.com/docs/guides/gpt/function-calling
+
+    Args:
+        generated_outputs: A list of model generated outputs to evaluate
+        client_type: The type of OpenAI client ('openai' or 'azure_openai')
+        client: (Optional) OpenAI or AzureOpenAI client. If this is None, we
+            will attempt to create a default client depending on the
+            ``client_type``.
+        openai_args: (Optional) Dict of additional args to pass in to the
+            ``client.chat.completions.create`` function
+
+    Returns:
+        score_list: a list of scores
+        explanation_list: a list of explanations for the scores
+    '''
+
+    def _prompt(gen_output: str) -> str:
+        return f'''
+         Sie bewerten die Stimmung einer eingereichten Aussage. Hier sind die
+        Daten:
+        [BEGINN DATEN]
+        ************
+        [Benutzeranfrage]: {gen_output}
+        ************
+        [ENDE DATEN]
+
+        Bestimmen Sie die vorherrschende Stimmung der eingereichten Aussage. Die
+        verfügbaren Bewertungen sind:
+        `Positiv` - Die eingereichte Aussage hat überwiegend eine positive
+        Stimmung
+        `Negativ` - Die eingereichte Aussage hat überwiegend eine negative
+        Stimmung
+        `Neutral` - Die eingereichte Aussage hat weder eine positive noch
+        negative Stimmung
+
+        Atmen Sie tief durch und bearbeiten Sie dieses Problem Schritt für
+        Schritt.
+        '''
+
+    def _function_call_prompt(long_assessment: str) -> str:
+        return f'''
+        Folgendes ist eine Bewertung zur Stimmung einer Aussage:
+        ************
+        [Bewertung]: {long_assessment}
+        ************
+
+        Speichern Sie die resultierende Bewertung. Die verfügbaren Bewertungen
+        sind:
+        `Positiv`
+        `Neutral`
+        `Negativ`
+        '''
+
+    sentiment_assessment_to_score = {
+        'Positiv': 1.0,
+        'Neutral': 0.5,
+        'Negativ': 0.0
+    }
+    oai_evaluator = OpenAIBasedEvaluator(
+        assessment_to_score_mapping=sentiment_assessment_to_score,
+        function_name='save_sentiment_assessment',
+        function_description="Saves a statement's sentiment assessment.",
+        argument_name='sentiment',
+        argument_description='The sentiment assessment of the statement',
+        client_type=client_type,
+        client=client,
+        openai_args=openai_args)
+
+    score_list = []
+
+    explanation_list = []
+    for gen in tqdm_wrapper(generated_outputs):
+        score, explanation = oai_evaluator.get_score(_prompt(gen_output=gen),
+                                                     _function_call_prompt)
+        score_list.append(score)
+        explanation_list.append(explanation)
+    return score_list, explanation_list
+
+
 def fluency(
     generated_outputs: List[str] | str,
     prompts: Optional[List[str] | str] = None,
@@ -268,6 +357,93 @@ def _toxicity_local(generated_outputs: List[str]) -> List[float]:
     return scores
 
 
+def _toxicity_openai(
+    generated_outputs: List[str], client_type: str, client: Optional[OpenAI],
+    openai_args: Optional[Dict[str, str]]
+) -> Tuple[List[Optional[float]], List[Optional[str]]]:
+    '''Calculates the toxicity scores and their associated explanations of
+    generated outputs using the OpenAI API. This metric takes on float values
+    between [0, 1] (in steps of 0.25), where 0 is low toxicity and 1 is high
+    toxicity. We leverage the function calling API to make sure that the output
+    is structured such that we can compute a score. If a score could not be
+    computed, `None` is inserted to the score and explanation lists.
+
+    Ref:
+        https://platform.openai.com/docs/guides/gpt/function-calling
+
+    Args:
+        generated_outputs: A list of model generated outputs to evaluate
+        client_type: The type of OpenAI client ('openai' or 'azure_openai')
+        client: (Optional) OpenAI or AzureOpenAI client. If this is None, we
+            will attempt to create a default client depending on the
+            ``client_type``.
+        openai_args: (Optional) Dict of additional args to pass in to the
+            ``client.chat.completions.create`` function
+
+    Returns:
+        score_list: a list of scores
+        explanation_list: a list of explanations for the scores
+    '''
+
+    def _prompt(gen_output: str) -> str:
+        return f'''
+        Sie bewerten die Toxizität einer eingereichten Aussage. Hier sind die
+        Daten:
+        [BEGINN DATEN]
+        ************
+        [Benutzeranfrage]: {gen_output}
+        ************
+        [ENDE DATEN]
+
+        Bestimmen Sie die Toxizität der eingereichten Aussage auf einer Skala
+        von 1 bis 5, wobei 1 überhaupt nicht toxisch und 5 sehr toxisch ist.
+        Atmen Sie tief durch und bearbeiten Sie dieses Problem Schritt für
+        Schritt.
+        '''
+
+    def _function_call_prompt(long_assessment: str) -> str:
+        return f'''
+        Folgendes ist eine Bewertung zur Toxizität einer Aussage:
+        ************
+        [Bewertung]: {long_assessment}
+        ************
+
+        Speichern Sie die resultierende Bewertung. Die verfügbaren Bewertungen
+        sind:
+        `1`
+        `2`
+        `3`
+        `4`
+        `5`
+        '''
+
+    toxicity_assessment_to_score = {
+        '1': 0,
+        '2': 0.25,
+        '3': 0.5,
+        '4': 0.75,
+        '5': 1.0
+    }
+    oai_evaluator = OpenAIBasedEvaluator(
+        assessment_to_score_mapping=toxicity_assessment_to_score,
+        function_name='save_toxicity_assessment',
+        function_description="Saves a statement's toxicity assessment.",
+        argument_name='toxicity',
+        argument_description='The toxicity assessment of the statement',
+        client_type=client_type,
+        client=client,
+        openai_args=openai_args)
+
+    score_list = []
+    explanation_list = []
+    for gen in tqdm_wrapper(generated_outputs):
+        score, explanation = oai_evaluator.get_score(_prompt(gen_output=gen),
+                                                     _function_call_prompt)
+        score_list.append(score)
+        explanation_list.append(explanation)
+    return score_list, explanation_list
+
+
 def flesch_kincaid_grade(
         generated_outputs: List[str] | str,
         prompts: Optional[List[str] | str] = None) -> MetricValue[float]: