diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationRequest.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationRequest.java index c8ba072c90d..6a1b490cb6f 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationRequest.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationRequest.java @@ -12,6 +12,7 @@ * correctness of the chat response based on the context. * * @author Mark Pollack + * @author EddĂș MelĂ©ndez * @since 1.0.0 M1 */ public class EvaluationRequest { @@ -26,6 +27,10 @@ public EvaluationRequest(String userText, String responseContent) { this(userText, Collections.emptyList(), responseContent); } + public EvaluationRequest(List dataList, String responseContent) { + this("", dataList, responseContent); + } + public EvaluationRequest(String userText, List dataList, String responseContent) { this.userText = userText; this.dataList = dataList; diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationResponse.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationResponse.java index a22d738c6ec..f866cb5e247 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationResponse.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationResponse.java @@ -5,14 +5,15 @@ public class EvaluationResponse { - private boolean pass; + private final boolean pass; - private float score; + private final float score; - private String feedback; + private final String feedback; - Map metadata; + private final Map metadata; + @Deprecated public EvaluationResponse(boolean pass, float score, String feedback, Map metadata) { this.pass = pass; this.score = score; @@ -20,6 +21,13 @@ public EvaluationResponse(boolean pass, float score, String feedback, Map metadata) { + this.pass = pass; + this.score = 0; + this.feedback = feedback; + this.metadata = metadata; + } + public boolean isPass() { return pass; } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/Evaluator.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/Evaluator.java index 7cfdbbf674c..b14fe2adb90 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/Evaluator.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/Evaluator.java @@ -1,8 +1,22 @@ package org.springframework.ai.evaluation; +import org.springframework.ai.model.Content; +import org.springframework.util.StringUtils; + +import java.util.List; +import java.util.stream.Collectors; + @FunctionalInterface public interface Evaluator { EvaluationResponse evaluate(EvaluationRequest evaluationRequest); + default String doGetSupportingData(EvaluationRequest evaluationRequest) { + List data = evaluationRequest.getDataList(); + return data.stream() + .map(Content::getContent) + .filter(StringUtils::hasText) + .collect(Collectors.joining(System.lineSeparator())); + } + } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/HallucinationDetectionEvaluator.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/HallucinationDetectionEvaluator.java new file mode 100644 index 00000000000..7541bd76d4c --- /dev/null +++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/HallucinationDetectionEvaluator.java @@ -0,0 +1,37 @@ +package org.springframework.ai.evaluation; + +import org.springframework.ai.chat.client.ChatClient; + +import java.util.Collections; + +public class HallucinationDetectionEvaluator implements Evaluator { + + private static final String DEFAULT_EVALUATION_PROMPT_TEXT = """ + Document: \\n {document}\\n + Claim: \\n {claim} + """; + + private final ChatClient.Builder chatClientBuilder; + + public HallucinationDetectionEvaluator(ChatClient.Builder chatClientBuilder) { + this.chatClientBuilder = chatClientBuilder; + } + + @Override + public EvaluationResponse evaluate(EvaluationRequest evaluationRequest) { + var response = evaluationRequest.getResponseContent(); + var context = doGetSupportingData(evaluationRequest); + + String evaluationResponse = this.chatClientBuilder.build() + .prompt() + .user(userSpec -> userSpec.text(DEFAULT_EVALUATION_PROMPT_TEXT) + .param("document", context) + .param("claim", response)) + .call() + .content(); + + boolean passing = evaluationResponse.equalsIgnoreCase("yes"); + return new EvaluationResponse(passing, "", Collections.emptyMap()); + } + +} diff --git a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/RelevancyEvaluator.java b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/RelevancyEvaluator.java index 9dd75053713..5a0ec203a6d 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/evaluation/RelevancyEvaluator.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/evaluation/RelevancyEvaluator.java @@ -1,12 +1,8 @@ package org.springframework.ai.evaluation; import org.springframework.ai.chat.client.ChatClient; -import org.springframework.ai.model.Content; import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import org.springframework.util.StringUtils; public class RelevancyEvaluator implements Evaluator { @@ -53,12 +49,4 @@ public EvaluationResponse evaluate(EvaluationRequest evaluationRequest) { return new EvaluationResponse(passing, score, "", Collections.emptyMap()); } - protected String doGetSupportingData(EvaluationRequest evaluationRequest) { - List data = evaluationRequest.getDataList(); - return data.stream() - .map(Content::getContent) - .filter(StringUtils::hasText) - .collect(Collectors.joining(System.lineSeparator())); - } - }