[WIP] Introduce a new evaluator to detect hallucinations

spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationRequest.java

@@ -12,6 +12,7 @@
  * correctness of the chat response based on the context.
  *
  * @author Mark Pollack
+ * @author Eddú Meléndez
  * @since 1.0.0 M1
  */
 public class EvaluationRequest {
@@ -26,6 +27,10 @@ public EvaluationRequest(String userText, String responseContent) {
 		this(userText, Collections.emptyList(), responseContent);
 	}
 
+	public EvaluationRequest(List<Content> dataList, String responseContent) {
+		this("", dataList, responseContent);
+	}
+
 	public EvaluationRequest(String userText, List<Content> dataList, String responseContent) {
 		this.userText = userText;
 		this.dataList = dataList;

spring-ai-core/src/main/java/org/springframework/ai/evaluation/EvaluationResponse.java

@@ -5,21 +5,29 @@
 
 public class EvaluationResponse {
 
-	private boolean pass;
+	private final boolean pass;
 
-	private float score;
+	private final float score;
 
-	private String feedback;
+	private final String feedback;
 
-	Map<String, Object> metadata;
+	private final Map<String, Object> metadata;
 
+	@Deprecated
 	public EvaluationResponse(boolean pass, float score, String feedback, Map<String, Object> metadata) {
 		this.pass = pass;
 		this.score = score;
 		this.feedback = feedback;
 		this.metadata = metadata;
 	}
 
+	public EvaluationResponse(boolean pass, String feedback, Map<String, Object> metadata) {
+		this.pass = pass;
+		this.score = 0;
+		this.feedback = feedback;
+		this.metadata = metadata;
+	}
+
 	public boolean isPass() {
 		return pass;
 	}

spring-ai-core/src/main/java/org/springframework/ai/evaluation/Evaluator.java

@@ -1,8 +1,22 @@
 package org.springframework.ai.evaluation;
 
+import org.springframework.ai.model.Content;
+import org.springframework.util.StringUtils;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
 @FunctionalInterface
 public interface Evaluator {
 
 	EvaluationResponse evaluate(EvaluationRequest evaluationRequest);
 
+	default String doGetSupportingData(EvaluationRequest evaluationRequest) {
+		List<Content> data = evaluationRequest.getDataList();
+		return data.stream()
+			.map(Content::getContent)
+			.filter(StringUtils::hasText)
+			.collect(Collectors.joining(System.lineSeparator()));
+	}
+
 }

spring-ai-core/src/main/java/org/springframework/ai/evaluation/HallucinationDetectionEvaluator.java

@@ -0,0 +1,37 @@
+package org.springframework.ai.evaluation;
+
+import org.springframework.ai.chat.client.ChatClient;
+
+import java.util.Collections;
+
+public class HallucinationDetectionEvaluator implements Evaluator {
+
+	private static final String DEFAULT_EVALUATION_PROMPT_TEXT = """
+			    Document: \\n {document}\\n
+			    Claim: \\n {claim}
+			""";
+
+	private final ChatClient.Builder chatClientBuilder;
+
+	public HallucinationDetectionEvaluator(ChatClient.Builder chatClientBuilder) {
+		this.chatClientBuilder = chatClientBuilder;
+	}
+
+	@Override
+	public EvaluationResponse evaluate(EvaluationRequest evaluationRequest) {
+		var response = evaluationRequest.getResponseContent();
+		var context = doGetSupportingData(evaluationRequest);
+
+		String evaluationResponse = this.chatClientBuilder.build()
+			.prompt()
+			.user(userSpec -> userSpec.text(DEFAULT_EVALUATION_PROMPT_TEXT)
+				.param("document", context)
+				.param("claim", response))
+			.call()
+			.content();
+
+		boolean passing = evaluationResponse.equalsIgnoreCase("yes");
+		return new EvaluationResponse(passing, "", Collections.emptyMap());
+	}
+
+}

spring-ai-core/src/main/java/org/springframework/ai/evaluation/RelevancyEvaluator.java

@@ -1,12 +1,8 @@
 package org.springframework.ai.evaluation;
 
 import org.springframework.ai.chat.client.ChatClient;
-import org.springframework.ai.model.Content;
 
 import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.springframework.util.StringUtils;
 
 public class RelevancyEvaluator implements Evaluator {
 
@@ -53,12 +49,4 @@ public EvaluationResponse evaluate(EvaluationRequest evaluationRequest) {
 		return new EvaluationResponse(passing, score, "", Collections.emptyMap());
 	}
 
-	protected String doGetSupportingData(EvaluationRequest evaluationRequest) {
-		List<Content> data = evaluationRequest.getDataList();
-		return data.stream()
-			.map(Content::getContent)
-			.filter(StringUtils::hasText)
-			.collect(Collectors.joining(System.lineSeparator()));
-	}
-
 }