Skip to content

Commit

Permalink
Merge pull request #552 from takahirom/takahirom/include-ai-assertion…
Browse files Browse the repository at this point in the history
…-into-roborazzi-test-report/2024-11-13

Include AI assertion results in the Roborazzi test report
takahirom authored Nov 13, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents 7e37f91 + 9663593 commit dc9f27f
Showing 4 changed files with 45 additions and 21 deletions.
Original file line number Diff line number Diff line change
@@ -173,18 +173,18 @@ data class RoborazziOptions(
}
}
aiResult?.aiAssertionResults
?.filter { conditionResult -> conditionResult.requiredFulfillmentPercent != null && conditionResult.failIfNotFulfilled }
?.forEach { conditionResult ->
if (conditionResult.fulfillmentPercent < conditionResult.requiredFulfillmentPercent!!) {
?.filter { assertionResult -> assertionResult.requiredFulfillmentPercent != null && assertionResult.failIfNotFulfilled }
?.forEach { assertionResult: AiAssertionResult ->
if (assertionResult.fulfillmentPercent < assertionResult.requiredFulfillmentPercent!!) {
throw AssertionError(
"The generated image did not meet the required prompt fulfillment percentage.\n" +
"* Condition:\n" +
" - assertionPrompt: ${conditionResult.assertionPrompt}\n" +
" - failIfNotFulfilled: ${conditionResult.failIfNotFulfilled}\n" +
" - requiredFulfillmentPercent: ${conditionResult.requiredFulfillmentPercent}\n" +
" - assertionPrompt: ${assertionResult.assertionPrompt}\n" +
" - failIfNotFulfilled: ${assertionResult.failIfNotFulfilled}\n" +
" - requiredFulfillmentPercent: ${assertionResult.requiredFulfillmentPercent}\n" +
"* Result:\n" +
" - fulfillmentPercent: ${conditionResult.fulfillmentPercent}\n" +
" - explanation: ${conditionResult.explanation}\n" +
" - fulfillmentPercent: ${assertionResult.fulfillmentPercent}\n" +
" - explanation: ${assertionResult.explanation}\n" +
" - referenceFile: ${captureResult.goldenFile}\n" +
" - compareFile: ${captureResult.compareFile}\n" +
" - actualFile: ${captureResult.actualFile}\n"
Original file line number Diff line number Diff line change
@@ -24,6 +24,38 @@ sealed interface CaptureResult {
val goldenFile: String?
val contextData: Map<String, @Contextual Any>

@InternalRoborazziApi
val aiAssertionResultsOrNull: AiAssertionResults?
get() = when (this) {
is Added -> aiAssertionResults
is Changed -> aiAssertionResults
else -> null
}

@InternalRoborazziApi
fun reportText(): String {
return buildString {
append(reportFile.name)
if (contextData.isNotEmpty() && contextData.all {
it.value.toString() != "null" && it.value.toString().isNotEmpty()
}) {
appendLine("contextData:$contextData")
}
aiAssertionResultsOrNull?.aiAssertionResults?.forEach { assertionResult ->
appendLine("aiAssertionResults:")
appendLine(
"* Condition:\n" +
" - assertionPrompt: ${assertionResult.assertionPrompt}\n" +
" - failIfNotFulfilled: ${assertionResult.failIfNotFulfilled}\n" +
" - requiredFulfillmentPercent: ${assertionResult.requiredFulfillmentPercent}\n" +
"* Result:\n" +
" - fulfillmentPercent: ${assertionResult.fulfillmentPercent}\n" +
" - explanation: ${assertionResult.explanation}\n"
)
}
}
}

val reportFile: String
get() = when (val result = this) {
is Added -> result.actualFile
Original file line number Diff line number Diff line change
@@ -132,15 +132,7 @@ data class CaptureResults(
append("<tbody>")
images.forEach { image ->
append("<tr class=\"row\">")

val contextData = if (image.contextData.isNotEmpty() && image.contextData.all {
it.value.toString() != "null" && it.value.toString().isNotEmpty()
}) {
"<br>contextData:${image.contextData}"
} else {
""
}
append("<td class=\"$fileNameClass\" style=\"$fileNameStyle\">${image.reportFile.name}$contextData</td>")
append("<td class=\"$fileNameClass\" style=\"$fileNameStyle\">${image.reportText().lines().joinToString("<br>")}</td>")
append(
"<td class=\"$imgClass\"><img $imgAttributes src=\"${
image.reportFile.pathFrom(
Original file line number Diff line number Diff line change
@@ -540,7 +540,7 @@ fun SemanticsNodeInteraction.captureRoboImage(
actualFile = actualFilePath,
goldenFile = goldenFilePath,
timestampNs = getNanoTime(),
diffPercentage = Float.NaN,
diffPercentage = null,
aiAssertionResults = null,
contextData = emptyMap()
)
@@ -585,7 +585,7 @@ fun SemanticsNodeInteraction.captureRoboImage(
actualFile = actualFilePath,
goldenFile = goldenFilePath,
timestampNs = getNanoTime(),
diffPercentage = Float.NaN,
diffPercentage = null,
aiAssertionResults = null,
contextData = emptyMap()
)
@@ -630,7 +630,7 @@ fun SemanticsNodeInteraction.captureRoboImage(
actualFile = goldenFilePath,
goldenFile = goldenFilePath,
timestampNs = getNanoTime(),
diffPercentage = Float.NaN,
diffPercentage = null,
aiAssertionResults = null,
contextData = emptyMap()
)
@@ -675,7 +675,7 @@ fun SemanticsNodeInteraction.captureRoboImage(
actualFile = goldenFilePath,
goldenFile = goldenFilePath,
timestampNs = getNanoTime(),
diffPercentage = Float.NaN,
diffPercentage = null,
aiAssertionResults = null,
contextData = emptyMap()
)

0 comments on commit dc9f27f

Please sign in to comment.