Skip to content

Commit

Permalink
Verify emitted events also for error test suite tests (#392)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Vampire
Vampire authored Feb 11, 2025
1 parent 8beaaea commit 434ac1f
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 55 deletions.
2 changes: 1 addition & 1 deletion src/commonTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/test_suite/ParseResult.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ package it.krzeminski.snakeyaml.engine.kmp.test_suite
import it.krzeminski.snakeyaml.engine.kmp.events.Event

internal data class ParseResult(
val events: List<Event>?,
val events: List<Event>,
val error: Throwable? = null,
)
76 changes: 59 additions & 17 deletions src/commonTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/test_suite/SuiteUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package it.krzeminski.snakeyaml.engine.kmp.test_suite

import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings
import it.krzeminski.snakeyaml.engine.kmp.api.lowlevel.Parse
import it.krzeminski.snakeyaml.engine.kmp.events.Event
import it.krzeminski.snakeyaml.engine.kmp.exceptions.YamlEngineException

/**
Expand All @@ -12,15 +13,17 @@ internal object SuiteUtils {
fun parseData(data: YamlTestData): ParseResult {
val settings = LoadSettings.builder().setLabel(data.label).build()

return try {
ParseResult(Parse(settings).parse(data.inYaml).toList())
} catch (e: YamlEngineException) {
ParseResult(null, e)
}
val events = mutableListOf<Event>()
return runCatching {
Parse(settings).parse(data.inYaml).forEach(events::add)
ParseResult(events)
}.recover {
ParseResult(events, it)
}.getOrThrow()
}

/**
* IDs of cases that should **fail** according to YAML Test Suite, but they actually succeed :(
* IDs of cases that do **not** have the expected result according to YAML Test Suite :(
*
* Such deviations are probably because of an enigmatic bug in SnakeYAML-KMP that hasn't been
* identified and fixed. This is common for YAML libraries, because YAML is far too complicated!
Expand All @@ -31,25 +34,18 @@ internal object SuiteUtils {
* If by some chance you've made a change and a case now passes: well done!
* Remove the case ID from this list.
*
* @see deviationsWithError
* @see deviationsInEvents
*/
val deviationsWithSuccess: Set<YamlTestData.Id> = setOf(
val deviationsInResult: Set<YamlTestData.Id> = setOf(
// should fail but pass
"9C9N",
"9JBA",
"CVW2",
"QB6E",
"SU5Z",
"DK95:01", // https://matrix.yaml.info/details/DK95:01.html
"JEF9:02", // https://matrix.yaml.info/details/JEF9:02.html
"L24T:01", // https://matrix.yaml.info/details/L24T:01.html
).mapToYamlTestDataId()

/**
* IDs of cases that should **pass** according to YAML Test Suite, but they actually fail :(
*
* @see deviationsWithSuccess for further explanation.
*/
val deviationsWithError: Set<YamlTestData.Id> = setOf(
// should pass but fail
"3RLN-01",
"3RLN-04",
"4MUZ",
Expand Down Expand Up @@ -110,6 +106,52 @@ internal object SuiteUtils {
"DK95:07", // https://matrix.yaml.info/details/DK95:07.html
).mapToYamlTestDataId()

/**
* IDs of cases that **do** have the expected result according to YAML Test Suite
* but did not emit the expected events according to the test suite :(
*
* @see deviationsInResult for further explanation.
*/
val deviationsInEvents: Set<YamlTestData.Id> = setOf(
// pass but did emit the wrong events
"JEF9:02", // https://matrix.yaml.info/details/JEF9:02.html
"L24T:01", // https://matrix.yaml.info/details/L24T:01.html

// fail but did emit the wrong events
"2CMS",
"4H7K",
"4JVG",
"7MNF",
"9CWY",
"9KBC",
"CXX2",
"DK95:06",
"EB22",
"EW3V",
"G5U8",
"H7J7",
"HU3P",
"JKF3",
"KS4U",
"MUS6:01",
"P2EQ",
"RHX7",
"SR86",
"SU74",
"T833",
"VJP3:00",
"Y79Y:000",
"Y79Y:003",
"Y79Y:004",
"Y79Y:005",
"Y79Y:006",
"Y79Y:007",
"Y79Y:008",
"Y79Y:009",
"YJV2",
"ZCZ6",
).mapToYamlTestDataId()

private fun Set<String>.mapToYamlTestDataId(): Set<YamlTestData.Id> =
map { id ->
val updatedId = id.replace("-", ":")
Expand Down
100 changes: 69 additions & 31 deletions src/commonTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/test_suite/TestSuiteTests.kt
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
package it.krzeminski.snakeyaml.engine.kmp.test_suite

import io.kotest.assertions.assertSoftly
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.maps.shouldHaveSize
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import io.kotest.matchers.string.shouldNotBeEmpty
import it.krzeminski.snakeyaml.engine.kmp.test_suite.SuiteUtils.deviationsWithError
import it.krzeminski.snakeyaml.engine.kmp.test_suite.SuiteUtils.deviationsWithSuccess
import it.krzeminski.snakeyaml.engine.kmp.test_suite.SuiteUtils.deviationsInEvents
import it.krzeminski.snakeyaml.engine.kmp.test_suite.SuiteUtils.deviationsInResult


class TestSuiteTests : FunSpec({
Expand All @@ -19,9 +20,7 @@ class TestSuiteTests : FunSpec({
test("expect all test data can be read - $id") {
withClue(data) {
data.label.shouldNotBeEmpty()
if (data is YamlTestData.Success) {
data.testEvent.shouldNotBeEmpty()
}
data.testEvent.shouldNotBeEmpty()
}
}
}
Expand All @@ -30,48 +29,87 @@ class TestSuiteTests : FunSpec({
val result = SuiteUtils.parseData(data)

when (id) {
in deviationsWithError ->
test("expect test ${data.id} is ignored because it fails, but should succeed") {
withClue("Expected result had an error, but got none") {
result.error shouldNotBe null
}
in deviationsInResult ->
when (data) {
is YamlTestData.Error ->
test("expect test ${data.id} is ignored because it succeeds, but should fail") {
assertSoftly {
withClue("Expected result did not have an error, but it did") {
result.error shouldBe null
}
result shouldNotHaveEvents data.testEvent
}
}

is YamlTestData.Success ->
test("expect test ${data.id} is ignored because it fails, but should succeed") {
assertSoftly {
withClue("Expected result had an error, but got none") {
result.error shouldNotBe null
}
result shouldNotHaveEvents data.testEvent
}
}
}

in deviationsWithSuccess ->
test("expect test ${data.id} is ignored because succeeds, but it should fail") {
withClue("Expected result did not have an error, but it did") {
result.error shouldBe null
}
if (data is YamlTestData.Success) {
result shouldNotHaveEvents data.testEvent
}
in deviationsInEvents ->
when (data) {
is YamlTestData.Error ->
test("expect test ${data.id} is ignored because it fails, but with wrong events emitted") {
assertSoftly {
withClue("Expected error, but got none") {
result.error shouldNotBe null
}
result shouldNotHaveEvents data.testEvent
}
}

is YamlTestData.Success ->
test("expect test ${data.id} is ignored because it succeeds, but with wrong events emitted") {
assertSoftly {
withClue("Expected no error, but got ${result.error}") {
result.error shouldBe null
}
result shouldNotHaveEvents data.testEvent
}
}
}

else -> {
// use if-statement instead of when-statement because of https://youtrack.jetbrains.com/issue/KT-59274
if (data is YamlTestData.Error) test("expect test $id cannot be parsed - ${data.label}") {
withClue("Expected error, but got none") {
result.error shouldNotBe null
}
} else if (data is YamlTestData.Success) test("expect test $id can be parsed - ${data.label}") {
withClue("Expected no error, but got ${result.error}") {
result.error shouldBe null
}
result shouldHaveEvents data.testEvent
else -> {
when (data) {
is YamlTestData.Error ->
test("expect test $id cannot be parsed - ${data.label}") {
assertSoftly {
withClue("Expected error, but got none") {
result.error shouldNotBe null
}
result shouldHaveEvents data.testEvent
}
}

is YamlTestData.Success ->
test("expect test $id can be parsed - ${data.label}") {
assertSoftly {
withClue("Expected no error, but got ${result.error}") {
result.error shouldBe null
}
result shouldHaveEvents data.testEvent
}
}
}
}
}
}
})

private infix fun ParseResult.shouldHaveEvents(expectedEvents: String) {
val actualEvents = events?.joinToString("\n")
val actualEvents = events.joinToString("\n")
actualEvents shouldBe expectedEvents.normalizeLineEndings()

}

private infix fun ParseResult.shouldNotHaveEvents(expectedEvents: String) {
val actualEvents = events?.joinToString("\n")
val actualEvents = events.joinToString("\n")
actualEvents shouldNotBe expectedEvents.normalizeLineEndings()
}

Expand Down
9 changes: 3 additions & 6 deletions src/commonTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/test_suite/YamlTestData.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ internal sealed interface YamlTestData {
val outYaml: String?
val emitYaml: String?
val inJson: String?
val testEvent: String?
val testEvent: String

@JvmInline
value class Id(private val value: String) {
Expand All @@ -27,12 +27,9 @@ internal sealed interface YamlTestData {
override val outYaml: String? get() = null
override val emitYaml: String? get() = null
override val inJson: String? get() = null
override val testEvent: String? get() = null
}

interface Success : YamlTestData {
override val testEvent: String
}
interface Success : YamlTestData
}

val YamlTestResourcesFileSystem = buildFileSystem(YamlTestSuiteResources.resourcesMap)
Expand All @@ -55,7 +52,7 @@ private fun generateYamlTestDataObject(
override val id: YamlTestData.Id = id
override val label: String = (path / "===").readUtf8()
override val inYaml: String = (path / "in.yaml").readUtf8()

override val testEvent: String = (path / "test.event").readUtf8()
}
} else {
object : YamlTestData.Success {
Expand Down

0 comments on commit 434ac1f

Please sign in to comment.