From edc0757a7ea29fd3bccc7b1236ce7413d31d7897 Mon Sep 17 00:00:00 2001 From: soywiz Date: Thu, 25 Jul 2024 08:18:11 +0200 Subject: [PATCH] Initial commit --- README.md | 2 +- .../.gitignore | 0 .../module.yaml | 5 +- .../src/korlibs/io/lang/Properties.kt | 44 ++ .../src/korlibs/io/lang/SystemProperties.kt | 3 + .../src/korlibs/io/serialization/csv/CSV.kt | 105 ++++ .../src/korlibs/io/serialization/json/Json.kt | 318 ++++++++++ .../src/korlibs/io/serialization/toml/TOML.kt | 249 ++++++++ .../src/korlibs/io/serialization/xml/Xml.kt | 549 ++++++++++++++++++ .../src/korlibs/io/serialization/yaml/Yaml.kt | 363 ++++++++++++ .../src/korlibs/io/util/htmlspecialchars.kt | 14 + .../korlibs/io/lang/SystemProperties.js.kt | 4 + .../korlibs/io/lang/SystemProperties.jvm.kt | 12 + .../io/lang/SystemProperties.native.kt | 4 + .../korlibs/io/lang/SystemProperties.wasm.kt | 4 + .../test/korlibs/io/lang/PropertiesTest.kt | 37 ++ .../korlibs/io/serialization/csv/CSVTest.kt | 41 ++ .../io/serialization/json/JsonPrettyTest.kt | 86 +++ .../korlibs/io/serialization/json/JsonTest.kt | 49 ++ .../korlibs/io/serialization/toml/TOMLTest.kt | 79 +++ .../io/serialization/xml/XmlEntitiesTest.kt | 22 + .../korlibs/io/serialization/xml/XmlTest.kt | 159 +++++ .../korlibs/io/serialization/yaml/YamlTest.kt | 350 +++++++++++ korlibs-simple/src/korlibs/simple/Simple.kt | 4 - 24 files changed, 2497 insertions(+), 6 deletions(-) rename {korlibs-simple => korlibs-serialization}/.gitignore (100%) rename {korlibs-simple => korlibs-serialization}/module.yaml (70%) create mode 100644 korlibs-serialization/src/korlibs/io/lang/Properties.kt create mode 100644 korlibs-serialization/src/korlibs/io/lang/SystemProperties.kt create mode 100644 korlibs-serialization/src/korlibs/io/serialization/csv/CSV.kt create mode 100644 korlibs-serialization/src/korlibs/io/serialization/json/Json.kt create mode 100644 korlibs-serialization/src/korlibs/io/serialization/toml/TOML.kt create mode 100644 korlibs-serialization/src/korlibs/io/serialization/xml/Xml.kt create mode 100644 korlibs-serialization/src/korlibs/io/serialization/yaml/Yaml.kt create mode 100644 korlibs-serialization/src/korlibs/io/util/htmlspecialchars.kt create mode 100644 korlibs-serialization/src@js/korlibs/io/lang/SystemProperties.js.kt create mode 100644 korlibs-serialization/src@jvmAndAndroid/korlibs/io/lang/SystemProperties.jvm.kt create mode 100644 korlibs-serialization/src@native/korlibs/io/lang/SystemProperties.native.kt create mode 100644 korlibs-serialization/src@wasm/korlibs/io/lang/SystemProperties.wasm.kt create mode 100644 korlibs-serialization/test/korlibs/io/lang/PropertiesTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/csv/CSVTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/json/JsonPrettyTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/json/JsonTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/toml/TOMLTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/xml/XmlEntitiesTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/xml/XmlTest.kt create mode 100644 korlibs-serialization/test/korlibs/io/serialization/yaml/YamlTest.kt delete mode 100644 korlibs-simple/src/korlibs/simple/Simple.kt diff --git a/README.md b/README.md index bca9b5c..0af6674 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# korlibs-library-template \ No newline at end of file +# korlibs-serialization diff --git a/korlibs-simple/.gitignore b/korlibs-serialization/.gitignore similarity index 100% rename from korlibs-simple/.gitignore rename to korlibs-serialization/.gitignore diff --git a/korlibs-simple/module.yaml b/korlibs-serialization/module.yaml similarity index 70% rename from korlibs-simple/module.yaml rename to korlibs-serialization/module.yaml index 399d46a..0359490 100644 --- a/korlibs-simple/module.yaml +++ b/korlibs-serialization/module.yaml @@ -8,6 +8,9 @@ aliases: - jvmAndAndroid: [jvm, android] dependencies: + - com.soywiz:korlibs-string:6.0.0 + - com.soywiz:korlibs-math-core:6.0.0 + - com.soywiz:korlibs-datastructure-core:6.0.0 + - com.soywiz:korlibs-platform:6.0.0 test-dependencies: - diff --git a/korlibs-serialization/src/korlibs/io/lang/Properties.kt b/korlibs-serialization/src/korlibs/io/lang/Properties.kt new file mode 100644 index 0000000..a6f96e1 --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/lang/Properties.kt @@ -0,0 +1,44 @@ +package korlibs.io.lang + +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.set + +open class Properties(map: Map? = null) { + //private val map = FastStringMap() + // This is required to work with K/N memory model + private val map = LinkedHashMap().also { + if (map != null) it.putAll(map) + } + + open operator fun contains(key: String): Boolean = get(key) != null + open operator fun get(key: String): String? = map[key] + open operator fun set(key: String, value: String) { map[key] = value } + open fun setAll(values: Map) { + for ((key, value) in values) set(key, value) + } + open fun remove(key: String) { map.remove(key) } + open fun getAll(): Map = map.toMap() + + override fun toString(): String = buildString { + for ((key, value) in map) { + appendLine("$key=${value.replace("\n", "\\n")}") + } + } + + companion object { + fun parseString(data: String): Properties { + val props = LinkedHashMap() + for (line in data.lines()) { + val (rline) = line.trim().split('#') + if (rline.isEmpty()) continue + val key = rline.substringBefore('=', "").trim() + val value = rline.substringAfter('=', "").trim() + if (key.isNotEmpty() && value.isNotEmpty()) { + props[key] = value + } + } + return Properties(props) + } + } +} diff --git a/korlibs-serialization/src/korlibs/io/lang/SystemProperties.kt b/korlibs-serialization/src/korlibs/io/lang/SystemProperties.kt new file mode 100644 index 0000000..5e5f1bb --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/lang/SystemProperties.kt @@ -0,0 +1,3 @@ +package korlibs.io.lang + +expect object SystemProperties : Properties diff --git a/korlibs-serialization/src/korlibs/io/serialization/csv/CSV.kt b/korlibs-serialization/src/korlibs/io/serialization/csv/CSV.kt new file mode 100644 index 0000000..a2b0991 --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/serialization/csv/CSV.kt @@ -0,0 +1,105 @@ +@file:Suppress("PackageDirectoryMismatch") + +package korlibs.io.serialization.csv + +import korlibs.util.* + +class CSV(val lines: List>, val names: List? = null) : Collection { + val namesToIndex: Map = names?.withIndex()?.associate { it.value to it.index } ?: emptyMap() + val linesWithNames = if (names != null) listOf(names, *lines.toTypedArray()) else lines + val records: List = lines.map { Record(it) } + + override val size: Int get() = records.size + operator fun get(index: Int) = records[index] + + override fun iterator(): Iterator = records.iterator() + override fun contains(element: Record): Boolean = records.contains(element) + override fun containsAll(elements: Collection): Boolean = records.containsAll(elements) + override fun isEmpty(): Boolean = records.isEmpty() + + inner class Record(val cells: List) { + operator fun get(index: Int): String = getOrNull(index) ?: error("Can't find element at $index") + operator fun get(name: String): String = getOrNull(name) ?: error("Can't find element '$name'") + + fun getOrNull(index: Int): String? = cells.getOrNull(index) + fun getOrNull(name: String): String? = namesToIndex[name]?.let { cells[it] } + + fun toMap(): Map = if (names != null) cells.zip(names).associate { it.first to it.second } else cells.mapIndexed { index, s -> index to s }.associate { "${it.first}" to it.second } + override fun toString(): String = if (names != null) "${toMap()}" else "$cells" + } + + fun toString(separator: Char): String = linesWithNames.joinToString("\n") { serializeLine(it, separator) } + override fun toString(): String = toString(DEFAULT_SEPARATOR) + + companion object { + const val DEFAULT_SEPARATOR = ',' + + internal fun serializeElement(value: String, separator: Char): String { + if (!value.contains('"') && !value.contains('\n') && !value.contains(separator)) return value + val out = StringBuilder(value.length) + for (n in 0 until value.length) { + out.append(value[n]) + } + return out.toString() + } + + fun serializeLine(values: List, separator: Char = DEFAULT_SEPARATOR): String { + return values.joinToString("$separator") { serializeElement(it, separator) } + } + + fun parseLine(line: String, separator: Char = DEFAULT_SEPARATOR): List = parseLine(SimpleStrReader(line), separator) + + fun parseLine(line: SimpleStrReader, separator: Char = DEFAULT_SEPARATOR): List { + val out = arrayListOf() + val str = StringBuilder() + while (line.hasMore) { + val c = line.readChar() + when (c) { + // Quoted string + '"' -> { + loop@while (line.hasMore) { + val c2 = line.readChar() + when (c2) { + '"' -> { + if (line.peekChar() == '"') { + line.readChar() + str.append('"') + } else { + break@loop + } + } + else -> str.append(c2) + } + } + } + // Line break + '\n' -> { + break + } + // Empty string + separator -> { + out.add(str.toString()) + str.clear() + } + // Normal string + else -> { + str.append(c) + } + } + } + out.add(str.toString()) + str.clear() + return out + } + + fun parse(s: SimpleStrReader, separator: Char = DEFAULT_SEPARATOR, headerNames: Boolean = true): CSV { + val lines = arrayListOf>() + while (s.hasMore) { + lines.add(parseLine(s, separator)) + } + return if (headerNames) CSV(lines.drop(1), lines[0]) else CSV(lines, null) + } + + fun parse(str: String, separator: Char = DEFAULT_SEPARATOR, headerNames: Boolean = true): CSV = parse(SimpleStrReader(str), separator, headerNames) + } +} diff --git a/korlibs-serialization/src/korlibs/io/serialization/json/Json.kt b/korlibs-serialization/src/korlibs/io/serialization/json/Json.kt new file mode 100644 index 0000000..c88335c --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/serialization/json/Json.kt @@ -0,0 +1,318 @@ +@file:Suppress("PackageDirectoryMismatch") + +package korlibs.io.serialization.json + +import korlibs.datastructure.* +import korlibs.io.util.* +import korlibs.util.* +import kotlin.collections.set + +object JsonFast : Json() { + override val optimizeNumbers: Boolean = true + override fun createDoubleArrayList(doubles: MiniNumberArrayList): Any = doubles +} + +open class Json { + companion object : Json() { } + + fun parse(s: String): Any? = parse(SimpleStrReader(s)) + + fun stringify(obj: Any?, pretty: Boolean = false): String = when { + pretty -> SimpleIndenter().apply { stringifyPretty(obj, this) }.toString() + else -> StringBuilder().apply { stringify(obj, this) }.toString() + } + + protected open val optimizeNumbers: Boolean = false + //protected open val optimizeNumbers: Boolean = true + protected open fun createArrayList(capacity: Int = 16): MutableList = ArrayList(capacity) + protected open fun createDoubleArrayList(doubles: MiniNumberArrayList): Any = doubles.toDoubleArray().toList() + //protected open fun createDoubleArrayList(doubles: MiniNumberArrayList): Any = doubles + + protected class MiniNumberArrayList : DoubleList { + override var size: Int = 0 + private set + + @PublishedApi internal var items = DoubleArray(16) + val capacity get() = items.size + + fun clear() { + size = 0 + } + + override operator fun get(index: Int): Double = items[index] + + fun add(value: Double) { + if (size >= capacity) { + items = items.copyOf(items.size * 3) + } + items[size++] = value + } + + inline fun fastForEach(block: (Double) -> Unit) { + for (n in 0 until size) block(items[n]) + } + + override fun toDoubleArray(): DoubleArray = items.copyOf(size) + override fun clone(): DoubleList = MiniNumberArrayList().also { + it.size = size + it.items = items.copyOf() + } + + override fun equals(other: Any?): Boolean { + if (other !is Collection<*>) return false + if (other.size != this.size) return false + if (other is DoubleList) { + for (n in 0 until size) if (this[n] != other[n]) return false + return true + } + if (other is List<*>) { + for (n in 0 until size) if (this[n] != other[n]) return false + return true + } + var n = 0 + for (v in other) if (this[n++] != v) return false + return true + } + + override fun hashCode(): Int = this.items.contentHashCode(0, size) + + private inline fun hashCoder(count: Int, gen: (index: Int) -> Int): Int { + var out = 0 + for (n in 0 until count) { + out *= 7 + out += gen(n) + } + return out + } + private fun DoubleArray.contentHashCode(src: Int, dst: Int): Int = hashCoder(dst - src) { this[src + it].toInt() } // Do not want to use Long (.toRawBits) to prevent boxing on JS + + override fun toString(): String = StringBuilder(2 + 5 * size).also { sb -> + sb.append('[') + for (n in 0 until size) { + if (n != 0) sb.append(", ") + val v = this.getAt(n) + if (v.toInt().toDouble() == v) sb.append(v.toInt()) else sb.append(v) + } + sb.append(']') + }.toString() + } + + interface CustomSerializer { + fun encodeToJson(b: StringBuilder) + } + + fun parse(s: SimpleStrReader): Any? = when (val ic = s.skipSpaces().peekChar()) { + '{' -> parseObject(s) + '[' -> parseArray(s) + //'-', '+', in '0'..'9' -> { // @TODO: Kotlin native doesn't optimize char ranges + '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' -> { + val dres = parseNumber(s) + if (dres.toInt().toDouble() == dres) dres.toInt() else dres + } + 't' -> true.also { s.expect("true") } + 'f' -> false.also { s.expect("false") } + 'n' -> null.also { s.expect("null") } + 'u' -> null.also { s.expect("undefined") } + '"' -> s.readStringLit().toString() + else -> invalidJson("Not expected '$ic' in $s") + } + + private fun parseObject(s: SimpleStrReader): Map { + s.skipExpect('{') + return LinkedHashMap().apply { + obj@ while (true) { + when (s.skipSpaces().peekChar()) { + '}' -> { s.readChar(); break@obj } + ',' -> { s.readChar(); continue@obj } + else -> Unit + } + val key = parse(s) as String + s.skipSpaces().skipExpect(':') + val value = parse(s) + this[key] = value + } + } + } + + private fun parseArray(s: SimpleStrReader): Any { + var out: MutableList? = null + var outNumber: MiniNumberArrayList? = null + s.skipExpect('[') + array@ while (true) { + when (s.skipSpaces().peekChar()) { + ']' -> { s.readChar(); break@array } + ',' -> { s.readChar(); continue@array } + else -> Unit + } + val v = s.peekChar() + if (out == null && optimizeNumbers && (v in '0'..'9' || v == '-')) { + if (outNumber == null) { + outNumber = MiniNumberArrayList() + } + outNumber.add(parseNumber(s)) + } else { + if (out == null) out = createArrayList(outNumber?.size ?: 16) + if (outNumber != null) { + outNumber.fastForEach { out.add(it) } + outNumber = null + } + out.add(parse(s)) + } + } + return outNumber?.let { createDoubleArrayList(outNumber) } ?: out ?: createArrayList() + } + + private fun parseNumber(s: SimpleStrReader): Double = NumberParser.parseDouble { + val c = s.peekChar() + val isC = ((c >= '0') && (c <= '9')) || c == '.' || c == 'e' || c == 'E' || c == '-' || c == '+' + if (isC) s.readChar() + if (isC) c else '\u0000' + } + + fun stringify(obj: Any?, b: StringBuilder) { + when (obj) { + null -> b.append("null") + is Boolean -> b.append(if (obj) "true" else "false") + is Map<*, *> -> { + b.append('{') + for ((i, v) in obj.entries.withIndex()) { + if (i != 0) b.append(',') + stringify(v.key, b) + b.append(':') + stringify(v.value, b) + } + b.append('}') + } + is Iterable<*> -> { + b.append('[') + for ((i, v) in obj.withIndex()) { + if (i != 0) b.append(',') + stringify(v, b) + } + b.append(']') + } + is Enum<*> -> encodeString(obj.name, b) + is String -> encodeString(obj, b) + is Number -> b.append("$obj") + is CustomSerializer -> obj.encodeToJson(b) + else -> throw IllegalArgumentException("Don't know how to serialize $obj") //encode(ClassFactory(obj::class).toMap(obj), b) + } + } + + fun stringifyPretty(obj: Any?, b: SimpleIndenter) { + when (obj) { + null -> b.inline("null") + is Boolean -> b.inline(if (obj) "true" else "false") + is Map<*, *> -> { + b.line("{") + b.indent { + val entries = obj.entries + for ((i, v) in entries.withIndex()) { + if (i != 0) b.line(",") + b.inline(encodeString("" + v.key)) + b.inline(": ") + stringifyPretty(v.value, b) + if (i == entries.size - 1) b.line("") + } + } + b.inline("}") + } + is Iterable<*> -> { + b.line("[") + b.indent { + val entries = obj.toList() + for ((i, v) in entries.withIndex()) { + if (i != 0) b.line(",") + stringifyPretty(v, b) + if (i == entries.size - 1) b.line("") + } + } + b.inline("]") + } + is String -> b.inline(encodeString(obj)) + is Number -> b.inline("$obj") + is CustomSerializer -> b.inline(StringBuilder().apply { obj.encodeToJson(this) }.toString()) + else -> { + throw IllegalArgumentException("Don't know how to serialize $obj") + //encode(ClassFactory(obj::class).toMap(obj), b) + } + } + } + + private fun encodeString(str: String) = StringBuilder().apply { encodeString(str, this) }.toString() + + private fun encodeString(str: String, b: StringBuilder) { + b.append('"') + for (c in str) { + when (c) { + '\\' -> b.append("\\\\"); '/' -> b.append("\\/"); '\'' -> b.append("\\'") + '"' -> b.append("\\\""); '\b' -> b.append("\\b"); '\u000c' -> b.append("\\f") + '\n' -> b.append("\\n"); '\r' -> b.append("\\r"); '\t' -> b.append("\\t") + else -> b.append(c) + } + } + b.append('"') + } + + private fun invalidJson(msg: String = "Invalid JSON"): Nothing = throw IllegalArgumentException(msg) + + fun SimpleStrReader.readStringLit(reportErrors: Boolean = true, out: StringBuilder = StringBuilder()): StringBuilder { + val quotec = readChar() + when (quotec) { + '"', '\'' -> Unit + else -> throw IllegalArgumentException("Invalid string literal") + } + var closed = false + loop@ while (hasMore) { + when (val c = readChar()) { + '\\' -> { + val cc = readChar() + val c: Char = when (cc) { + '\\' -> '\\'; '/' -> '/'; '\'' -> '\''; '"' -> '"' + 'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t' + 'u' -> NumberParser.parseInt(radix = 16) { if (it >= 4) NumberParser.END else readChar() }.toChar() + else -> throw IllegalArgumentException("Invalid char '$cc'") + } + out.append(c) + } + quotec -> { + closed = true + break@loop + } + else -> out.append(c) + } + } + if (!closed && reportErrors) { + throw RuntimeException("String literal not closed! '${this}'") + } + return out + } + + private fun SimpleStrReader.expect(str: String) { + for (n in str.indices) { + val c = readChar() + if (c != str[n]) throw IllegalStateException("Expected '$str' but found '$c' at $n") + } + } + + private fun SimpleStrReader.skipSpaces(): SimpleStrReader { + this.skipWhile { it.isWhitespaceFast() } + return this + } + + private inline fun SimpleStrReader.skipWhile(filter: (Char) -> Boolean) { + while (hasMore && filter(this.peekChar())) { + this.readChar() + } + } + + private fun SimpleStrReader.skipExpect(expected: Char) { + val readed = this.readChar() + if (readed != expected) throw IllegalArgumentException("Expected '$expected' but found '$readed' at $pos") + } + + private fun Char.isWhitespaceFast(): Boolean = this == ' ' || this == '\t' || this == '\r' || this == '\n' +} + +fun String.fromJson(): Any? = Json.parse(this) +fun Map<*, *>.toJson(pretty: Boolean = false): String = Json.stringify(this, pretty) diff --git a/korlibs-serialization/src/korlibs/io/serialization/toml/TOML.kt b/korlibs-serialization/src/korlibs/io/serialization/toml/TOML.kt new file mode 100644 index 0000000..109c046 --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/serialization/toml/TOML.kt @@ -0,0 +1,249 @@ +@file:Suppress("PackageDirectoryMismatch") + +package korlibs.io.serialization.toml + +//import org.intellij.lang.annotations.Language +import kotlin.math.min + +// @TODO: Dates +// @TODO: Special cases for ' & ''' & """ +// @TODO: Check for bugs and for invalid scenarios that shouldn't be accepted +object TOML { + fun parseToml( + //@Language("toml") + str: String, out: MutableMap = LinkedHashMap()): Map { + return StrReader(str).parseToml(out) + } + + private fun Char.isLetterOrDigitOrUndescore(): Boolean = this.isLetterOrDigit() || this == '_' + + fun Char.isValidLiteralChar(): Boolean { + val it = this + return it.isLetterOrDigitOrUndescore() || it == '+' || it == '-' || it == ':' || it == '.' + } + + private fun StrReader.parseToml(out: MutableMap): Map { + var current: MutableMap = out + while (hasMore) { + val c = peek() + //println("PEEK: '$c'") + when (c) { + ' ', '\t', '\n', '\r' -> { + skip() + continue + } + '#' -> { + skipUntil { it == '\n' } + } + '[' -> { + expect('[') + val append = expectOpt('[') + val keys = parseKeys() + if (append) expect(']') + expect(']') + current = out + for ((index, key) in keys.withIndex()) { + val isLast = index == keys.size -1 + if (isLast && append) { + val array = current.getOrPut(key) { arrayListOf>() } as MutableList> + val obj = LinkedHashMap() + array.add(obj) + current = obj + } else { + current = current.getOrPut(key) { LinkedHashMap() } as MutableMap + } + } + //println("SECTION: $keys") + } + else -> { + skipSpacesOrNLs() + //println(" ---> reader=$this") + val keys = parseKeys() + skipSpacesOrNLs() + expect('=') + skipSpacesOrNLs() + val value = parseLiteral() + //println("KEYS[a]: $keys, value=$value, reader=$this") + skipSpacesOrNLs() + //println("KEYS[b]: $keys, value=$value, reader=$this") + var ccurrent = current + for ((index, key) in keys.withIndex()) { + val isLast = index == keys.size -1 + if (isLast) { + ccurrent[key] = value + } else { + ccurrent = ccurrent.getOrPut(key) { LinkedHashMap() } as MutableMap + } + } + //println("ITEM: $keys=[${value!!::class}]$value") + } + } + } + //println(out) + return out + } + private fun StrReader.parseLiteral(): Any? { + skipSpacesOrNLs() + return when (val c = peek()) { + '{' -> { + val items = LinkedHashMap() + expect('{') + while (hasMore) { + skipSpacesOrNLs() + if (expectOpt('}')) break + val keys = parseKeys() + skipSpacesOrNLs() + expect('=') + skipSpacesOrNLs() + val item = parseLiteral() + var citems = items + for ((index, key) in keys.withIndex()) { + val isLast = index == keys.size - 1 + if (isLast) { + citems[key] = item + } else { + citems = citems.getOrPut(key) { LinkedHashMap() } as LinkedHashMap + } + } + if (expectOpt(',')) continue + } + return items + } + '[' -> { + val items = arrayListOf() + expect('[') + while (hasMore) { + skipSpacesOrNLs() + if (expectOpt(']')) break + items += parseLiteral() + skipSpacesOrNLs() + if (expectOpt(',')) continue + } + return items + } + '"', '\'' -> parseStringLiteral() + else -> { + if (!c.isValidLiteralChar()) error("Expected literal but found = '$c'") + val value = readWhile { it.isValidLiteralChar() } + return when (value) { + "null" -> null + "true" -> true + "false" -> false + else -> value.toIntOrNull() ?: value.toDoubleOrNull() ?: value + }.also { + //println("VALUE[${it!!::class}]: $it") + } + } + } + } + + private fun StrReader.parseKeys(): List { + val keys = arrayListOf() + do { + skipSpacesOrNLs() + keys += parseKey() + skipSpacesOrNLs() + } while (expectOpt('.')) + return keys + } + + private fun StrReader.parseKey(): String { + skipSpacesOrNLs() + val c = peek() + return if (c == '"' || c == '\'') { + parseStringLiteral() + } else if (c.isLetterOrDigitOrUndescore()) { + val key = readWhile { it.isLetterOrDigitOrUndescore() } + //println("KEY: '$key'") + key + } else { + return "" + } + } + + private fun StrReader.parseStringLiteral(): String { + val sb = StringBuilder() + val parseStart = peek(0) + if (parseStart != '\'' && parseStart != '"') error("Invalid string $parseStart") + + val triplet = (peek(1) == parseStart && peek(2) == parseStart) + if (triplet) skip(3) else skip(1) + + while (hasMore) { + val c = peek(0) + if (triplet && peek(0) == parseStart && peek(1) == parseStart && peek(2) == parseStart) { + skip(3) + break + } + if (c == parseStart) { + skip(1) + break + } + if (c == '\\') { + skip() + sb.append(when (val cc = read()) { + 'b' -> '\u0008' + 't' -> '\u0009' + 'n' -> '\u000a' + 'f' -> '\u000c' + 'r' -> '\u000d' + //'"' -> '"' + //'\\' -> '\\' + 'u' -> (readString { skip(4) }.toIntOrNull(16) ?: 0).toChar() + 'U' -> (readString { skip(8) }.toIntOrNull(16) ?: 0).toChar() + else -> cc + }) + } else { + skip() + sb.append(c) + } + } + var str = sb.toString() + if (triplet && str.isNotEmpty() && str[0] == '\n') str = str.substring(1) + return str + } + + private class StrReader(val str: String, var pos: Int = 0) { + val len: Int get() = str.length + val available: Int get() = len - pos + val hasMore: Boolean get() = pos < len + val eof: Boolean get() = !hasMore + fun peek(offset: Int = 0): Char = str.getOrElse(pos + offset) { '\u0000' } + fun skip(n: Int = 1): Unit { + pos += n + } + fun read(): Char = peek().also { skip(1) } + fun expectOpt(c: Char): Boolean { + if (peek() == c) { + skip() + return true + } + return false + } + fun expect(c: Char) { + val p = read() + if (p != c) error("Expected '$c' but found '$p'") + } + + inline fun readString(block: () -> Unit): String { + val spos = pos + block() + return str.substring(spos, pos) + } + + inline fun skipWhile(block: (Char) -> Boolean) { + while (hasMore) { + if (!block(peek())) break + skip() + } + } + inline fun skipUntil(block: (Char) -> Boolean) = readWhile { !block(it) } + inline fun readWhile(block: (Char) -> Boolean): String = readString { skipWhile(block) } + inline fun readUntil(block: (Char) -> Boolean): String = readWhile { !block(it) } + + fun skipSpaces() = skipWhile { it == ' ' || it == '\t' } + fun skipSpacesOrNLs() = skipWhile { it == ' ' || it == '\t' || it == '\r' || it == '\n' } + + override fun toString(): String = "StrReader[$len](pos=$pos, peek='${str.substring(pos, min(len, pos + 10))}')" + } +} diff --git a/korlibs-serialization/src/korlibs/io/serialization/xml/Xml.kt b/korlibs-serialization/src/korlibs/io/serialization/xml/Xml.kt new file mode 100644 index 0000000..afc59f4 --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/serialization/xml/Xml.kt @@ -0,0 +1,549 @@ +@file:Suppress("PackageDirectoryMismatch") + +package korlibs.io.serialization.xml + +import korlibs.io.stream.* +import korlibs.io.util.* +import korlibs.util.* +import korlibs.util.CharReaderStrReader +import kotlin.collections.set + +data class Xml( + val type: Type, + val name: String, + val attributes: Map, + val allChildren: List, + val content: String +) { + fun withExtraChild(node: Xml) = copy(allChildren = allChildren + node) + + val attributesLC: Map = attributes.toCaseInsensitiveMap() + val nameLC: String = name.lowercase().trim() + val descendants: Sequence get() = allChildren.asSequence().flatMap { it.descendants + it } + val allChildrenNoComments get() = allChildren.filter { !it.isComment } + val allNodeChildren get() = allChildren.filter { it.isNode } + + companion object { + private const val NAME_RAW = "_raw_" + private const val NAME_TEXT = "_text_" + private const val NAME_CDATA = "_cdata_" + private const val NAME_COMMENT = "_comment_" + + fun Tag(tagName: String, attributes: Map, children: List): Xml = + Xml(Xml.Type.NODE, tagName, attributes.filter { it.value != null }.map { it.key to it.value.toString() }.toMap(), children, "") + fun Raw(text: String): Xml = Xml(Xml.Type.TEXT, NAME_RAW, LinkedHashMap(), listOf(), text) + fun Text(text: String): Xml = Xml(Xml.Type.TEXT, NAME_TEXT, LinkedHashMap(), listOf(), text) + fun CData(text: String): Xml = Xml(Xml.Type.TEXT, NAME_CDATA, LinkedHashMap(), listOf(), text) + fun Comment(text: String): Xml = Xml(Xml.Type.COMMENT, NAME_COMMENT, LinkedHashMap(), listOf(), text) + + //operator fun invoke(@Language("xml") str: String): Xml = parse(str) + + fun parse(str: String, collapseSpaces: Boolean = true, processNamespaces: Boolean = false): Xml { + try { + val stream = Xml.Stream.xmlSequence(str, collapseSpaces, processNamespaces).iterator() + + data class Level(val children: List, val close: Xml.Stream.Element.CloseTag?) + + fun level(): Level { + val children = arrayListOf() + + var textNodes = 0 + var endTag: Xml.Stream.Element.CloseTag? = null + loop@while (stream.hasNext()) { + when (val tag = stream.next()) { + is Xml.Stream.Element.ProcessingInstructionTag -> Unit + is Xml.Stream.Element.CommentTag -> children.add(Xml.Comment(tag.text)) + is Xml.Stream.Element.Text -> children.add((if (tag.cdata) Xml.CData(tag.text) else Xml.Text(tag.text))).also { textNodes++ } + is Xml.Stream.Element.OpenCloseTag -> children.add(Xml.Tag(tag.name, tag.attributes, listOf())) + is Xml.Stream.Element.OpenTag -> { + val out = level() + if (out.close?.name?.equals(tag.name, ignoreCase = true) != true) { + throw IllegalArgumentException("Expected '${tag.name}' but was ${out.close?.name}") + } + children.add(Xml(Xml.Type.NODE, tag.name, tag.attributes, out.children, "")) + } + is Xml.Stream.Element.CloseTag -> { + endTag = tag + break@loop + } + } + } + + return Level(children.mapIndexed { i, it -> + when { + it.type == Xml.Type.TEXT -> { + val firstText = i == 0 + val lastText = i == children.size - 1 + when{ + firstText && lastText -> it.copy(content = it.content.trim()) + firstText -> it.copy(content = it.content.trimStart()) + lastText -> it.copy(content = it.content.trimEnd()) + else -> it + } + } + else -> it + } + }, endTag) + } + + val children = level().children + return children.firstOrNull { it.type == Xml.Type.NODE } + ?: children.firstOrNull() + ?: Xml.Text("") + } catch (t: NoSuchElementException) { + println("ERROR: XML: $str thrown a NoSuchElementException") + return Xml.Text("!!ERRORED!!") + } + } + } + + val text: String get() = when (type) { + Type.NODE -> allChildren.joinToString("") { it.text } + Type.TEXT -> content + Type.COMMENT -> "" + } + + fun toOuterXmlIndentedString(indenter: SimpleIndenter = SimpleIndenter(trailingLine = true)): String = toOuterXmlIndented(indenter).toString() + "\n" + + fun toOuterXmlIndented(indenter: SimpleIndenter = SimpleIndenter(trailingLine = true)): SimpleIndenter = indenter.apply { + when (type) { + Type.NODE -> { + if (allChildren.isEmpty()) { + line("<$name$attributesStr/>") + } else if (allChildren.size == 1 && allChildren[0].type == Type.TEXT) { + inline("<$name$attributesStr>") + inline(allChildren[0].content) + line("") + } else { + line("<$name$attributesStr>") + indent { + allChildren.fastForEach { child -> + child.toOuterXmlIndented(indenter) + } + } + line("") + } + } + else -> line(outerXml) + } + } + + val attributesStr: String get() = attributes.toList().map { " ${it.first}=\"${it.second}\"" }.joinToString("") + + val outerXml: String + get() = when (type) { + Type.NODE -> when { + allChildren.isEmpty() -> "<$name$attributesStr/>" + else -> { + val children = this.allChildren.joinToString("") { it.outerXml } + "<$name$attributesStr>$children" + } + } + Type.TEXT -> when (name) { + NAME_TEXT -> Entities.encode(content) + NAME_CDATA -> "" + NAME_RAW -> content + else -> content + } + Type.COMMENT -> "" + } + + val innerXml: String get() = when (type) { + Type.NODE -> this.allChildren.joinToString("") { it.outerXml } + else -> outerXml + } + + operator fun get(name: String): Iterable = children(name) + + fun children(name: String): Iterable = allChildren.filter { it.name.equals(name, ignoreCase = true) } + fun child(name: String): Xml? = children(name).firstOrNull() + fun childText(name: String): String? = child(name)?.text + + fun hasAttribute(key: String): Boolean = this.attributesLC.containsKey(key) + fun attribute(name: String): String? = this.attributesLC[name] + + fun getString(name: String): String? = this.attributesLC[name] + fun getInt(name: String): Int? = this.attributesLC[name]?.toInt() + fun getLong(name: String): Long? = this.attributesLC[name]?.toLong() + fun getDouble(name: String): Double? = this.attributesLC[name]?.toDouble() + fun getFloat(name: String): Float? = this.attributesLC[name]?.toFloat() + + fun double(name: String, defaultValue: Double = 0.0): Double = + this.attributesLC[name]?.toDoubleOrNull() ?: defaultValue + + fun boolean(name: String, defaultValue: Boolean = false): Boolean = booleanOrNull(name) ?: defaultValue + + fun booleanOrNull(name: String): Boolean? = + when (str(name).toLowerCase()) { + "true", "1" -> true + "false", "0" -> false + else -> null + } + + fun float(name: String, defaultValue: Float = 0f): Float = this.attributesLC[name]?.toFloatOrNull() ?: defaultValue + fun int(name: String, defaultValue: Int = 0): Int = this.attributesLC[name]?.toIntOrNull() ?: defaultValue + fun long(name: String, defaultValue: Long = 0): Long = this.attributesLC[name]?.toLongOrNull() ?: defaultValue + fun str(name: String, defaultValue: String = ""): String = this.attributesLC[name] ?: defaultValue + fun uint(name: String, defaultValue: UInt = 0u): UInt = this.attributesLC[name]?.toUIntOrNull() ?: defaultValue + + fun doubleNull(name: String): Double? = this.attributesLC[name]?.toDoubleOrNull() + fun floatNull(name: String): Float? = this.attributesLC[name]?.toFloatOrNull() + fun intNull(name: String): Int? = this.attributesLC[name]?.toIntOrNull() + fun longNull(name: String): Long? = this.attributesLC[name]?.toLongOrNull() + fun strNull(name: String): String? = this.attributesLC[name] + + //override fun toString(): String = innerXml + override fun toString(): String = outerXml + + enum class Type { NODE, TEXT, COMMENT } + + object Encode { + fun encodeOpenTag(name: String, map: Map, selfClose: Boolean = false): String = buildString { + append("<") + append(name) + if (map.isNotEmpty()) { + append(" ") + append(quoteParams(map)) + } + if (selfClose) { + append("/") + } + append(">") + } + fun encodeCloseTag(name: String): String = "" + fun quoteParams(map: Map): String = map.entries.joinToString(" ") { it.key + "=" + quote(it.value) } + + fun quote(value: Any?): String = when (value) { + is Number, is Boolean -> value.toString() + else -> value?.toString()?.let { quote(it) } ?: "\"\"" + } + fun quote(str: String): String = "\"${Entities.encode(str)}\"" + } + + class Literals( + private val lits: Array, + private val map: MutableMap, + val lengths: Array + ) { + companion object { + fun invoke(vararg lits: String): Literals = + fromList(lits.toCollection(arrayListOf()).toTypedArray()) + + //fun invoke(lits:Array): Literals = fromList(lits) + fun fromList(lits: Array): Literals { + val lengths = lits.map { it.length }.sorted().reversed().distinct().toTypedArray() + val map = linkedMapOf() + lits.fastForEach { lit -> + map[lit] = true + } + return Literals(lits, map, lengths) + } + } + + fun contains(lit: String) = map.containsKey(lit) + + fun matchAt(str: String, offset: Int): String? { + lengths.fastForEach { len -> + val id = str.substr(offset, len) + if (contains(id)) return id + } + return null + } + + private fun String.substr(start: Int, length: Int): String { + val low = (if (start >= 0) start else this.length + start).coerceIn(0, this.length) + val high = (if (length >= 0) low + length else this.length + length).coerceIn(0, this.length) + return if (high >= low) this.substring(low, high) else "" + } + + override fun toString() = "Literals(${lits.joinToString(" ")})" + } + + object Entities { + // Predefined entities in XML 1.0 + private val charToEntity = linkedMapOf('"' to """, '\'' to "'", '<' to "<", '>' to ">", '&' to "&") + private val entities = Literals.fromList(charToEntity.values.toTypedArray()) + private val entityToChar = charToEntity.flip() + + fun encode(str: String): String = str.eachBuilder { + val entry = charToEntity[it] + when { + entry != null -> append(entry) + else -> append(it) + } + } + fun decode(str: String): String = decode(SimpleStrReader(str)) + fun decode(r: SimpleStrReader): String = buildString { + val sb = StringBuilder() + while (!r.eof) { + @Suppress("LiftReturnOrAssignment") // Performance? + append(r.readUntilBuilder('&', sb.clear())) + if (r.eof) break + + r.skipExpect('&') + val value = r.readUntilBuilder(';', sb.clear(), included = true) + val full = "&$value" + when { + value.startsWith('#') -> { + var base = 10 + var str = value.substring(1, value.length - 1) + if(str.startsWith("x")) { + base = 16 + str = str.substring(1) + } + append(str.toIntOrNull(base)?.toChar()) + } + entityToChar.contains(full) -> append(entityToChar[full]) + else -> append(full) + } + } + } + } + + object Stream { + // https://wiki.tei-c.org/index.php/XML_Whitespace + fun parse(str: String, collapseSpaces: Boolean = true): Iterable = parse(SimpleStrReader(str), collapseSpaces) + fun parse(r: SimpleStrReader, collapseSpaces: Boolean = true): Iterable = Xml2Iterable(r, collapseSpaces) + fun parse(r: CharReader, collapseSpaces: Boolean = true): Iterable = Xml2Iterable(CharReaderStrReader(r), collapseSpaces) + + private fun SimpleStrReader.matchStringOrId(out: StringBuilder): StringBuilder? = matchSingleOrDoubleQuoteString(out) ?: matchIdentifier(out) + + private val SPACES = Regex("\\s+") + + fun xmlSequence(str: String, collapseSpaces: Boolean = true, processNamespaces: Boolean = false): Sequence = xmlSequence(SimpleStrReader(str), collapseSpaces, processNamespaces) + fun xmlSequence(r: SimpleStrReader, collapseSpaces: Boolean = true, processNamespaces: Boolean = false): Sequence = sequence { + val sb = StringBuilder(128) + + loop@while (!r.eof) { + val str = r.readUntilBuilder('<', sb.clear(), included = false) + if (str.isNotEmpty()) { + val text = str.toString() + val textNs = when { + collapseSpaces -> text.replace(SPACES, " ").let { if (it.all { it.isWhitespaceFast() }) "" else it } + else -> text + } + if (textNs.isNotEmpty()) { + yield(Element.Text(Xml.Entities.decode(textNs))) + } + } + if (r.eof) break + + r.skipExpect('<') + r.skipSpaces() + sb.clear() + val processingInstruction = r.tryExpect('?') + val processingEntityOrDocType = r.tryExpect('!') + if (processingEntityOrDocType) { + sb.append('!') + while (!r.eof) { + val c = r.peekChar() + if (c == '>' || c.isWhitespaceFast() || c == '/') break + + sb.append(r.readChar()) + + if (sb.startsWith("!--")) { + sb.clear() + while (!r.eof) { + sb.append(r.readChar()) + if (sb.endsWith("-->")) { + sb.deleteRange(sb.length - 3, sb.length) + break + } + } + yield(Element.CommentTag(sb.toString())) + continue@loop + } + if (sb.startsWith("![CDATA[")) { + sb.clear() + while (!r.eof) { + sb.append(r.readChar()) + if (sb.endsWith("]]>")) { + sb.deleteRange(sb.length - 3, sb.length) + break + } + } + yield(Element.Text(sb.toString()).also { it.cdata = true }) + continue@loop + } + } + sb.deleteAt(0) + } + val close = r.tryExpect('/') || processingEntityOrDocType + r.skipSpaces() + val name = sb.takeIf { it.isNotEmpty() }?.toString() + ?: r.matchIdentifier(sb.clear())?.toString() + ?: error("Couldn't match identifier after '<', offset=${r.pos}, near='${r.toStringContext()}'") + r.skipSpaces() + val attributes = linkedMapOf() + while (r.peekChar() != '?' && r.peekChar() != '/' && r.peekChar() != '>') { + val key = r.matchStringOrId(sb.clear())?.toString() ?: throw IllegalArgumentException( + "Malformed document or unsupported xml construct around ~${r.toStringContext()}~ for name '$name'" + ) + r.skipSpaces() + if (r.tryExpect('=')) { + r.skipSpaces() + val argsQuote = r.matchStringOrId(sb.clear())?.toString() + attributes[key] = when { + argsQuote != null && !(argsQuote.startsWith("'") || argsQuote.startsWith("\"")) -> argsQuote + argsQuote != null -> Xml.Entities.decode(argsQuote.substring(1, argsQuote.length - 1)) + else -> Xml.Entities.decode(r.matchIdentifier(sb.clear())!!.toString()) + } + } else { + attributes[key] = key + } + r.skipSpaces() + } + val openclose = r.tryExpect('/') + val processingInstructionEnd = r.tryExpect('?') + r.skipExpect('>') + + // Handle namespace processing based on the processNamespaces flag + val elementName = if (!processNamespaces && name.contains(':')) { + name.substringAfter(':') + } else { + name + } + + yield(when { + processingInstruction || processingEntityOrDocType -> Element.ProcessingInstructionTag(elementName, attributes) + openclose -> Element.OpenCloseTag(elementName, attributes) + close -> Element.CloseTag(elementName) + else -> Element.OpenTag(elementName, attributes) + }) + } + } + + class Xml2Iterable(val reader2: SimpleStrReader, val skipSpaces: Boolean) : Iterable { + val reader = reader2.clone() + override fun iterator(): Iterator = xmlSequence(reader, skipSpaces).iterator() + } + + sealed class Element { + data class ProcessingInstructionTag(val name: String, val attributes: Map) : Element() + data class OpenCloseTag(val name: String, val attributes: Map) : Element() + data class OpenTag(val name: String, val attributes: Map) : Element() + data class CommentTag(val text: String) : Element() + data class CloseTag(val name: String) : Element() + data class Text(val text: String, var cdata: Boolean = false) : Element() + } + } +} + +val Xml.isText get() = this.type == Xml.Type.TEXT +val Xml.isComment get() = this.type == Xml.Type.COMMENT +val Xml.isNode get() = this.type == Xml.Type.NODE + +fun Iterable.str(name: String, defaultValue: String = ""): String = this.first().attributes[name] ?: defaultValue +fun Iterable.children(name: String): Iterable = this.flatMap { it.children(name) } +val Iterable.allChildren: Iterable get() = this.flatMap(Xml::allChildren) +val Iterable.allNodeChildren: Iterable get() = this.flatMap(Xml::allNodeChildren) +val Iterable.firstText: String? get() = this.firstOrNull()?.text +val Iterable.text: String get() = this.joinToString("") { it.text } +operator fun Iterable.get(name: String): Iterable = this.children(name) + +fun Sequence.str(name: String, defaultValue: String = ""): String = this.first().attributes[name] ?: defaultValue +fun Sequence.children(name: String): Sequence = this.flatMap { it.children(name) } +val Sequence.allChildren: Sequence get() = this.flatMap(Xml::allChildren) +val Sequence.allNodeChildren: Sequence get() = this.flatMap(Xml::allNodeChildren) +val Sequence.firstText: String? get() = this.firstOrNull()?.text +val Sequence.text: String get() = this.joinToString("") { it.text } +operator fun Sequence.get(name: String): Sequence = this.children(name) + +fun String.toXml(collapseSpaces: Boolean = true): Xml = Xml.parse(this, collapseSpaces) + +// language=html +fun Xml( + // language=html + str: String, collapseSpaces: Boolean = true, processNamespaces: Boolean = false +): Xml = Xml.parse(str, collapseSpaces, processNamespaces) + +fun Xml.descendants(name: String) = descendants.filter { it.name.equals(name, ignoreCase = true) } +fun Xml.firstDescendant(name: String) = descendants(name).firstOrNull() + +class XmlBuilder @PublishedApi internal constructor() { + @PublishedApi + internal val nodes = arrayListOf() + fun node(node: Xml) = node.also { nodes += node } + inline fun node(tag: String, vararg props: Pair, block: XmlBuilder.() -> Unit = {}): Xml = + Xml.Tag(tag, props.filter { it.second != null }.toMap(), XmlBuilder().apply(block).nodes).also { nodes += it } + fun comment(comment: String): Xml = Xml.Comment(comment).also { nodes += it } + fun text(text: String): Xml = Xml.Text(text).also { nodes += it } + fun cdata(text: String): Xml = Xml.CData(text).also { nodes += it }.also { if (text.contains("]]>")) error("A cdata node cannot contain the ]]> literal") } + fun raw(text: String): Xml = Xml.Raw(text).also { nodes += it } +} + +inline fun buildXml(rootTag: String, vararg props: Pair, crossinline block: XmlBuilder.() -> Unit = {}): Xml = + XmlBuilder().node(rootTag, *props, block = block) + +inline fun Xml(rootTag: String, vararg props: Pair, block: XmlBuilder.() -> Unit = {}): Xml = + XmlBuilder().node(rootTag, *props, block = block) +inline fun Xml(rootTag: String, props: Map?, block: XmlBuilder.() -> Unit = {}): Xml = + XmlBuilder().node(rootTag, *(props ?: emptyMap()).map { it.key to it.value }.toTypedArray(), block = block) + + + +/** + * [Map] with [String] keys that are treated in a insensitive manner. + */ +private class CaseInsensitiveStringMap private constructor( + private val mapOrig: MutableMap, + private val lcToOrig: MutableMap, + private val mapLC: MutableMap +) : MutableMap by mapOrig { + constructor() : this(LinkedHashMap(), LinkedHashMap(), LinkedHashMap()) + constructor(data: Map) : this() { putAll(data) } + constructor(vararg items: Pair) : this() { putAll(items.toList()) } + + override fun containsKey(key: String): Boolean = mapLC.containsKey(key.toLowerCase()) + + override fun clear() { + mapOrig.clear() + mapLC.clear() + lcToOrig.clear() + } + + override fun get(key: String): T? = mapLC[key.toLowerCase()] + + override fun put(key: String, value: T): T? { + remove(key) + mapOrig[key] = value + lcToOrig[key.toLowerCase()] = key + return mapLC.put(key.toLowerCase(), value) + } + + override fun putAll(from: Map) { + for (v in from) put(v.key, v.value) + } + + override fun remove(key: String): T? { + val lkey = key.toLowerCase() + val okey = lcToOrig[lkey] + mapOrig.remove(okey) + val res = mapLC.remove(lkey) + lcToOrig.remove(lkey) + return res + } + + override fun equals(other: Any?): Boolean = (other is CaseInsensitiveStringMap<*>) && this.mapLC == other.mapLC + override fun hashCode(): Int = mapLC.hashCode() +} + +private fun Map.toCaseInsensitiveMap(): Map = + CaseInsensitiveStringMap().also { it.putAll(this) } + +private fun Map.flip(): Map = this.map { Pair(it.value, it.key) }.toMap() + +private inline fun String.eachBuilder(transform: StringBuilder.(Char) -> Unit): String = buildString { + @Suppress("ReplaceManualRangeWithIndicesCalls") // Performance reasons? Check that plain for doesn't allocate + for (n in 0 until this@eachBuilder.length) transform(this, this@eachBuilder[n]) +} + +private inline fun List.fastForEach(callback: (T) -> Unit) { + var n = 0 + while (n < size) callback(this[n++]) +} + +private inline fun Array.fastForEach(callback: (T) -> Unit) { + var n = 0 + while (n < size) callback(this[n++]) +} diff --git a/korlibs-serialization/src/korlibs/io/serialization/yaml/Yaml.kt b/korlibs-serialization/src/korlibs/io/serialization/yaml/Yaml.kt new file mode 100644 index 0000000..dd89c8e --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/serialization/yaml/Yaml.kt @@ -0,0 +1,363 @@ +package korlibs.io.serialization.yaml + +import kotlin.collections.set + +object Yaml { + fun decode(str: String): Any? = read(ListReader(tokenize(str)), level = 0) + fun read(str: String): Any? = read(ListReader(tokenize(str)), level = 0) + + private fun parseStr(toks: List): Any? { + if (toks.size == 1 && toks[0] is Token.STR) return toks[0].ustr + return parseStr(toks.joinToString("") { it.ustr }) + } + + private fun parseStr(str: String) = when (str) { + "null" -> null + "true" -> true + "false" -> false + else -> str.toIntOrNull() ?: str.toDoubleOrNull() ?: str + } + + //const val TRACE = true + const val TRACE = false + private val EMPTY_SET = setOf() + private val SET_COMMA_END_ARRAY = setOf(",", "]") + + private fun read(s: ListReader, level: Int): Any? = s.run { + var list: ArrayList? = null + var map: MutableMap? = null + var lastMapKey: String? = null + var lastMapValue: Any? = null + + val levelStr = if (TRACE) " ".repeat(level) else "" + + linehandle@ while (s.hasMore) { + val token = s.peek() + val line = token as? Token.LINE + val lineLevel = line?.level + if (TRACE && line != null) println("${levelStr}LINE($lineLevel)") + if (lineLevel != null && lineLevel > level) { + // child level + val res = read(s, lineLevel) + if (list != null) { + if (TRACE) println("${levelStr}CHILD.list.add: $res") + list.add(res) + } else { + if (TRACE) println("${levelStr}CHILD.return: $res") + return res + } + } else if (lineLevel != null && lineLevel < level) { + // parent level + if (TRACE) println("${levelStr}PARENT: level < line.level") + break + } else { + // current level + if (line != null) s.read() + if (s.eof) break + val item = s.peek() + when (item.str) { + "-" -> { + if (s.read().str != "-") invalidOp + if (list == null) { + list = arrayListOf() + if (map != null && lastMapKey != null && lastMapValue == null) { + map[lastMapKey] = list + } + } + if (TRACE) println("${levelStr}LIST_ITEM...") + val res = read(s, level + 1) + if (TRACE) println("${levelStr}LIST_ITEM: $res") + list.add(res) + } + "[" -> { + if (s.read().str != "[") invalidOp + val olist = arrayListOf() + array@ while (s.peek().str != "]") { + olist += readOrString(s, level, SET_COMMA_END_ARRAY, supportNonSpaceSymbols = false) + val p = s.peek().str + when (p) { + "," -> { s.read(); continue@array } + "]" -> break@array + else -> invalidOp("Unexpected '$p'") + } + } + if (s.read().str != "]") invalidOp + return olist + } + else -> { + val keyIds = s.readId() + val sp = s.peekOrNull() ?: Token.EOF + if (s.eof || (sp.str != ":" || (sp is Token.SYMBOL && !sp.isNextWhite))) { + val key = parseStr(keyIds) + if (TRACE) println("${levelStr}LIT: $key") + return key + } else { + val key = parseStr(keyIds).toString() + if (map == null) map = LinkedHashMap() + if (s.read().str != ":") invalidOp + if (TRACE) println("${levelStr}MAP[$key]...") + val next = s.peekOrNull() + val nextStr = next?.str + val hasSpaces = next is Token.SYMBOL && next.isNextWhite + val nextIsSpecialSymbol = nextStr == "[" || nextStr == "{" || (nextStr == "-" && hasSpaces) + val value = readOrString(s, level, EMPTY_SET, supportNonSpaceSymbols = !nextIsSpecialSymbol) + lastMapKey = key + lastMapValue = value + map[key] = value + list = null + if (TRACE) println("${levelStr}MAP[$key]: $value") + } + } + } + } + } + + if (TRACE) println("${levelStr}RETURN: list=$list, map=$map") + + return map ?: list + } + + private fun ListReader.readId(): List { + val tokens = arrayListOf() + while (hasMore) { + val token = peek() + if (token is Token.ID || token is Token.STR || ((token is Token.SYMBOL) && token.str == "-") || ((token is Token.SYMBOL) && token.str == ":" && !token.isNextWhite)) { + tokens.add(token) + read() + } else { + break + } + } + return tokens + } + + private fun readOrString(s: ListReader, level: Int, delimiters: Set, supportNonSpaceSymbols: Boolean): Any? { + val sp = s.peek() + return if (sp is Token.ID || (supportNonSpaceSymbols && sp is Token.SYMBOL && !sp.isNextWhite)) { + var str = "" + str@while (s.hasMore) { + val p = s.peek() + if (p is Token.LINE) break@str + if (p.str in delimiters) break@str + str += s.read().str + } + parseStr(str) + } else { + read(s, level + 1) + } + } + + fun tokenize(str: String): List = StrReader(str.replace("\r\n", "\n")).tokenize() + + private fun StrReader.tokenize(): List { + val out = arrayListOf() + + val s = this + var str = "" + fun flush() { + if (str.isNotBlank() && str.isNotEmpty()) { + out += Token.ID(str.trim()); str = "" + } + } + + val indents = ArrayList() + linestart@ while (hasMore) { + // Line start + flush() + val indentStr = readWhile(kotlin.Char::isWhitespace).replace("\t", " ") + if (indentStr.contains('\n')) continue@linestart // ignore empty lines with possible additional indent + val indent = indentStr.length + if (indents.isEmpty() || indent > indents.last()) { + indents += indent + } else { + while (indents.isNotEmpty() && indent < indents.last()) indents.removeAt(indents.size - 1) + if (indents.isEmpty()) invalidOp + } + val indentLevel = indents.size - 1 + while (out.isNotEmpty() && out.last() is Token.LINE) out.removeAt(out.size - 1) + out += Token.LINE(indentStr, indentLevel) + while (hasMore) { + val c = read() + when (c) { + ':', '-', '[', ']', ',' -> { + flush(); out += Token.SYMBOL("$c", peekChar()) + } + '#' -> { + if (str.lastOrNull()?.isWhitespaceFast() == true || (str == "" && out.lastOrNull() is Token.LINE)) { + flush(); readUntilLineEnd(); skip(); continue@linestart + } else { + str += c + } + } + '\n' -> { + flush(); continue@linestart + } + '"', '\'' -> { + flush() + val last = out.lastOrNull() + //println("out=$last, c='$c', reader=$this") + if (last is Token.SYMBOL && (last.str == ":" || last.str == "[" || last.str == "{" || last.str == "," || last.str == "-")) { + s.unread() + //println(" -> c='$c', reader=$this") + out += Token.STR(s.readStringLit()) + } else { + str += c + } + } + else -> str += c + } + } + } + flush() + return out + } + + interface Token { + val str: String + val ustr get() = str + + object EOF : Token { + override val str: String = "" + } + + data class LINE(override val str: String, val level: Int) : Token { + override fun toString(): String = "LINE($level)" + } + + data class ID(override val str: String) : Token + data class STR(override val str: String) : Token { + override val ustr = str.unquote() + } + + data class SYMBOL(override val str: String, val next: Char) : Token { + val isNextWhite: Boolean get() = next == ' ' || next == '\t' || next == '\n' || next == '\r' + } + } + + private fun StrReader.readUntilLineEnd() = this.readUntil { it == '\n' } + + private val invalidOp: Nothing get() = throw RuntimeException() + private fun invalidOp(msg: String): Nothing = throw RuntimeException(msg) + + private class ListReader(val list: List, val ctx: T? = null) { + class OutOfBoundsException(val list: ListReader<*>, val pos: Int) : RuntimeException() + var position = 0 + val eof: Boolean get() = position >= list.size + val hasMore: Boolean get() = position < list.size + fun peekOrNull(): T? = list.getOrNull(position) + fun peek(): T = list.getOrNull(position) ?: throw OutOfBoundsException(this, position) + fun skip(count: Int = 1) = this.apply { this.position += count } + fun read(): T = peek().apply { skip(1) } + override fun toString(): String = "ListReader($list)" + } + + private class StrReader(val str: String, var pos: Int = 0) { + val length get() = str.length + val hasMore get() = pos < length + + inline fun skipWhile(f: (Char) -> Boolean) { while (hasMore && f(peek())) skip() } + fun skipUntil(f: (Char) -> Boolean): Unit = skipWhile { !f(it) } + + // @TODO: https://youtrack.jetbrains.com/issue/KT-29577 + private fun posSkip(count: Int): Int { + val out = this.pos + this.pos += count + return out + } + + fun skip() = skip(1) + fun peek(): Char = if (hasMore) this.str[this.pos] else '\u0000' + fun peekChar(): Char = peek() + fun read(): Char = if (hasMore) this.str[posSkip(1)] else '\u0000' + fun unread() = skip(-1) + + fun substr(start: Int, len: Int = length - pos): String { + val start = (start).coerceIn(0, length) + val end = (start + len).coerceIn(0, length) + return this.str.substring(start, end) + } + + fun skip(count: Int) = this.apply { this.pos += count } + fun peek(count: Int): String = this.substr(this.pos, count) + fun read(count: Int): String = this.peek(count).also { skip(count) } + + private inline fun readBlock(callback: () -> Unit): String { + val start = pos + callback() + val end = pos + return substr(start, end - start) + } + + fun readWhile(f: (Char) -> Boolean): String = readBlock { skipWhile(f) } + fun readUntil(f: (Char) -> Boolean): String = readBlock { skipUntil(f) } + + fun readStringLit(reportErrors: Boolean = true): String { + val out = StringBuilder() + val quotec = read() + when (quotec) { + '"', '\'' -> Unit + else -> throw RuntimeException("Invalid string literal") + } + var closed = false + while (hasMore) { + val c = read() + if (c == '\\') { + val cc = read() + out.append( + when (cc) { + '\\' -> '\\'; '/' -> '/'; '\'' -> '\''; '"' -> '"' + 'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t' + 'u' -> read(4).toInt(0x10).toChar() + else -> throw RuntimeException("Invalid char '$cc'") + } + ) + } else if (c == quotec) { + closed = true + break + } else { + out.append(c) + } + } + if (!closed && reportErrors) { + throw RuntimeException("String literal not closed! '${this.str}'") + } + return out.toString() + } + + override fun toString(): String = "StrReader(str=${str.length}, pos=$pos, range='${str.substring(pos.coerceIn(str.indices), (pos + 10).coerceIn(str.indices)).replace("\n", "\\n")}')" + } + + private fun Char.isWhitespaceFast(): Boolean = this == ' ' || this == '\t' || this == '\r' || this == '\n' + private fun String.isQuoted(): Boolean = this.startsWith('"') && this.endsWith('"') + private fun String.unquote(): String = if (isQuoted()) this.substring(1, this.length - 1).unescape() else this + private fun String.unescape(): String { + val out = StringBuilder(this.length) + var n = 0 + while (n < this.length) { + val c = this[n++] + when (c) { + '\\' -> { + val c2 = this[n++] + when (c2) { + '\\' -> out.append('\\') + '"' -> out.append('\"') + 'n' -> out.append('\n') + 'r' -> out.append('\r') + 't' -> out.append('\t') + 'x', 'u' -> { + val N = if (c2 == 'u') 4 else 2 + val chars = this.substring(n, n + N) + n += N + out.append(chars.toInt(16).toChar()) + } + else -> { + out.append("\\$c2") + } + } + } + else -> out.append(c) + } + } + return out.toString() + } +} diff --git a/korlibs-serialization/src/korlibs/io/util/htmlspecialchars.kt b/korlibs-serialization/src/korlibs/io/util/htmlspecialchars.kt new file mode 100644 index 0000000..cb4ab03 --- /dev/null +++ b/korlibs-serialization/src/korlibs/io/util/htmlspecialchars.kt @@ -0,0 +1,14 @@ +package korlibs.io.util + +fun String.htmlspecialchars(): String = buildString(this@htmlspecialchars.length + 16) { + for (it in this@htmlspecialchars) { + when (it) { + '"' -> append(""") + '\'' -> append("'") + '<' -> append("<") + '>' -> append(">") + '&' -> append("&") + else -> append(it) + } + } +} diff --git a/korlibs-serialization/src@js/korlibs/io/lang/SystemProperties.js.kt b/korlibs-serialization/src@js/korlibs/io/lang/SystemProperties.js.kt new file mode 100644 index 0000000..fce5892 --- /dev/null +++ b/korlibs-serialization/src@js/korlibs/io/lang/SystemProperties.js.kt @@ -0,0 +1,4 @@ +package korlibs.io.lang + +actual object SystemProperties : Properties() { +} diff --git a/korlibs-serialization/src@jvmAndAndroid/korlibs/io/lang/SystemProperties.jvm.kt b/korlibs-serialization/src@jvmAndAndroid/korlibs/io/lang/SystemProperties.jvm.kt new file mode 100644 index 0000000..95dd499 --- /dev/null +++ b/korlibs-serialization/src@jvmAndAndroid/korlibs/io/lang/SystemProperties.jvm.kt @@ -0,0 +1,12 @@ +package korlibs.io.lang + +import korlibs.io.lang.* + +actual object SystemProperties : Properties() { + // Uses querystring on JS/Browser, and proper env vars in the rest + override operator fun get(key: String): String? = System.getProperty(key) + override operator fun set(key: String, value: String) { System.setProperty(key, value) } + override fun remove(key: String) { System.clearProperty(key) } + override fun getAll() = System.getProperties().toMap() as Map +} + diff --git a/korlibs-serialization/src@native/korlibs/io/lang/SystemProperties.native.kt b/korlibs-serialization/src@native/korlibs/io/lang/SystemProperties.native.kt new file mode 100644 index 0000000..fce5892 --- /dev/null +++ b/korlibs-serialization/src@native/korlibs/io/lang/SystemProperties.native.kt @@ -0,0 +1,4 @@ +package korlibs.io.lang + +actual object SystemProperties : Properties() { +} diff --git a/korlibs-serialization/src@wasm/korlibs/io/lang/SystemProperties.wasm.kt b/korlibs-serialization/src@wasm/korlibs/io/lang/SystemProperties.wasm.kt new file mode 100644 index 0000000..fce5892 --- /dev/null +++ b/korlibs-serialization/src@wasm/korlibs/io/lang/SystemProperties.wasm.kt @@ -0,0 +1,4 @@ +package korlibs.io.lang + +actual object SystemProperties : Properties() { +} diff --git a/korlibs-serialization/test/korlibs/io/lang/PropertiesTest.kt b/korlibs-serialization/test/korlibs/io/lang/PropertiesTest.kt new file mode 100644 index 0000000..78137c6 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/lang/PropertiesTest.kt @@ -0,0 +1,37 @@ +package korlibs.io.lang + +import kotlin.test.* + +class PropertiesTest { + @Test + fun testParse() { + val properties = Properties.parseString(""" + hello=world + hi=there + #foo=bar + """.trimIndent()) + assertEquals("world", properties["hello"]) + assertEquals("there", properties["hi"]) + assertEquals(null, properties["foo"]) + } + + @Test + fun testBuild() { + val properties = Properties(mapOf("hello" to "world", "hi" to "there")) + assertEquals("world", properties["hello"]) + assertEquals("there", properties["hi"]) + assertEquals(null, properties["foo"]) + } + + @Test + fun testToString() { + assertEquals( + """ + hello=world + hi=there + + """.trimIndent(), + Properties(mapOf("hello" to "world", "hi" to "there")).toString() + ) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/csv/CSVTest.kt b/korlibs-serialization/test/korlibs/io/serialization/csv/CSVTest.kt new file mode 100644 index 0000000..6b707a8 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/csv/CSVTest.kt @@ -0,0 +1,41 @@ +package korlibs.io.serialization.csv + +import kotlin.test.Test +import kotlin.test.assertEquals + +class CSVTest { + val csvStr = """ + a,b,c + hello,world,this + is,a,test + """.trimIndent() + + @Test + fun testParseLine() { + assertEquals(listOf(""), CSV.parseLine("")) + assertEquals(listOf("a"), CSV.parseLine("a")) + assertEquals(listOf("a", ""), CSV.parseLine("a,")) + assertEquals(listOf("a", "b"), CSV.parseLine("a,b")) + assertEquals(listOf("a", "b"), CSV.parseLine("a,\"b\"")) + assertEquals(listOf("a", "b"), CSV.parseLine("\"a\",\"b\"")) + assertEquals(listOf("a", "\""), CSV.parseLine("\"a\",\"\"\"\"")) + } + + @Test + fun testParse() { + assertEquals(csvStr, CSV.parse(csvStr).toString()) + } + + @Test + fun testParseGet() { + val csv = CSV.parse(csvStr) + + assertEquals(listOf("hello", "is"), csv.map { it["a"] }) + assertEquals(listOf("world", "a"), csv.map { it["b"] }) + assertEquals(listOf("this", "test"), csv.map { it["c"] }) + + assertEquals(listOf("hello", "is"), csv.map { it[0] }) + assertEquals(listOf("world", "a"), csv.map { it[1] }) + assertEquals(listOf("this", "test"), csv.map { it[2] }) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/json/JsonPrettyTest.kt b/korlibs-serialization/test/korlibs/io/serialization/json/JsonPrettyTest.kt new file mode 100644 index 0000000..efdd757 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/json/JsonPrettyTest.kt @@ -0,0 +1,86 @@ +package korlibs.io.serialization.json + +import kotlin.test.* + +class JsonPrettyTest { + @Test + fun encode1() { + assertEquals("1", Json.stringify(1, pretty = true)) + //assertEquals("null", Json.encodePretty(null, mapper)) + assertEquals("true", Json.stringify(true, pretty = true)) + assertEquals("false", Json.stringify(false, pretty = true)) + assertEquals("{\n}", Json.stringify(mapOf(), pretty = true)) + assertEquals("[\n]", Json.stringify(listOf(), pretty = true)) + assertEquals("\"a\"", Json.stringify("a", pretty = true)) + } + + @Test + fun encode2() { + assertEquals( + """ + |[ + | 1, + | 2, + | 3 + |] + """.trimMargin(), Json.stringify(listOf(1, 2, 3), pretty = true) + ) + + assertEquals( + """ + |{ + | "a": 1, + | "b": 2 + |} + """.trimMargin(), Json.stringify(linkedMapOf("a" to 1, "b" to 2), pretty = true) + ) + } + + @Test + fun encodeTyped() { + assertEquals( + """ + |{ + | "a": 1, + | "b": "test" + |} + """.trimMargin(), Json.stringify(mapOf("a" to 1, "b" to "test"), pretty = true) + ) + } + + @Test + fun encodeMix() { + assertEquals( + """ + |{ + | "a": [ + | 1, + | 2, + | 3, + | 4 + | ], + | "b": [ + | 5, + | 6 + | ], + | "c": { + | "a": true, + | "b": null, + | "c": "hello" + | } + |} + """.trimMargin(), + Json.stringify( + linkedMapOf( + "a" to listOf(1, 2, 3, 4), + "b" to listOf(5, 6), + "c" to linkedMapOf( + "a" to true, + "b" to null, + "c" to "hello" + ) + ), pretty = true + ) + ) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/json/JsonTest.kt b/korlibs-serialization/test/korlibs/io/serialization/json/JsonTest.kt new file mode 100644 index 0000000..4e8c5b7 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/json/JsonTest.kt @@ -0,0 +1,49 @@ +package korlibs.io.serialization.json + +import kotlin.test.* + +class JsonTest { + enum class MyEnum { DEMO, HELLO, WORLD } + data class ClassWithEnum(val a: MyEnum = MyEnum.HELLO) + + class Demo2 { + var a: Int = 10 + + companion object { + //@JvmField + var b: String = "test" + } + } + + data class Demo(val a: Int, val b: String) + + data class DemoList(val demos: ArrayList) + + data class DemoSet(val demos: Set) + + @kotlin.test.Test + fun decode1() { + assertEquals(linkedMapOf("a" to 1).toString(), Json.parse("""{"a":1}""").toString()) + //assertEquals(-1e7, Json.decode("""-1e7""")) + assertEquals(-10000000, Json.parse("""-1e7""")) + } + + @kotlin.test.Test + fun decode2() { + assertEquals( + listOf("a", 1, -1, 0.125, 0, 11, true, false, null, listOf(), mapOf()).toString(), + Json.parse("""["a", 1, -1, 0.125, 0, 11, true, false, null, [], {}]""").toString() + ) + } + + @kotlin.test.Test + fun decode3() { + assertEquals("\"", Json.parse(""" "\"" """)) + assertEquals(listOf(1, 2).toString(), Json.parse(""" [ 1 , 2 ]""").toString()) + } + + @kotlin.test.Test + fun decodeUnicode() { + assertEquals("aeb", Json.parse(""" "a\u0065b" """)) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/toml/TOMLTest.kt b/korlibs-serialization/test/korlibs/io/serialization/toml/TOMLTest.kt new file mode 100644 index 0000000..1e379e0 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/toml/TOMLTest.kt @@ -0,0 +1,79 @@ +package korlibs.io.serialization.toml + +import kotlin.test.Test +import kotlin.test.assertEquals + +class TOMLTest { + @Test + fun test() { + assertEquals( + mapOf( + "title" to "TOML Example", + "owner" to mapOf( + "name" to "Tom Preston-Werner", + "dob" to "1979-05-27T07:32:00-08:00" + ), + "database" to mapOf( + "enabled" to true, + "ports" to listOf(+8000, -8001, 8002), + "data" to listOf(listOf("delta", "phi"), listOf(3.14)), + "temp_targets" to mapOf("cpu" to 79.5, "case" to 72.0) + ), + "servers" to mapOf( + "alpha" to mapOf("ip" to "10.0.0.1", "role" to "frontend"), + "beta" to mapOf("ip" to "10.0.0.2", "role" to "backend"), + ), + "products" to listOf( + mapOf("name" to "Hammer", "sku" to 738594937), + mapOf(), + mapOf("name" to "Nail", "sku" to 284758393, "color" to "gray"), + ), + "demo" to mapOf("hello" to mapOf("test" to 10, "demo" to 11, "world" to 12)), + ), + TOML.parseToml( + """ + # This is a TOML document + + title = "TOML Example" + + [owner] + name = "Tom Preston-Werner" + dob = 1979-05-27T07:32:00-08:00 + + [database] + enabled = true + ports = [ +8000, -8001, 8002 ] + data = [ ["delta", "phi"], [3.14] ] + temp_targets = { cpu = 79.5, case = 72.0 } + + [servers] + + [servers.alpha] + ip = "10.0.0.1" + role = "frontend" + + [servers.beta] + ip = "10.0.0.2" + role = "backend" + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] # empty table within the array + + [[products]] + name = "Nail" + sku = 284758393 + + color = "gray" + + [demo] + hello.test = 10 + hello."demo" = 11 + hello.'world' = 12 + """.trimIndent() + ) + ) + } +} \ No newline at end of file diff --git a/korlibs-serialization/test/korlibs/io/serialization/xml/XmlEntitiesTest.kt b/korlibs-serialization/test/korlibs/io/serialization/xml/XmlEntitiesTest.kt new file mode 100644 index 0000000..bb882ec --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/xml/XmlEntitiesTest.kt @@ -0,0 +1,22 @@ +package korlibs.io.serialization.xml + +import kotlin.test.Test +import kotlin.test.assertEquals + +class XmlEntitiesTest { + @Test + fun testDecode() { + assertEquals("hello", Xml.Entities.decode("hello")) + assertEquals("\"", Xml.Entities.decode(""")) + assertEquals("hello\"world", Xml.Entities.decode("hello"world")) + assertEquals("hello\"world\"", Xml.Entities.decode("hello"world"")) + } + + @Test + fun testEncode() { + assertEquals("hello", Xml.Entities.encode("hello")) + assertEquals(""", Xml.Entities.encode("\"")) + assertEquals("hello"world", Xml.Entities.encode("hello\"world")) + assertEquals("hello"world"", Xml.Entities.encode("hello\"world\"")) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/xml/XmlTest.kt b/korlibs-serialization/test/korlibs/io/serialization/xml/XmlTest.kt new file mode 100644 index 0000000..bc3fc24 --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/xml/XmlTest.kt @@ -0,0 +1,159 @@ +package korlibs.io.serialization.xml + +import kotlin.test.* + +class XmlTest { + @Test + fun name() { + val xml = Xml("") + assertEquals(10, xml.int("a")) + assertEquals(10, xml.int("A")) + assertEquals(20, xml.int("zZ")) + assertEquals("hello", xml.name) + assertEquals(7, xml["demo"].first().int("c")) + assertEquals(7, xml["Demo"].first().int("c")) + assertEquals("""""", xml.toString()) + } + + @Test + fun testSkipSpaces() { + val xml = Xml(" hello ") + assertEquals("A", xml.name) + + assertEquals("hello", xml.text) + assertEquals("hello", xml.innerXml) + } + + @Test + fun name2() { + assertEquals( + listOf(Xml.Stream.Element.OpenCloseTag("a_b", mapOf())), + Xml.Stream.xmlSequence("").toList() + ) + assertEquals("a_b", Xml(""). name) + } + + @Test + fun name3() { + assertEquals("""""", Xml.Tag("test", linkedMapOf("z" to 1, "b" to 2), listOf()).outerXml) + } + + @Test + fun name4() { + val xmlStr = """ + + + + + + + + + + + + + + + + + + + + + + + + """.trimIndent() + Xml(xmlStr) + + val items = Xml.Stream.xmlSequence(xmlStr).toList() + assertEquals(Xml.Stream.Element.ProcessingInstructionTag("xml", mapOf("version" to "1.0", "encoding" to "UTF-8")), items[0]) + assertEquals(Xml.Stream.Element.ProcessingInstructionTag("DOCTYPE", mapOf("svg" to "svg", "PUBLIC" to "PUBLIC", "\"-//W3C//DTD SVG 1.1//EN\"" to "\"-//W3C//DTD SVG 1.1//EN\"", "\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\"" to "\"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\"")), items[1]) + + //println("items=$items") + } + + @Test + fun testIndent() { + val xml1 = buildXml("name") { + text("SimpleName") + } + assertEquals("SimpleName\n", xml1.toOuterXmlIndented().toString()) + + val xml2 = buildXml("name") { + text("SimpleName") + text("ComplicatedName") + } + assertEquals("\n\tSimpleName\n\tComplicatedName\n\n", xml2.toOuterXmlIndented().toString()) + + val xml3 = buildXml("name", "number" to 1) { + comment("Some comment") + node("value", "number" to 2) { + text("SimpleName") + } + } + val result2 = "\n\t\n\tSimpleName\n\n" + assertEquals(result2, xml3.toOuterXmlIndented().toString()) + } + + @Test + fun testOutputQuote() { + assertEquals("<&>", buildXml("t") { text("<&>") }.innerXml) + assertEquals("<&>", buildXml("t") { raw("<&>") }.innerXml) + assertEquals("]]>", buildXml("t") { cdata("<&>") }.innerXml) + assertFails { + buildXml("t") { cdata("]]>") }.innerXml + } + } + + @Test + fun testParseCData() { + assertEquals("]]>", Xml("]]>").outerXml) + assertEquals("<&>", Xml("<&>").outerXml) + } + + @Test + fun testNamedDescendant() { + val xml = Xml("") + assertEquals(4, xml.descendants("b").count()) + } + + @Test + fun testAttrWithoutQuotes() { + assertEquals("world", Xml("").strNull("hello")) + assertEquals(20, Xml("").intNull("size")) + } + + @Test + fun testEntities() { + assertEquals("&", Xml.Entities.decode("&")) + assertEquals("’", Xml.Entities.decode("’")) + assertEquals("’", Xml.Entities.decode("’")) + } + + // Verifies: https://wiki.tei-c.org/index.php/XML_Whitespace + @Test + fun testTrimMixed() { + assertEquals("The cat ate the grande croissant. I didn't!", Xml("

The cat ate the grande croissant. I didn't!\n

").text) + assertEquals("hello world", Xml("

hello world

").text) + } + + @Test + fun testXmlNamespaceProcessing() { + val xmlStr = """ + + Content + Content2 + + """.trimIndent() + + val xml = Xml(xmlStr, processNamespaces = false) + assertEquals("child", xml.allNodeChildren[0].name) + assertEquals("child", xml.allNodeChildren[1].name) + + val xml2 = Xml(xmlStr, processNamespaces = true) + assertEquals("ns:child", xml2.allNodeChildren[0].name) + assertEquals("child", xml2.allNodeChildren[1].name) + } +} diff --git a/korlibs-serialization/test/korlibs/io/serialization/yaml/YamlTest.kt b/korlibs-serialization/test/korlibs/io/serialization/yaml/YamlTest.kt new file mode 100644 index 0000000..99b5ded --- /dev/null +++ b/korlibs-serialization/test/korlibs/io/serialization/yaml/YamlTest.kt @@ -0,0 +1,350 @@ +package korlibs.io.serialization.yaml + +import kotlin.test.* + +// http://nodeca.github.io/js-yaml/ +class YamlTest { + @kotlin.test.Test + fun basic() { + assertEquals("str", Yaml.read("str")) + assertEquals(10, Yaml.read("10")) + } + + @kotlin.test.Test + fun array() { + assertEquals(listOf(1, 2, 3), Yaml.read("[1,2,3]")) + } + + @kotlin.test.Test + fun name() { + assertEquals( + listOf(1, 2, 3), + Yaml.read( + """ + - 1 + - 2 + - 3 + """.trimIndent() + ) + ) + } + + @kotlin.test.Test + fun name2() { + assertEquals( + linkedMapOf("hr" to 65, "avg" to 0.278, "rbi" to 147), + Yaml.read( + """ + hr: 65 # Home runs + avg: 0.278 # Batting average + rbi: 147 # Runs Batted In + """.trimIndent() + ) + ) + } + + @kotlin.test.Test + fun name3() { + assertEquals( + listOf(listOf(listOf(1))), + Yaml.read("- - - 1") + ) + } + + @kotlin.test.Test + fun name4() { + assertEquals( + listOf(linkedMapOf("a" to 1), linkedMapOf("a" to 2)), + Yaml.read( + """ + |- + | a: 1 + |- + | a: 2 + """.trimMargin() + ) + ) + } + + @kotlin.test.Test + fun name5() { + assertEquals( + listOf( + linkedMapOf( + "name" to "Mark McGwire", + "hr" to 65, + "avg" to 0.278 + ), + linkedMapOf( + "name" to "Sammy Sosa", + "hr" to 63, + "avg" to 0.288 + ) + ), + Yaml.read( + """ + |- + | name: Mark McGwire + | hr: 65 + | avg: 0.278 + |- + | name: Sammy Sosa + | hr: 63 + | avg: 0.288 + """.trimMargin() + ) + ) + } + + @kotlin.test.Test + fun name6() { + assertEquals( + linkedMapOf( + "hr" to listOf("Mark McGwire", "Sammy Sosa"), + "rbi" to listOf("Sammy Sosa", "Ken Griffey") + ), + Yaml.read( + """ + |hr: # 1998 hr ranking + | - Mark McGwire + | - Sammy Sosa + |rbi: + | # 1998 rbi ranking + | - Sammy Sosa + | - Ken Griffey + """.trimMargin() + ) + ) + } + + @kotlin.test.Test + fun name7() { + assertEquals( + linkedMapOf( + "null" to null, + "booleans" to listOf(true, false), + "string" to "012345" + ), + Yaml.read( + """ + |null: + |booleans: [ true, false ] + |string: '012345' + + """.trimMargin() + ) + ) + } + + enum class MyEnum { DEMO, HELLO, WORLD } + data class ClassWithEnum(val size: Int = 70, val a: MyEnum = MyEnum.HELLO) + + @Test + fun testArrayInMap() { + val yamlStr = """ + tags: [lorem,lorem-ipsum] + """.trimIndent() + //println(Yaml.tokenize(yamlStr)) + Yaml.decode(yamlStr).also { + assertEquals( + mapOf( + "tags" to listOf("lorem", "lorem-ipsum"), + ), + it + ) + } + + } + + @Test + fun testChunk() { + val yamlStr = """ + layout: post + layout2: null + demo: false + permalink: /lorem-ipsum/ + title: "Lorem Ipsum" + feature_image: "/images/2019/lorem_ipsum.jpg" + tags: [lorem,lorem-ipsum] + date: 2019-10-07 00:00:00 + """.trimIndent() + //println(Yaml.tokenize(yamlStr)) + assertEquals( + mapOf( + "layout" to "post", + "layout2" to null, + "demo" to false, + "permalink" to "/lorem-ipsum/", + "title" to "Lorem Ipsum", + "feature_image" to "/images/2019/lorem_ipsum.jpg", + "tags" to listOf("lorem", "lorem-ipsum"), + "date" to "2019-10-07 00:00:00" + ), + Yaml.decode(yamlStr) + ) + assertEquals( + "layout:null", + Yaml.decode("layout:null") + ) + } + + @Test + fun testChunk2() { + assertEquals( + mapOf( + "tags" to listOf("lorem", "ipsum"), + "layout" to "post", + "hello" to mapOf("world" to listOf("a", "b")), + "title" to "demo: 2D test demo lorem ipsum", + "title_es" to "lorem: ipsum sim de 2D te test", + "date" to "2009-05-05T10:45:00.000+02:00", + "author" to "abc def hij", + "feature_image" to "/images/2009/ipsum.png", + "modified_time" to "2011-05-14T15:39:49.185+02:00", + "thumbnail" to "http://1.bp.example.com/-lorem/ipsum/sit/AMEN-demo/s72-c/test.png", + "blogger_id" to "tag:blogger.com,1999:blog-1212121212121212121212122.post-12121212121212121212121", + "blogger_orig_url" to "http://blog.example.es/2009/05/demo-loreim-ip-sit-demo.html" + ), + Yaml.decode( + """ + tags: + - lorem + - ipsum + layout: post + hello: + world: + - a + - b + title: 'demo: 2D test demo lorem ipsum' + title_es: 'lorem: ipsum sim de 2D te test' + date: '2009-05-05T10:45:00.000+02:00' + author: abc def hij + feature_image: /images/2009/ipsum.png + modified_time: '2011-05-14T15:39:49.185+02:00' + thumbnail: http://1.bp.example.com/-lorem/ipsum/sit/AMEN-demo/s72-c/test.png + blogger_id: tag:blogger.com,1999:blog-1212121212121212121212122.post-12121212121212121212121 + blogger_orig_url: http://blog.example.es/2009/05/demo-loreim-ip-sit-demo.html + """.trimIndent() + ) + ) + } + //@Test + //fun name8() { + // assertEquals( + // null, + // Yaml.read("[a:1,b:2]") + // ) + //} + + @Test + fun testMapListIssue() { + val testYmlString = """ + hello: + - a + + - b + + lineWithSpaces: + + + - aa + - bb + + world: + - c + - d + test: + - e + - f + """.trimIndent() + //println("\n\n[[[$testYmlString]]]\n\n") + assertEquals( + mapOf( + "hello" to listOf("a", "b"), + "lineWithSpaces" to listOf("aa", "bb"), + "world" to listOf("c", "d"), + "test" to listOf("e", "f") + ), + Yaml.decode(testYmlString) + ) + } + + @Test + fun testWindowsLineEndings() { + assertEquals( + mapOf( + "key1" to mapOf("read" to true), + "key2" to mapOf("read" to false), + ), + Yaml.decode("key1:\r\n read: true\r\nkey2:\r\n read: false\r\n") + ) + } + + @Test + fun testWindowsLineEndings2() { + assertEquals( + mapOf( + "key1" to mapOf("read" to true), + "key2" to mapOf("read" to false), + ), + Yaml.decode("key1:\r\n read: true\r\n\r\nkey2:\r\n read: false\r\n") + ) + } + + @Test + fun testHyphenInKeys() { + assertEquals( + mapOf( + "this-is-an-example" to mapOf("fail" to true), + ), + Yaml.decode(""" + this-is-an-example: + fail: true + """.trimIndent()) + ) + } + + @Test + fun testDoubleColon() { + assertEquals(listOf( + "git::adder::korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db", + ), Yaml.decode(""" + - git::adder::korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db + """.trimIndent())) + + assertEquals(listOf(mapOf( + "git::adder" to ":korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db", + )), Yaml.decode(""" + - git::adder: :korlibs/kproject::/modules/adder::54f73b01cea9cb2e8368176ac45f2fca948e57db + """.trimIndent())) + } + + @Test + fun testSingleQuoteInString() { + assertEquals( + mapOf( + "hello" to "world", + "title" to "What's Happening", + "demo" to listOf("hello", "world", "test", "what's happening", "yeah"), + "dependencies" to listOf( + "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f", + "https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f", + "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f", + "https://github.com/korlibs/kproject.git/samples/demo2", + ), + ), + Yaml.decode(""" + hello: 'world' + title: What's Happening + demo: ["hello", "world", "test", what's happening, yeah] + # hi + dependencies: + - "https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f" + - "https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f" + - https://github.com/korlibs/kproject.git/samples/demo2#95696dd942ebc8db4ee9d9f4835ce12d853ff16f + - https://github.com/korlibs/kproject.git/samples/demo2 #95696dd942ebc8db4ee9d9f4835ce12d853ff16f + # hello + """.trimIndent()) + ) + } +} diff --git a/korlibs-simple/src/korlibs/simple/Simple.kt b/korlibs-simple/src/korlibs/simple/Simple.kt deleted file mode 100644 index 8349a68..0000000 --- a/korlibs-simple/src/korlibs/simple/Simple.kt +++ /dev/null @@ -1,4 +0,0 @@ -package korlibs.simple - -class Simple { -} \ No newline at end of file