From 98ce1c9c708bd61d02f40dea39b35c012f5eec06 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 16 May 2023 14:00:15 -0700 Subject: [PATCH] Proof of concept--using Kotlin in Ion Java --- build.gradle.kts | 55 +++++ config/r8/rules.txt | 6 + .../ion/{IntegerSize.java => IntegerSize.kt} | 27 +-- src/com/amazon/ion/IonExtensions.kt | 116 ++++++++++ src/com/amazon/ion/impl/IonIteratorImpl.java | 219 ------------------ src/com/amazon/ion/impl/IonIteratorImpl.kt | 99 ++++++++ test/com/amazon/ion/IonExtensionsTest.kt | 38 +++ 7 files changed, 326 insertions(+), 234 deletions(-) create mode 100644 config/r8/rules.txt rename src/com/amazon/ion/{IntegerSize.java => IntegerSize.kt} (50%) create mode 100644 src/com/amazon/ion/IonExtensions.kt delete mode 100644 src/com/amazon/ion/impl/IonIteratorImpl.java create mode 100644 src/com/amazon/ion/impl/IonIteratorImpl.kt create mode 100644 test/com/amazon/ion/IonExtensionsTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 6a637759a2..bf84748df5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,10 +4,13 @@ import java.util.Properties plugins { + kotlin("jvm") version "1.8.21" java `maven-publish` jacoco signing + id("com.github.johnrengelman.shadow") version "8.1.1" + id("org.cyclonedx.bom") version "1.7.2" id("com.github.spotbugs") version "5.0.13" // TODO: more static analysis. E.g.: @@ -16,15 +19,22 @@ plugins { repositories { mavenCentral() + google() } +val r8 = configurations.create("r8") + dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib:1.8.0") + testImplementation("org.junit.jupiter:junit-jupiter:5.7.1") testCompileOnly("junit:junit:4.13") testRuntimeOnly("org.junit.vintage:junit-vintage-engine") testImplementation("org.hamcrest:hamcrest:2.2") testImplementation("pl.pragmatists:JUnitParams:1.1.1") testImplementation("com.google.code.tempus-fugit:tempus-fugit:1.1") + + r8("com.android.tools:r8:8.0.40") } group = "com.amazon.ion" @@ -50,12 +60,47 @@ sourceSets { } } +kotlin { + jvmToolchain(8) +} + tasks { withType { options.encoding = "UTF-8" // In Java 9+ we can use `release` but for now we're still building with JDK 8, 11 } + shadowJar { + relocate("kotlin", "com.amazon.ion_.shaded.kotlin") + minimize() + } + + val minifiedJar by register("r8Jar") { + val rules = file("config/r8/rules.txt") + + inputs.file("build/libs/ion-java-$version-all.jar") + inputs.file(rules) + outputs.file("build/libs/ion-java-$version-r8.jar") + + dependsOn(shadowJar) + dependsOn(configurations.runtimeClasspath) + + classpath(r8) + mainClass.set("com.android.tools.r8.R8") + args = listOf( + "--release", + "--classfile", + "--output", "build/libs/ion-java-$version-minified.jar", + "--pg-conf", rules.toString(), + "--lib", System.getProperty("java.home").toString(), + "build/libs/ion-java-$version-all.jar", + ) + } + + build { + dependsOn(minifiedJar) + } + javadoc { // Suppressing Javadoc warnings is clunky, but there doesn't seem to be any nicer way to do it. // https://stackoverflow.com/questions/62894307/option-xdoclintnone-does-not-work-in-gradle-to-ignore-javadoc-warnings @@ -165,6 +210,16 @@ tasks { finalizedBy(jacocoTestReport) } + val testMinified by register("testMinified") { + maxHeapSize = "1g" // When this line was added Xmx 512m was the default, and we saw OOMs + maxParallelForks = Math.max(1, Runtime.getRuntime().availableProcessors() / 2) + group = "verification" + testClassesDirs = project.sourceSets.test.get().output.classesDirs + classpath = project.sourceSets.test.get().runtimeClasspath + minifiedJar.outputs.files + dependsOn(minifiedJar) + useJUnitPlatform() + } + withType { setOnlyIf { isReleaseVersion && gradle.taskGraph.hasTask(":publish") } } diff --git a/config/r8/rules.txt b/config/r8/rules.txt new file mode 100644 index 0000000000..744b81af4e --- /dev/null +++ b/config/r8/rules.txt @@ -0,0 +1,6 @@ +-keepattributes SourceFile, LineNumberTable + +-keep class com.amazon.ion.*** +-keep interface com.amazon.ion.*** +-keep enum com.amazon.ion.*** +-keeppackagenames *** diff --git a/src/com/amazon/ion/IntegerSize.java b/src/com/amazon/ion/IntegerSize.kt similarity index 50% rename from src/com/amazon/ion/IntegerSize.java rename to src/com/amazon/ion/IntegerSize.kt index 7ced059a0f..87b60a5cf6 100644 --- a/src/com/amazon/ion/IntegerSize.java +++ b/src/com/amazon/ion/IntegerSize.kt @@ -1,5 +1,5 @@ /* - * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -12,33 +12,30 @@ * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ - -package com.amazon.ion; +package com.amazon.ion /** - * Indicates the smallest-possible Java type of an Ion {@code int} value. + * Indicates the smallest-possible Java type for an Ion `int` value. */ -public enum IntegerSize -{ +enum class IntegerSize { /** - * Fits in the Java {@code int} primitive (four bytes). - * The value can be retrieved through methods like {@link IonReader#intValue()} - * or {@link IonInt#intValue()} without data loss. + * Fits in the Java `int` primitive (four bytes). + * The value can be retrieved through methods like [IonReader.intValue] + * or [IonInt.intValue] without data loss. */ INT, /** - * Fits in the Java {@code int} primitive (eight bytes). - * The value can be retrieved through methods like {@link IonReader#longValue()} - * or {@link IonInt#longValue()} without data loss. + * Fits in the Java `int` primitive (eight bytes). + * The value can be retrieved through methods like [IonReader.longValue] + * or [IonInt.longValue] without data loss. */ LONG, /** * Larger than eight bytes. This value can be retrieved through methods like - * {@link IonReader#bigIntegerValue()} or {@link IonInt#bigIntegerValue()} + * [IonReader.bigIntegerValue] or [IonInt.bigIntegerValue] * without data loss. */ - BIG_INTEGER, - + BIG_INTEGER } diff --git a/src/com/amazon/ion/IonExtensions.kt b/src/com/amazon/ion/IonExtensions.kt new file mode 100644 index 0000000000..ce61f2e68e --- /dev/null +++ b/src/com/amazon/ion/IonExtensions.kt @@ -0,0 +1,116 @@ +package com.amazon.ion + +import java.math.BigInteger +import java.time.Instant + +/** + * Returns `this` value, without annotations. + * If this value has no annotations, returns `this`; + * otherwise, returns a clone of `this` with the annotations removed. + */ +inline fun T.withoutTypeAnnotations(): T = + if (typeAnnotations.isNotEmpty()) { + clone().apply { clearTypeAnnotations() } as T + } else { + this + } + +/** + * Makes an IonValue instance read-only. + */ +fun T.markReadOnly(): T { + this.makeReadOnly() + return this +} + +/** + * Create a [Timestamp] from [Instant] with millisecond precision and UTC offset. + */ +fun Instant.toTimestamp(): Timestamp = Timestamp.forMillis(this.toEpochMilli(), 0) + +/** + * Convert an [Timestamp] into an [Instant], truncating any fractional seconds smaller than millisecond precision. + */ +fun Timestamp.toInstant(): Instant = Instant.ofEpochMilli(this.millis) + +/** + * Kotlin-friendly extension function for building a new [IonStruct]. + */ +inline fun ValueFactory.buildStruct(init: IonStruct.() -> Unit): IonStruct = newEmptyStruct().apply(init) + +/** + * Kotlin-friendly extension function for building a new [IonSexp]. + */ +inline fun ValueFactory.buildSexp(init: IonSexp.() -> Unit): IonSexp = newEmptySexp().apply(init) + +/** + * Kotlin-friendly extension function for building a new [IonList]. + */ +inline fun ValueFactory.buildList(init: IonList.() -> Unit): IonList = newEmptyList().apply(init) + +/** + * Returns the value of an [IonString] or [IonSymbol], or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.stringValueOrNull(): String? = (this as? IonText)?.stringValue() + +/** + * Returns the value of an [IonBlob] or [IonClob], or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.bytesValueOrNull(): ByteArray? = (this as? IonLob)?.bytes + +/** + * Returns the value of an [IonBool] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.boolValueOrNull(): Boolean? = (this as? IonBool)?.booleanValue() + +/** + * Returns the value of an [IonInt] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.bigIntValueOrNull(): BigInteger? = (this as? IonInt)?.bigIntegerValue() + +/** + * Returns the value of an [IonFloat] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.doubleValueOrNull(): Double? = (this as? IonFloat)?.doubleValue() + +/** + * Returns the value of an [IonDecimal] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.decimalValueOrNull(): Decimal? = (this as? IonDecimal)?.decimalValue() + +/** + * Returns the value of an [IonTimestamp] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.timestampValueOrNull(): Timestamp? = (this as? IonTimestamp)?.timestampValue() + +/** + * Returns the child values of an [IonList] or [IonSexp] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.childValuesOrNull(): List? = (this as? IonSequence)?.takeIf { !it.isNullValue } + +/** + * Returns the fields of an [IonStruct] or `null` if this [IonValue] is another type or an Ion null. + */ +fun IonValue.fieldsOrNull(): List>? = (this as? IonStruct)?.takeIf { !it.isNullValue }?.map { it.fieldName to it } + +/** + * If this [IonValue] is a container type, returns an iterator over the contained values. Otherwise returns an iterator + * containing this [IonValue]. + */ +fun IonValue.asIterator(): Iterator { + return when (this) { + is IonContainer -> iterator() + else -> SingletonIterator(this) + } +} + +/** + * An iterator over one value. + */ +private class SingletonIterator(value: E): Iterator { + private var nextValue: E? = value + + override fun hasNext(): Boolean = nextValue != null + + override fun next(): E = nextValue.also { nextValue = null } ?: throw NoSuchElementException() +} diff --git a/src/com/amazon/ion/impl/IonIteratorImpl.java b/src/com/amazon/ion/impl/IonIteratorImpl.java deleted file mode 100644 index 2083575f07..0000000000 --- a/src/com/amazon/ion/impl/IonIteratorImpl.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.amazon.ion.impl; - -import com.amazon.ion.IonLob; -import com.amazon.ion.IonReader; -import com.amazon.ion.IonSequence; -import com.amazon.ion.IonStruct; -import com.amazon.ion.IonType; -import com.amazon.ion.IonValue; -import com.amazon.ion.SymbolTable; -import com.amazon.ion.SymbolToken; -import com.amazon.ion.ValueFactory; -import java.util.Iterator; -import java.util.NoSuchElementException; - - -final class IonIteratorImpl - implements Iterator -{ - private final ValueFactory _valueFactory; - private final IonReader _reader; - private boolean _at_eof; - private IonValue _curr; - private IonValue _next; - - - - /** - * @throws NullPointerException if any parameter is null. - */ - public IonIteratorImpl(ValueFactory valueFactory, - IonReader input) - { - if (valueFactory == null || input == null) - { - throw new NullPointerException(); - } - - _valueFactory = valueFactory; - _reader = input; - } - - - /** - * Returns true if the iteration has more elements. - * here we actually walk ahead and get the next value (it's - * the only way we know if there are more and clear out the - * various $ion noise out of the way - */ - public boolean hasNext() { - if (_at_eof) return false; - if (_next != null) return true; - return (prefetch() != null); - } - - private IonValue prefetch() - { - assert !_at_eof && _next == null; - - IonType type = _reader.next(); - if (type == null) - { - _at_eof = true; - } - else - { - _next = readValue(); - } - - return _next; - } - - - private IonValue readValue() - { - IonType type = _reader.getType(); - - SymbolToken[] annotations = _reader.getTypeAnnotationSymbols(); - - IonValue v; - - if (_reader.isNullValue()) - { - v = _valueFactory.newNull(type); - } - else - { - switch (type) { - case NULL: - // Handled above - throw new IllegalStateException(); - case BOOL: - v = _valueFactory.newBool(_reader.booleanValue()); - break; - case INT: - v = _valueFactory.newInt(_reader.bigIntegerValue()); - break; - case FLOAT: - v = _valueFactory.newFloat(_reader.doubleValue()); - break; - case DECIMAL: - v = _valueFactory.newDecimal(_reader.decimalValue()); - break; - case TIMESTAMP: - v = _valueFactory.newTimestamp(_reader.timestampValue()); - break; - case STRING: - v = _valueFactory.newString(_reader.stringValue()); - break; - case SYMBOL: - // TODO always pass the SID? Is it correct? - v = _valueFactory.newSymbol(_reader.symbolValue()); - break; - case BLOB: - { - IonLob lob = _valueFactory.newNullBlob(); - lob.setBytes(_reader.newBytes()); - v = lob; - break; - } - case CLOB: - { - IonLob lob = _valueFactory.newNullClob(); - lob.setBytes(_reader.newBytes()); - v = lob; - break; - } - case STRUCT: - { - IonStruct struct = _valueFactory.newEmptyStruct(); - _reader.stepIn(); - while (_reader.next() != null) - { - SymbolToken name = _reader.getFieldNameSymbol(); - IonValue child = readValue(); - struct.add(name, child); - } - _reader.stepOut(); - v = struct; - break; - } - case LIST: - { - IonSequence seq = _valueFactory.newEmptyList(); - _reader.stepIn(); - while (_reader.next() != null) - { - IonValue child = readValue(); - seq.add(child); - } - _reader.stepOut(); - v = seq; - break; - } - case SEXP: - { - IonSequence seq = _valueFactory.newEmptySexp(); - _reader.stepIn(); - while (_reader.next() != null) - { - IonValue child = readValue(); - seq.add(child); - } - _reader.stepOut(); - v = seq; - break; - } - default: - throw new IllegalStateException(); - } - } - - // TODO this is too late in the case of system reading - // when v is a local symtab (it will get itself, not the prior symtab) - SymbolTable symtab = _reader.getSymbolTable(); - ((_Private_IonValue)v).setSymbolTable(symtab); - - if (annotations.length != 0) { - ((_Private_IonValue)v).setTypeAnnotationSymbols(annotations); - } - - return v; - } - - - public IonValue next() { - if (! _at_eof) { - _curr = null; - if (_next == null) { - prefetch(); - } - if (_next != null) { - _curr = _next; - _next = null; - return _curr; - } - } - throw new NoSuchElementException(); - } - - - public void remove() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/com/amazon/ion/impl/IonIteratorImpl.kt b/src/com/amazon/ion/impl/IonIteratorImpl.kt new file mode 100644 index 0000000000..f6361473cf --- /dev/null +++ b/src/com/amazon/ion/impl/IonIteratorImpl.kt @@ -0,0 +1,99 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazon.ion.impl + +import com.amazon.ion.IonReader +import com.amazon.ion.IonType +import com.amazon.ion.IonValue +import com.amazon.ion.ValueFactory +import com.amazon.ion.buildList +import com.amazon.ion.buildSexp +import com.amazon.ion.buildStruct + +internal class IonIteratorImpl(private val valueFactory: ValueFactory, private val reader: IonReader) : MutableIterator { + + private var _at_eof = false + private var _next: IonValue? = null + + /** + * Returns true if the iteration has more elements. + * here we actually walk ahead and get the next value (it's + * the only way we know if there are more and clear out the + * various $ion noise out of the way + */ + override fun hasNext(): Boolean { + if (_at_eof) return false + return if (_next != null) true else prefetch() != null + } + + private fun prefetch(): IonValue? { + assert(!_at_eof && _next == null) + val type = reader.next() + if (type == null) { + _at_eof = true + } else { + _next = reader.readValue() + } + return _next + } + + private fun IonReader.readValue(): IonValue { + val annotations = reader.typeAnnotationSymbols + val v = if (reader.isNullValue) { + valueFactory.newNull(reader.type) + } else { + when (reader.type) { + IonType.NULL -> throw IllegalStateException() + IonType.BOOL -> valueFactory.newBool(booleanValue()) + IonType.INT -> valueFactory.newInt(bigIntegerValue()) + IonType.FLOAT -> valueFactory.newFloat(doubleValue()) + IonType.DECIMAL -> valueFactory.newDecimal(decimalValue()) + IonType.TIMESTAMP -> valueFactory.newTimestamp(timestampValue()) + IonType.STRING -> valueFactory.newString(stringValue()) + // TODO always pass the SID? Is it correct? + IonType.SYMBOL -> valueFactory.newSymbol(symbolValue()) + IonType.BLOB -> valueFactory.newNullBlob().apply { bytes = newBytes() } + IonType.CLOB -> valueFactory.newNullClob().apply { bytes = newBytes() } + IonType.STRUCT -> valueFactory.buildStruct { forEachInContainer { add(fieldNameSymbol, readValue()) } } + IonType.LIST -> valueFactory.buildList { forEachInContainer { add(readValue()) } } + IonType.SEXP -> valueFactory.buildSexp { forEachInContainer { add(readValue()) } } + else -> throw IllegalStateException() + } + } + + // TODO this is too late in the case of system reading + // when v is a local symtab (it will get itself, not the prior symtab) + (v as _Private_IonValue).symbolTable = reader.symbolTable + if (annotations.isNotEmpty()) v.setTypeAnnotationSymbols(*annotations) + return v + } + + override fun next(): IonValue { + if (_at_eof) throw NoSuchElementException() + val value = _next ?: prefetch() + _next = null + return value ?: throw NoSuchElementException() + } + + override fun remove() { + throw UnsupportedOperationException() + } + + private inline fun IonReader.forEachInContainer(block: IonReader.() -> Unit) { + stepIn() + while (next() != null) { block() } + stepOut() + } +} \ No newline at end of file diff --git a/test/com/amazon/ion/IonExtensionsTest.kt b/test/com/amazon/ion/IonExtensionsTest.kt new file mode 100644 index 0000000000..0f88cf87d3 --- /dev/null +++ b/test/com/amazon/ion/IonExtensionsTest.kt @@ -0,0 +1,38 @@ +package com.amazon.ion + +import com.amazon.ion.system.IonSystemBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.math.BigInteger + +class IonExtensionsTest { + + val ION = IonSystemBuilder.standard().build() + + @Test + fun `asIterator() should iterate over child elements`() { + val ionValue = ION.singleValue("[1, a, false]") + val itr = ionValue.asIterator() + assertTrue(itr.hasNext()) + assertEquals(BigInteger.ONE, itr.next().bigIntValueOrNull()) + assertTrue(itr.hasNext()) + assertEquals("a", itr.next().stringValueOrNull()) + assertTrue(itr.hasNext()) + assertEquals(false, itr.next().boolValueOrNull()) + assertFalse(itr.hasNext()) + assertThrows { itr.next() } + } + + @Test + fun `asIterator() should return an iterator over self if this is not a container`() { + val ionValue = ION.singleValue("a") + val itr = ionValue.asIterator() + assertTrue(itr.hasNext()) + assertEquals("a", itr.next().stringValueOrNull()) + assertFalse(itr.hasNext()) + assertThrows { itr.next() } + } +} \ No newline at end of file