-
Notifications
You must be signed in to change notification settings - Fork 70
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add support for jsonUnknown trait #1574
Merged
Baccata
merged 9 commits into
disneystreaming:series/0.18
from
benoitlouy:support-jsonunknown-trait
Aug 23, 2024
Merged
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit
Hold shift + click to select a range
0c06186
add support for jsonUnknown trait
benoitlouy 4d44a97
split into 2 different struct JCodec implementations
benoitlouy 8d33364
use Document.Decoder to convert the unknown values to the expected type
benoitlouy 6962e61
remove useless val
benoitlouy ac62ec9
use in.decodeError
benoitlouy 72fd3fe
encode jsonUnknown field as a Document when inlining its values
benoitlouy f856658
update alloy to 0.3.13
benoitlouy eb8dad1
fix compilation
benoitlouy f0eb736
fix failing plugin test
benoitlouy File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
modules/bootstrapped/src/generated/smithy4s/example/AdditionalProperties.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package smithy4s.example | ||
|
||
import smithy4s.Document | ||
import smithy4s.Hints | ||
import smithy4s.Newtype | ||
import smithy4s.Schema | ||
import smithy4s.ShapeId | ||
import smithy4s.schema.Schema.bijection | ||
import smithy4s.schema.Schema.document | ||
import smithy4s.schema.Schema.map | ||
import smithy4s.schema.Schema.string | ||
|
||
object AdditionalProperties extends Newtype[Map[String, Document]] { | ||
val id: ShapeId = ShapeId("smithy4s.example", "AdditionalProperties") | ||
val hints: Hints = Hints.empty | ||
val underlyingSchema: Schema[Map[String, Document]] = map(string, document).withId(id).addHints(hints) | ||
implicit val schema: Schema[AdditionalProperties] = bijection(underlyingSchema, asBijection) | ||
} |
27 changes: 27 additions & 0 deletions
27
modules/bootstrapped/src/generated/smithy4s/example/JsonUnknownExample.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package smithy4s.example | ||
|
||
import smithy4s.Document | ||
import smithy4s.Hints | ||
import smithy4s.Schema | ||
import smithy4s.ShapeId | ||
import smithy4s.ShapeTag | ||
import smithy4s.schema.Schema.int | ||
import smithy4s.schema.Schema.string | ||
import smithy4s.schema.Schema.struct | ||
|
||
final case class JsonUnknownExample(s: Option[String] = None, i: Option[Int] = None, additionalProperties: Option[Map[String, Document]] = None) | ||
|
||
object JsonUnknownExample extends ShapeTag.Companion[JsonUnknownExample] { | ||
val id: ShapeId = ShapeId("smithy4s.example", "JsonUnknownExample") | ||
|
||
val hints: Hints = Hints.empty | ||
|
||
// constructor using the original order from the spec | ||
private def make(s: Option[String], i: Option[Int], additionalProperties: Option[Map[String, Document]]): JsonUnknownExample = JsonUnknownExample(s, i, additionalProperties) | ||
|
||
implicit val schema: Schema[JsonUnknownExample] = struct( | ||
string.optional[JsonUnknownExample]("s", _.s), | ||
int.optional[JsonUnknownExample]("i", _.i), | ||
AdditionalProperties.underlyingSchema.optional[JsonUnknownExample]("additionalProperties", _.additionalProperties).addHints(alloy.JsonUnknown()), | ||
)(make).withId(id).addHints(hints) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -94,7 +94,8 @@ object JsoniterCodecCompiler { | |
DiscriminatedUnionMember, | ||
Default, | ||
Required, | ||
Nullable | ||
Nullable, | ||
JsonUnknown | ||
) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -26,6 +26,7 @@ import com.github.plokhotnyuk.jsoniter_scala.core.JsonWriter | |
import smithy.api.JsonName | ||
import smithy.api.TimestampFormat | ||
import alloy.Discriminated | ||
import alloy.JsonUnknown | ||
import alloy.Nullable | ||
import alloy.Untagged | ||
import smithy4s.internals.DiscriminatedUnionMember | ||
|
@@ -1366,6 +1367,9 @@ private[smithy4s] class SchemaVisitorJCodec( | |
case Some(x) => x.value | ||
} | ||
|
||
private def isForJsonUnknown[Z, A](field: Field[Z, A]): Boolean = | ||
field.hints.has(JsonUnknown) | ||
|
||
private type Handler = (Cursor, JsonReader, util.HashMap[String, Any]) => Unit | ||
|
||
private def fieldHandler[Z, A]( | ||
|
@@ -1384,27 +1388,44 @@ private[smithy4s] class SchemaVisitorJCodec( | |
) | ||
} | ||
|
||
private def writeLabel(label: String, out: JsonWriter): Unit = | ||
if (label.forall(JsonWriter.isNonEscapedAscii)) { | ||
out.writeNonEscapedAsciiKey(label) | ||
} else out.writeKey(label) | ||
|
||
private def fieldEncoder[Z, A]( | ||
field: Field[Z, A] | ||
): (Z, JsonWriter) => Unit = { | ||
val codec = apply(field.schema) | ||
val jLabel = jsonLabel(field) | ||
val writeLabel: JsonWriter => Unit = | ||
if (jLabel.forall(JsonWriter.isNonEscapedAscii)) { | ||
_.writeNonEscapedAsciiKey(jLabel) | ||
} else _.writeKey(jLabel) | ||
|
||
if (explicitDefaultsEncoding) { (z: Z, out: JsonWriter) => | ||
writeLabel(out) | ||
writeLabel(jLabel, out) | ||
codec.encodeValue(field.get(z), out) | ||
} else { (z: Z, out: JsonWriter) => | ||
field.foreachUnlessDefault(z) { (a: A) => | ||
writeLabel(out) | ||
writeLabel(jLabel, out) | ||
codec.encodeValue(a, out) | ||
} | ||
} | ||
} | ||
|
||
private def jsonUnknownFieldEncoder[Z, A]( | ||
field: Field[Z, A] | ||
): (Z, JsonWriter) => Unit = { (z: Z, out: JsonWriter) => | ||
field.foreachUnlessDefault(z) { | ||
case m: Map[_, _] => | ||
m.foreach { case (label: String, value: Document) => | ||
writeLabel(label, out) | ||
documentJCodec.encodeValue(value, out) | ||
} | ||
case Some(m: Map[_, _]) => | ||
m.foreach { case (label: String, value: Document) => | ||
writeLabel(label, out) | ||
documentJCodec.encodeValue(value, out) | ||
} | ||
} | ||
} | ||
|
||
private type Fields[Z] = Vector[Field[Z, _]] | ||
private type LabelledFields[Z] = Vector[(Field[Z, _], String, Any)] | ||
private def labelledFields[Z](fields: Fields[Z]): LabelledFields[Z] = | ||
|
@@ -1415,7 +1436,122 @@ private[smithy4s] class SchemaVisitorJCodec( | |
(field, jLabel, default) | ||
} | ||
|
||
private def nonPayloadStruct[Z]( | ||
private def structRetainUnknownFields[Z]( | ||
allFields: LabelledFields[Z], | ||
knownFields: LabelledFields[Z], | ||
fieldsForUnknown: LabelledFields[Z], | ||
structHints: Hints | ||
)( | ||
const: Vector[Any] => Z, | ||
encode: (Z, JsonWriter, Vector[(Z, JsonWriter) => Unit]) => Unit | ||
): JCodec[Z] = | ||
new JCodec[Z] { | ||
|
||
private val fieldForUnknownDocumentDecoders = fieldsForUnknown.map { | ||
case (field, label, _) => | ||
label -> Document.Decoder.fromSchema(field.schema) | ||
}.toMap | ||
|
||
private[this] val handlers = | ||
new util.HashMap[String, Handler](knownFields.length << 1, 0.5f) { | ||
knownFields.foreach { case (field, jLabel, _) => | ||
put(jLabel, fieldHandler(field)) | ||
} | ||
} | ||
|
||
private[this] val documentEncoders = | ||
knownFields.map(labelledField => fieldEncoder(labelledField._1)) ++ | ||
fieldsForUnknown.map(f => jsonUnknownFieldEncoder(f._1)) | ||
|
||
def expecting: String = "object" | ||
|
||
override def canBeKey = false | ||
|
||
def decodeValue(cursor: Cursor, in: JsonReader): Z = | ||
decodeValue_(cursor, in)(emptyMetadata) | ||
|
||
private def decodeValue_( | ||
cursor: Cursor, | ||
in: JsonReader | ||
): scala.collection.Map[String, Any] => Z = { | ||
val unknownValues = ListBuffer[(String, Document)]() | ||
val buffer = new util.HashMap[String, Any](handlers.size << 1, 0.5f) | ||
if (in.isNextToken('{')) { | ||
if (!in.isNextToken('}')) { | ||
in.rollbackToken() | ||
while ({ | ||
val key = in.readKeyAsString() | ||
val handler = handlers.get(key) | ||
if (handler eq null) { | ||
val value = documentJCodec.decodeValue(cursor, in) | ||
unknownValues += (key -> value) | ||
} else handler(cursor, in, buffer) | ||
in.isNextToken(',') | ||
}) () | ||
if (!in.isCurrentToken('}')) in.objectEndOrCommaError() | ||
} | ||
} else in.decodeError("Expected JSON object") | ||
|
||
// At this point, we have parsed the json and retrieved | ||
// all the values that interest us for the construction | ||
// of our domain object. | ||
// We re-order the values following the order of the schema | ||
// fields before calling the constructor. | ||
{ (meta: scala.collection.Map[String, Any]) => | ||
meta.foreach(kv => buffer.put(kv._1, kv._2)) | ||
val stage2 = new VectorBuilder[Any] | ||
val unknownValue = | ||
if (unknownValues.nonEmpty) Document.obj(unknownValues) else null | ||
|
||
allFields.foreach { case (f, jsonLabel, default) => | ||
stage2 += { | ||
fieldForUnknownDocumentDecoders.get(jsonLabel) match { | ||
case None => | ||
val value = buffer.get(f.label) | ||
if (value == null) { | ||
if (default == null) | ||
cursor.requiredFieldError(jsonLabel, jsonLabel) | ||
else default | ||
} else value | ||
|
||
case Some(docDecoder) => | ||
if (unknownValue == null) { | ||
if (default == null) { | ||
docDecoder | ||
.decode(Document.obj()) | ||
.getOrElse( | ||
throw new RuntimeException( | ||
s"${cursor.getPath(Nil)} Failed translating a Document.DObject to the type targeted by ${f.label}." | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. use |
||
) | ||
) | ||
} else default | ||
} else { | ||
docDecoder | ||
.decode(unknownValue) | ||
.getOrElse( | ||
throw new RuntimeException( | ||
s"${cursor.getPath(Nil)} Failed translating a Document.DObject to the type targeted by ${f.label}." | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
) | ||
) | ||
} | ||
} | ||
} | ||
} | ||
const(stage2.result()) | ||
} | ||
} | ||
|
||
def encodeValue(z: Z, out: JsonWriter): Unit = | ||
encode(z, out, documentEncoders) | ||
|
||
def decodeKey(in: JsonReader): Z = | ||
in.decodeError("Cannot use products as keys") | ||
|
||
def encodeKey(x: Z, out: JsonWriter): Unit = | ||
out.encodeError("Cannot use products as keys") | ||
} | ||
|
||
private def structIgnoreUnknownFields[Z]( | ||
fields: LabelledFields[Z], | ||
structHints: Hints | ||
)( | ||
|
@@ -1491,6 +1627,28 @@ private[smithy4s] class SchemaVisitorJCodec( | |
out.encodeError("Cannot use products as keys") | ||
} | ||
|
||
private def nonPayloadStruct[Z]( | ||
fields: LabelledFields[Z], | ||
structHints: Hints | ||
)( | ||
const: Vector[Any] => Z, | ||
encode: (Z, JsonWriter, Vector[(Z, JsonWriter) => Unit]) => Unit | ||
): JCodec[Z] = { | ||
val (fieldsForUnknown, knownFields) = fields.partition { | ||
case (field, _, _) => isForJsonUnknown(field) | ||
} | ||
|
||
if (fieldsForUnknown.isEmpty) | ||
structIgnoreUnknownFields(fields, structHints)(const, encode) | ||
else | ||
structRetainUnknownFields( | ||
fields, | ||
knownFields, | ||
fieldsForUnknown, | ||
structHints | ||
)(const, encode) | ||
} | ||
|
||
private def basicStruct[A, S]( | ||
fields: LabelledFields[S], | ||
structHints: Hints | ||
|
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sorry, I should have signposted this bit as well, but this is falling under the similar notion to what I was describing here :
required
translates to Scala OptionDocuments
Therefore you need to pre-compile the
Document.Encoder
and encode the field value against it, and pattern-match against the produced document to check it's a DObject, in which case you run the exact sameforeach
you have thereThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This makes sense, thanks for the directions.