From cb30a46514503b325f8df97ce66720c9494969a6 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Thu, 19 Sep 2024 22:24:42 +0900 Subject: [PATCH 01/23] Fix readme --- upickleReadme/Readme.scalatex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upickleReadme/Readme.scalatex b/upickleReadme/Readme.scalatex index 4ac130656..92985db43 100644 --- a/upickleReadme/Readme.scalatex +++ b/upickleReadme/Readme.scalatex @@ -918,7 +918,7 @@ @sect{4.0.1} @ul @li - Respect @{Config#tagName} in @code{Reader.merge} and @code{ReadWriter.merge} @lnk("#619", "https://github.com/com-lihaoyi/upickle/pull/619") + Respect @code{Config#tagName} in @code{Reader.merge} and @code{ReadWriter.merge} @lnk("#619", "https://github.com/com-lihaoyi/upickle/pull/619") @sect{4.0.0} @ul @li From b50cf991adc20fd582467b514ed3712fa73c4d32 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Wed, 25 Sep 2024 22:36:39 +0900 Subject: [PATCH 02/23] wip --- .../upickle/implicits/internal/Macros.scala | 2 +- .../implicits/src/upickle/implicits/key.scala | 2 ++ upickle/test/src/upickle/MacroTests.scala | 31 +++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala index 67d3942c1..88e8ee8a3 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -177,7 +177,7 @@ object Macros { t.substituteTypes(typeParams, concrete) } else { - val TypeRef(pref, sym, args) = typeOf[Seq[Int]] + val TypeRef(pref, sym, _) = typeOf[Seq[Int]] import compat._ TypeRef(pref, sym, t.asInstanceOf[TypeRef].args) } diff --git a/upickle/implicits/src/upickle/implicits/key.scala b/upickle/implicits/src/upickle/implicits/key.scala index 3708bae11..36c013d71 100644 --- a/upickle/implicits/src/upickle/implicits/key.scala +++ b/upickle/implicits/src/upickle/implicits/key.scala @@ -29,3 +29,5 @@ class serializeDefaults(s: Boolean) extends StaticAnnotation * over upickle pickler-level configuration */ class allowUnknownKeys(b: Boolean) extends StaticAnnotation + +class flatten extends StaticAnnotation diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 2936ae1f1..22cc57a67 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -139,13 +139,35 @@ object TagName{ implicit val quxRw: TagNamePickler.ReadWriter[Qux] = TagNamePickler.macroRW implicit val fooRw: TagNamePickler.ReadWriter[Foo] = TagNamePickler.macroRW } + +case class Pagination(limit: Int, offset: Int, total: Int) + +object Pagination { + implicit val rw: RW[Pagination] = upickle.default.macroRW +} + +case class Users(Ids: List[Int], @upickle.implicits.flatten pagination: Pagination) + +object Users { + implicit val rw: RW[Users] = upickle.default.macroRW +} + +case class PackageManifest( + name: String, + @upickle.implicits.flatten otherStuff: Map[String, ujson.Value] + ) + +object PackageManifest { + implicit val rw: RW[PackageManifest] = upickle.default.macroRW +} + object MacroTests extends TestSuite { // Doesn't work :( // case class A_(objects: Option[C_]); case class C_(nodes: Option[C_]) // implicitly[Reader[A_]] -// implicitly[upickle.old.Writer[upickle.MixedIn.Obj.ClsB]] +// implicitly[upickle.old.Writer[upickle.MixedIn.Obj.ClsB]code] // println(write(ADTs.ADTc(1, "lol", (1.1, 1.2)))) // implicitly[upickle.old.Writer[ADTs.ADTc]] @@ -833,7 +855,6 @@ object MacroTests extends TestSuite { val customPicklerTest = new TestUtil(customPickler) - implicit def rwA: customPickler.ReadWriter[upickle.Hierarchy.A] = customPickler.macroRW implicit def rwB: customPickler.ReadWriter[upickle.Hierarchy.B] = customPickler.macroRW implicit def rwC: customPickler.ReadWriter[upickle.Hierarchy.C] = customPickler.macroRW @@ -872,5 +893,11 @@ object MacroTests extends TestSuite { ) } + + test("flatten"){ + val a = Users(List(1, 2, 3), Pagination(10, 20, 30)) + upickle.default.write[Users](a) ==> """{"Ids":[1,2,3],"limit":10,"offset":20,"total":30}""" + } + } } From 1ecc8dfaecce605eb62d2e7334fb1ea30e8c23a2 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Tue, 15 Oct 2024 21:56:30 +0900 Subject: [PATCH 03/23] sqaush wip make scala2 works Delete some files Add more tests more tests scala3 writer fix polish clean up Add helper function scala3 macro readers fix bug test pass polish polish wip fix remove unused Use ListBuffer to preserve orders polish Keep existing macros for bincompat Keep formatting --- .../upickle/implicits/MacroImplicits.scala | 4 +- .../upickle/implicits/internal/Macros.scala | 7 +- .../upickle/implicits/internal/Macros2.scala | 577 ++++++++++++++++++ .../src-3/upickle/implicits/Readers.scala | 59 +- .../src-3/upickle/implicits/Writers.scala | 4 +- .../src-3/upickle/implicits/macros.scala | 380 +++++++++--- .../upickle/implicits/ObjectContexts.scala | 13 +- .../implicits/src/upickle/implicits/key.scala | 12 + upickle/test/src/upickle/FailureTests.scala | 8 + upickle/test/src/upickle/MacroTests.scala | 102 +++- 10 files changed, 1036 insertions(+), 130 deletions(-) create mode 100644 upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala diff --git a/upickle/implicits/src-2/upickle/implicits/MacroImplicits.scala b/upickle/implicits/src-2/upickle/implicits/MacroImplicits.scala index 92ac78f93..c83d11ed4 100644 --- a/upickle/implicits/src-2/upickle/implicits/MacroImplicits.scala +++ b/upickle/implicits/src-2/upickle/implicits/MacroImplicits.scala @@ -43,7 +43,7 @@ trait MacroImplicits extends MacrosCommon { this: upickle.core.Types => def macroW[T]: Writer[T] = macro MacroImplicits.applyW[T] def macroRW[T]: ReadWriter[T] = macro MacroImplicits.applyRW[ReadWriter[T]] - def macroR0[T, M[_]]: Reader[T] = macro internal.Macros.macroRImpl[T, M] - def macroW0[T, M[_]]: Writer[T] = macro internal.Macros.macroWImpl[T, M] + def macroR0[T, M[_]]: Reader[T] = macro internal.Macros2.macroRImpl[T, M] + def macroW0[T, M[_]]: Writer[T] = macro internal.Macros2.macroWImpl[T, M] } diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala index 52350838f..3236716c3 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -10,6 +10,11 @@ import upickle.implicits.{MacrosCommon, key} import language.higherKinds import language.existentials +/** + * This file is deprecated and remained here for binary compatibility. + * Please use upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala instead. + */ + /** * Implementation of macros used by uPickle to serialize and deserialize * case classes automatically. You probably shouldn't need to use these @@ -177,7 +182,7 @@ object Macros { t.substituteTypes(typeParams, concrete) } else { - val TypeRef(pref, sym, _) = typeOf[Seq[Int]] + val TypeRef(pref, sym, args) = typeOf[Seq[Int]] import compat._ TypeRef(pref, sym, t.asInstanceOf[TypeRef].args) } diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala new file mode 100644 index 000000000..0682adb92 --- /dev/null +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -0,0 +1,577 @@ +package upickle.implicits.internal + +import scala.annotation.{nowarn, StaticAnnotation} +import scala.language.experimental.macros +import compat._ + +import acyclic.file +import upickle.core.Annotator +import upickle.implicits.{MacrosCommon, flatten, key} +import language.higherKinds +import language.existentials + +/** + * Implementation of macros used by uPickle to serialize and deserialize + * case classes automatically. You probably shouldn't need to use these + * directly, since they are called implicitly when trying to read/write + * types you don't have a Reader/Writer in scope for. + */ +@nowarn("cat=deprecation") +object Macros2 { + + trait DeriveDefaults[M[_]] { + val c: scala.reflect.macros.blackbox.Context + private def fail(tpe: c.Type, s: String) = c.abort(c.enclosingPosition, s) + + import c.universe._ + private def companionTree(tpe: c.Type): Tree = { + val companionSymbol = tpe.typeSymbol.companionSymbol + + if (companionSymbol == NoSymbol && tpe.typeSymbol.isClass) { + val clsSymbol = tpe.typeSymbol.asClass + val msg = "[error] The companion symbol could not be determined for " + + s"[[${clsSymbol.name}]]. This may be due to a bug in scalac (SI-7567) " + + "that arises when a case class within a function is upickle. As a " + + "workaround, move the declaration to the module-level." + fail(tpe, msg) + } else { + val symTab = c.universe.asInstanceOf[reflect.internal.SymbolTable] + val pre = tpe.asInstanceOf[symTab.Type].prefix.asInstanceOf[Type] + c.universe.treeBuild.mkAttributedRef(pre, companionSymbol) + } + + } + + /** + * If a super-type is generic, find all the subtypes, but at the same time + * fill in all the generic type parameters that are based on the super-type's + * concrete type + */ + private def fleshedOutSubtypes(tpe: Type) = { + for{ + subtypeSym <- tpe.typeSymbol.asClass.knownDirectSubclasses.filter(!_.toString.contains("")) + if subtypeSym.isType + st = subtypeSym.asType.toType + baseClsArgs = st.baseType(tpe.typeSymbol).asInstanceOf[TypeRef].args + } yield { + tpe match{ + case ExistentialType(_, TypeRef(pre, sym, args)) => + st.substituteTypes(baseClsArgs.map(_.typeSymbol), args) + case ExistentialType(_, _) => st + case TypeRef(pre, sym, args) => + st.substituteTypes(baseClsArgs.map(_.typeSymbol), args) + } + } + } + + private def deriveObject(tpe: c.Type) = { + val mod = tpe.typeSymbol.asClass.module + val symTab = c.universe.asInstanceOf[reflect.internal.SymbolTable] + val pre = tpe.asInstanceOf[symTab.Type].prefix.asInstanceOf[Type] + val mod2 = c.universe.treeBuild.mkAttributedRef(pre, mod) + + annotate(tpe)(wrapObject(mod2)) + + } + + private[upickle] def mergeTrait(tagKey: Option[String], subtrees: Seq[Tree], subtypes: Seq[Type], targetType: c.Type): Tree + + private[upickle] def derive(tpe: c.Type) = { + if (tpe.typeSymbol.asClass.isTrait || (tpe.typeSymbol.asClass.isAbstractClass && !tpe.typeSymbol.isJava)) { + val derived = deriveTrait(tpe) + derived + } + else if (tpe.typeSymbol.isModuleClass) deriveObject(tpe) + else deriveClass(tpe) + } + + private def deriveTrait(tpe: c.Type): c.universe.Tree = { + val clsSymbol = tpe.typeSymbol.asClass + + if (!clsSymbol.isSealed) { + fail(tpe, s"[error] The referenced trait [[${clsSymbol.name}]] must be sealed.") + }else if (clsSymbol.knownDirectSubclasses.filter(!_.toString.contains("")).isEmpty) { + val msg = + s"The referenced trait [[${clsSymbol.name}]] does not have any sub-classes. This may " + + "happen due to a limitation of scalac (SI-7046). To work around this, " + + "try manually specifying the sealed trait picklers as described in " + + "https://com-lihaoyi.github.io/upickle/#ManualSealedTraitPicklers" + fail(tpe, msg) + }else{ + val tagKey = customKey(clsSymbol) + val subTypes = fleshedOutSubtypes(tpe).toSeq.sortBy(_.typeSymbol.fullName) + // println("deriveTrait") + val subDerives = subTypes.map(subCls => q"implicitly[${typeclassFor(subCls)}]") + // println(Console.GREEN + "subDerives " + Console.RESET + subDrivess) + val merged = mergeTrait(tagKey, subDerives, subTypes, tpe) + merged + } + } + + private[upickle] def typeclass: c.WeakTypeTag[M[_]] + + private def typeclassFor(t: Type) = { + // println("typeclassFor " + weakTypeOf[M[_]](typeclass)) + + weakTypeOf[M[_]](typeclass) match { + case TypeRef(a, b, _) => + import compat._ + TypeRef(a, b, List(t)) + case ExistentialType(_, TypeRef(a, b, _)) => + import compat._ + TypeRef(a, b, List(t)) + case x => + println("Dunno Wad Dis Typeclazz Is " + x) + println(x) + println(x.getClass) + ??? + } + } + + sealed trait Flatten + + object Flatten { + case class Class(companion: Tree, fields: List[Field], varArgs: Boolean) extends Flatten + case object Map extends Flatten + case object None extends Flatten + } + + case class Field( + name: String, + mappedName: String, + tpe: Type, + symbol: Symbol, + defaultValue: Option[Tree], + flatten: Flatten, + ) { + lazy val allFields: List[Field] = { + def loop(field: Field): List[Field] = + field.flatten match { + case Flatten.Class(_, fields, _) => fields.flatMap(loop) + case Flatten.Map => List(field) + case Flatten.None => List(field) + } + loop(this) + } + } + + private def getFields(tpe: c.Type): (c.Tree, List[Field], Boolean) = { + def applyTypeArguments(t: c.Type ): c.Type = { + val typeParams = tpe.typeSymbol.asClass.typeParams + val typeArguments = tpe.normalize.asInstanceOf[TypeRef].args + if (t.typeSymbol != definitions.RepeatedParamClass) { + t.substituteTypes(typeParams, typeArguments) + } else { + val TypeRef(pref, sym, _) = typeOf[Seq[Int]] + internal.typeRef(pref, sym, t.asInstanceOf[TypeRef].args) + } + } + + val companion = companionTree(tpe) + //tickle the companion members -- Not doing this leads to unexpected runtime behavior + //I wonder if there is an SI related to this? + companion.tpe.members.foreach(_ => ()) + tpe.members.find(x => x.isMethod && x.asMethod.isPrimaryConstructor) match { + case None => fail(tpe, "Can't find primary constructor of " + tpe) + case Some(primaryConstructor) => + val params = primaryConstructor.asMethod.paramLists.flatten + val varArgs = params.lastOption.exists(_.typeSignature.typeSymbol == definitions.RepeatedParamClass) + val fields = params.zipWithIndex.map { case (param, i) => + val name = param.name.decodedName.toString + val mappedName = customKey(param).getOrElse(name) + val tpeOfField = applyTypeArguments(param.typeSignature) + val defaultValue = if (param.asTerm.isParamWithDefault) + Some(q"$companion.${TermName("apply$default$" + (i + 1))}") + else + None + val flatten = param.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { + case Some(_) => + if (tpeOfField.typeSymbol == typeOf[collection.immutable.Map[_, _]].typeSymbol) Flatten.Map + else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { + val (nestedCompanion, fields, nestedVarArgs) = getFields(tpeOfField) + Flatten.Class(nestedCompanion, fields, nestedVarArgs) + } + else fail(tpeOfField, + s"""Invalid type for flattening: $tpeOfField. + | Flatten only works on case classes and Maps""".stripMargin) + case None => + Flatten.None + } + Field(param.name.toString, mappedName, tpeOfField, param, defaultValue, flatten) + } + (companion, fields, varArgs) + } + } + + private def deriveClass(tpe: c.Type) = { + val (companion, fields, varArgs) = getFields(tpe) + // According to @retronym, this is necessary in order to force the + // default argument `apply$default$n` methods to be synthesized + companion.tpe.member(TermName("apply")).info + + val allFields = fields.flatMap(_.allFields) + validateFlattenAnnotation(allFields) + + val derive = + // Otherwise, reading and writing are kinda identical + wrapCaseN( + companion, + fields, + varArgs, + targetType = tpe, + ) + + annotate(tpe)(derive) + } + + private def validateFlattenAnnotation(fields: List[Field]): Unit = { + if (fields.count(_.flatten == Flatten.Map) > 1) { + fail(NoType, "Only one Map can be annotated with @upickle.implicits.flatten in the same level") + } + if (fields.map(_.mappedName).distinct.length != fields.length) { + fail(NoType, "There are multiple fields with the same key") + } + if (fields.exists(field => field.flatten == Flatten.Map && !(field.tpe <:< typeOf[Map[String, _]]))) { + fail(NoType, "The key type of a Map annotated with @flatten must be String.") + } + } + + /** If there is a sealed base class, annotate the derived tree in the JSON + * representation with a class label. + */ + private def annotate(tpe: c.Type)(derived: c.universe.Tree) = { + val sealedParents = tpe.baseClasses.filter(_.asClass.isSealed) + + if (sealedParents.isEmpty) derived + else { + val tagKey = MacrosCommon.tagKeyFromParents( + tpe.typeSymbol.name.toString, + sealedParents, + customKey, + (_: c.Symbol).name.toString, + fail(tpe, _), + ) + + val sealedClassSymbol: Option[Symbol] = sealedParents.find(_ == tpe.typeSymbol) + val segments = + sealedClassSymbol.toList.map(_.fullName.split('.')) ++ + sealedParents + .flatMap(_.asClass.knownDirectSubclasses) + .map(_.fullName.split('.')) + + + // -1 because even if there is only one subclass, and so no name segments + // are needed to differentiate between them, we want to keep at least + // the rightmost name segment + val identicalSegmentCount = Range(0, segments.map(_.length).max - 1) + .takeWhile(i => segments.map(_.lift(i)).distinct.size == 1) + .length + + val tagValue = customKey(tpe.typeSymbol) + .getOrElse(TypeName(tpe.typeSymbol.fullName).decodedName.toString) + + val shortTagValue = customKey(tpe.typeSymbol) + .getOrElse( + TypeName( + tpe.typeSymbol.fullName.split('.').drop(identicalSegmentCount).mkString(".") + ).decodedName.toString + ) + + val tagKeyExpr = tagKey match { + case Some(v) => q"$v" + case None => q"${c.prefix}.tagName" + } + q"${c.prefix}.annotate($derived, $tagKeyExpr, $tagValue, $shortTagValue)" + } + } + + private def customKey(sym: c.Symbol): Option[String] = { + sym.annotations + .find(_.tpe == typeOf[key]) + .flatMap(_.scalaArgs.headOption) + .map{case Literal(Constant(s)) => s.toString} + } + + private[upickle] def serializeDefaults(sym: c.Symbol): Option[Boolean] = { + sym.annotations + .find(_.tpe == typeOf[upickle.implicits.serializeDefaults]) + .flatMap(_.scalaArgs.headOption) + .map{case Literal(Constant(s)) => s.asInstanceOf[Boolean]} + } + + private[upickle] def wrapObject(obj: Tree): Tree + + private[upickle] def wrapCaseN(companion: Tree, + fields: List[Field], + varargs: Boolean, + targetType: c.Type): Tree + } + + abstract class Reading[M[_]] extends DeriveDefaults[M] { + val c: scala.reflect.macros.blackbox.Context + import c.universe._ + def wrapObject(t: c.Tree) = q"new ${c.prefix}.SingletonReader($t)" + + def wrapCaseN(companion: c.universe.Tree, fields: List[Field], varargs: Boolean, targetType: c.Type): c.universe.Tree = { + val allowUnknownKeysAnnotation = targetType.typeSymbol + .annotations + .find(_.tree.tpe == typeOf[upickle.implicits.allowUnknownKeys]) + .flatMap(_.tree.children.tail.headOption) + .map { case Literal(Constant(b)) => b.asInstanceOf[Boolean] } + + val allFields = fields.flatMap(_.allFields).toArray.filter(_.flatten != Flatten.Map) + val (hasFlattenOnMap, valueTypeOfMap) = fields.flatMap(_.allFields).find(_.flatten == Flatten.Map) match { + case Some(f) => + val TypeRef(_, _, _ :: valueType :: Nil) = f.tpe + (true, valueType) + case None => (false, NoType) + } + val numberOfFields = allFields.length + val (localReaders, aggregates) = allFields.zipWithIndex.map { case (_, idx) => + (TermName(s"localReader$idx"), TermName(s"aggregated$idx")) + }.unzip + + val fieldToId = allFields.zipWithIndex.toMap + def constructClass(companion: c.universe.Tree, fields: List[Field], varargs: Boolean): c.universe.Tree = + q""" + $companion.apply( + ..${ + fields.map { field => + field.flatten match { + case Flatten.Class(c, f, v) => constructClass(c, f, v) + case Flatten.Map => + val termName = TermName(s"aggregatedMap") + q"$termName.toMap" + case Flatten.None => + val idx = fieldToId(field) + val termName = TermName(s"aggregated$idx") + if (field == fields.last && varargs) q"$termName:_*" + else q"$termName" + } + } + } + ) + """ + + q""" + ..${ + for (i <- allFields.indices) + yield q"private[this] lazy val ${localReaders(i)} = implicitly[${c.prefix}.Reader[${allFields(i).tpe}]]" + } + ..${ + if (hasFlattenOnMap) + List( + q"private[this] lazy val localReaderMap = implicitly[${c.prefix}.Reader[$valueTypeOfMap]]", + ) + else Nil + } + new ${c.prefix}.CaseClassReader[$targetType] { + override def visitObject(length: Int, jsonableKeys: Boolean, index: Int) = new ${if (numberOfFields <= 64) tq"_root_.upickle.implicits.CaseObjectContext[$targetType]" else tq"_root_.upickle.implicits.HugeCaseObjectContext[$targetType]"}(${numberOfFields}) { + ..${ + for (i <- allFields.indices) + yield q"private[this] var ${aggregates(i)}: ${allFields(i).tpe} = _" + } + ..${ + if (hasFlattenOnMap) + List( + q"private[this] lazy val aggregatedMap: scala.collection.mutable.ListBuffer[(String, $valueTypeOfMap)] = scala.collection.mutable.ListBuffer.empty", + ) + else Nil + } + + def storeAggregatedValue(currentIndex: Int, v: Any): Unit = currentIndex match { + case ..${ + for (i <- aggregates.indices) + yield cq"$i => ${aggregates(i)} = v.asInstanceOf[${allFields(i).tpe}]" + } + case ..${ + if (hasFlattenOnMap) + List(cq"-1 => aggregatedMap += currentKey -> v.asInstanceOf[$valueTypeOfMap]") + else Nil + } + case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) + } + + def visitKeyValue(s: Any) = { + storeToMap = false + currentKey = ${c.prefix}.objectAttributeKeyReadMap(s.toString).toString + currentIndex = currentKey match { + case ..${ + for (i <- allFields.indices) + yield cq"${allFields(i).mappedName} => $i" + } + case _ => + ${ + (allowUnknownKeysAnnotation, hasFlattenOnMap) match { + case (_, true) => q"storeToMap = true; -1" + case (None, false) => + q""" + if (${ c.prefix }.allowUnknownKeys) -1 + else throw new _root_.upickle.core.Abort("Unknown Key: " + s.toString) + """ + case (Some(false), false) => q"""throw new _root_.upickle.core.Abort(" Unknown Key: " + s.toString)""" + case (Some(true), false) => q"-1" + } + } + } + } + + def visitEnd(index: Int) = { + ..${ + for(i <- allFields.indices if allFields(i).defaultValue.isDefined) + yield q"this.storeValueIfNotFound($i, ${allFields(i).defaultValue.get})" + } + + // Special-case 64 because java bit shifting ignores any RHS values above 63 + // https://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.19 + if (${ + if (numberOfFields <= 64) q"this.checkErrorMissingKeys(${if (numberOfFields == 64) -1 else (1L << numberOfFields) - 1})" + else q"this.checkErrorMissingKeys(${numberOfFields})" + }) { + this.errorMissingKeys(${numberOfFields}, ${allFields.map(_.mappedName)}) + } + + ${constructClass(companion, fields, varargs)} + } + + def subVisitor: _root_.upickle.core.Visitor[_, _] = currentIndex match { + case -1 => + ${ + if (hasFlattenOnMap) + q"localReaderMap" + else + q"_root_.upickle.core.NoOpVisitor" + } + case ..${ + for (i <- allFields.indices) + yield cq"$i => ${localReaders(i)} " + } + case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) + } + } + } + """ + } + + override def mergeTrait(tagKey: Option[String], subtrees: Seq[Tree], subtypes: Seq[Type], targetType: c.Type): Tree = { + val tagKeyExpr = tagKey match { + case Some(v) => q"$v" + case None => q"${c.prefix}.tagName" + } + q"${c.prefix}.Reader.merge[$targetType]($tagKeyExpr, ..$subtrees)" + } + } + + abstract class Writing[M[_]] extends DeriveDefaults[M] { + val c: scala.reflect.macros.blackbox.Context + import c.universe._ + def wrapObject(obj: c.Tree) = q"new ${c.prefix}.SingletonWriter($obj)" + + def internal = q"${c.prefix}.Internal" + + def wrapCaseN(companion: c.universe.Tree, fields: List[Field], varargs: Boolean, targetType: c.Type): c.universe.Tree = { + def serDfltVals(field: Field) = { + val b: Option[Boolean] = serializeDefaults(field.symbol).orElse(serializeDefaults(targetType.typeSymbol)) + b match { + case Some(b) => q"${b}" + case None => q"${c.prefix}.serializeDefaults" + } + } + + def write(field: Field, outer: c.universe.Tree): List[c.universe.Tree] = { + val select = Select(outer, TermName(field.name)) + field.flatten match { + case Flatten.Class(_, fields, _) => + fields.flatMap(write(_, select)) + case Flatten.Map => + val TypeRef(_, _, _ :: valueType :: Nil) = field.tpe + q""" + $select.foreach { case (key, value) => + this.writeSnippetMappedName[R, $valueType]( + ctx, + key.toString, + implicitly[${c.prefix}.Writer[$valueType]], + value + ) + } + """ :: Nil + case Flatten.None => + val snippet = + q""" + this.writeSnippetMappedName[R, ${field.tpe}]( + ctx, + ${c.prefix}.objectAttributeKeyWriteMap(${field.mappedName}), + implicitly[${c.prefix}.Writer[${field.tpe}]], + $select + ) + """ + val default = if (field.defaultValue.isEmpty) snippet + else q"""if (${serDfltVals(field)} || $select != ${field.defaultValue.get}) $snippet""" + default :: Nil + } + } + + def getLength(field: Field, outer: c.universe.Tree): List[c.universe.Tree] = { + val select = Select(outer, TermName(field.name)) + field.flatten match { + case Flatten.Class(_, fields, _) => fields.flatMap(getLength(_, select)) + case Flatten.Map => q"${select}.size" :: Nil + case Flatten.None => + ( + if (field.defaultValue.isEmpty) q"1" + else q"""if (${serDfltVals(field)} || ${select} != ${field.defaultValue}.get) 1 else 0""" + ) :: Nil + } + } + + q""" + new ${c.prefix}.CaseClassWriter[$targetType]{ + def length(v: $targetType) = { + ${ + fields.flatMap(getLength(_, q"v")) + .foldLeft[Tree](q"0") { case (prev, next) => q"$prev + $next" } + } + } + override def write0[R](out: _root_.upickle.core.Visitor[_, R], v: $targetType): R = { + if (v == null) out.visitNull(-1) + else { + val ctx = out.visitObject(length(v), true, -1) + ..${fields.flatMap(write(_, q"v"))} + ctx.visitEnd(-1) + } + } + def writeToObject[R](ctx: _root_.upickle.core.ObjVisitor[_, R], + v: $targetType): Unit = { + ..${fields.flatMap(write(_, q"v"))} + } + } + """ + } + + override def mergeTrait(tagKey: Option[String], subtree: Seq[Tree], subtypes: Seq[Type], targetType: c.Type): Tree = { + q"${c.prefix}.Writer.merge[$targetType](..$subtree)" + } + } + def macroRImpl[T, R[_]](c0: scala.reflect.macros.blackbox.Context) + (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[R[_]]): c0.Expr[R[T]] = { + import c0.universe._ + val res = new Reading[R]{ + val c: c0.type = c0 + def typeclass = e2 + }.derive(e1.tpe) +// println(c0.universe.showCode(res)) + c0.Expr[R[T]](res) + } + + def macroWImpl[T, W[_]](c0: scala.reflect.macros.blackbox.Context) + (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[W[_]]): c0.Expr[W[T]] = { + import c0.universe._ + val res = new Writing[W]{ + val c: c0.type = c0 + def typeclass = e2 + }.derive(e1.tpe) +// println(c0.universe.showCode(res)) + c0.Expr[W[T]](res) + } +} + diff --git a/upickle/implicits/src-3/upickle/implicits/Readers.scala b/upickle/implicits/src-3/upickle/implicits/Readers.scala index 7540cbbe8..aa93f4067 100644 --- a/upickle/implicits/src-3/upickle/implicits/Readers.scala +++ b/upickle/implicits/src-3/upickle/implicits/Readers.scala @@ -5,6 +5,7 @@ import deriving.Mirror import scala.util.NotGiven import upickle.core.{Annotator, ObjVisitor, Visitor, Abort, CurrentlyDeriving} import upickle.implicits.BaseCaseObjectContext +import scala.collection.mutable trait ReadersVersionSpecific extends MacrosCommon @@ -15,28 +16,46 @@ trait ReadersVersionSpecific abstract class CaseClassReader3[T](paramCount: Int, missingKeyCount: Long, allowUnknownKeys: Boolean, - construct: Array[Any] => T) extends CaseClassReader[T] { + construct: (Array[Any], scala.collection.mutable.Map[String, Any]) => T) extends CaseClassReader[T] { - def visitors0: Product - lazy val visitors = visitors0 - def fromProduct(p: Product): T + def visitors0: (AnyRef, Array[AnyRef]) + lazy val (visitorMap, visitors) = visitors0 + lazy val hasFlattenOnMap = visitorMap ne null def keyToIndex(x: String): Int def allKeysArray: Array[String] def storeDefaults(x: upickle.implicits.BaseCaseObjectContext): Unit trait ObjectContext extends ObjVisitor[Any, T] with BaseCaseObjectContext{ private val params = new Array[Any](paramCount) - - def storeAggregatedValue(currentIndex: Int, v: Any): Unit = params(currentIndex) = v + private val map = scala.collection.mutable.Map.empty[String, Any] + + def storeAggregatedValue(currentIndex: Int, v: Any): Unit = + if (currentIndex == -1) { + if (storeToMap) { + map(currentKey) = v + } + } else { + params(currentIndex) = v + } def subVisitor: Visitor[_, _] = - if (currentIndex == -1) upickle.core.NoOpVisitor - else visitors.productElement(currentIndex).asInstanceOf[Visitor[_, _]] + if (currentIndex == -1) { + if (hasFlattenOnMap) visitorMap.asInstanceOf[Visitor[_, _]] + else upickle.core.NoOpVisitor + } + else { + visitors(currentIndex).asInstanceOf[Visitor[_, _]] + } def visitKeyValue(v: Any): Unit = - val k = objectAttributeKeyReadMap(v.toString).toString - currentIndex = keyToIndex(k) - if (currentIndex == -1 && !allowUnknownKeys) { - throw new upickle.core.Abort("Unknown Key: " + k.toString) + storeToMap = false + currentKey = objectAttributeKeyReadMap(v.toString).toString + currentIndex = keyToIndex(currentKey) + if (currentIndex == -1) { + if (hasFlattenOnMap) { + storeToMap = true + } else if (!allowUnknownKeys) { + throw new upickle.core.Abort("Unknown Key: " + currentKey.toString) + } } def visitEnd(index: Int): T = @@ -47,7 +66,7 @@ trait ReadersVersionSpecific if (this.checkErrorMissingKeys(missingKeyCount)) this.errorMissingKeys(paramCount, allKeysArray) - construct(params) + construct(params, map) } override def visitObject(length: Int, jsonableKeys: Boolean, @@ -58,16 +77,18 @@ trait ReadersVersionSpecific inline def macroR[T](using m: Mirror.Of[T]): Reader[T] = inline m match { case m: Mirror.ProductOf[T] => + macros.validateFlattenAnnotation[T]() + val paramCount = macros.paramsCount[T] val reader = new CaseClassReader3[T]( - macros.paramsCount[T], - macros.checkErrorMissingKeysCount[T](), + paramCount, + if (paramCount <= 64) if (paramCount == 64) -1 else (1L << paramCount) - 1 + else paramCount, macros.extractIgnoreUnknownKeys[T]().headOption.getOrElse(this.allowUnknownKeys), - params => macros.applyConstructor[T](params) + (params: Array[Any], map :scala.collection.mutable.Map[String ,Any]) => macros.applyConstructor[T](params, map) ){ - override def visitors0 = compiletime.summonAll[Tuple.Map[m.MirroredElemTypes, Reader]] - override def fromProduct(p: Product): T = m.fromProduct(p) + override def visitors0 = macros.allReaders[T, Reader] override def keyToIndex(x: String): Int = macros.keyToIndex[T](x) - override def allKeysArray = macros.fieldLabels[T].map(_._2).toArray + override def allKeysArray = macros.allFieldsMappedName[T].toArray override def storeDefaults(x: upickle.implicits.BaseCaseObjectContext): Unit = macros.storeDefaults[T](x) } diff --git a/upickle/implicits/src-3/upickle/implicits/Writers.scala b/upickle/implicits/src-3/upickle/implicits/Writers.scala index 9cdae09d1..db2791446 100644 --- a/upickle/implicits/src-3/upickle/implicits/Writers.scala +++ b/upickle/implicits/src-3/upickle/implicits/Writers.scala @@ -23,7 +23,7 @@ trait WritersVersionSpecific if (v == null) out.visitNull(-1) else { val ctx = out.visitObject(length(v), true, -1) - macros.writeSnippets[R, T, Tuple.Map[m.MirroredElemTypes, Writer]]( + macros.writeSnippets[R, T, Writer]( outerThis, this, v, @@ -34,7 +34,7 @@ trait WritersVersionSpecific } def writeToObject[R](ctx: _root_.upickle.core.ObjVisitor[_, R], v: T): Unit = - macros.writeSnippets[R, T, Tuple.Map[m.MirroredElemTypes, Writer]]( + macros.writeSnippets[R, T, Writer]( outerThis, this, v, diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 50d50ee9d..0a65eba9c 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -3,11 +3,13 @@ package upickle.implicits.macros import scala.quoted.{ given, _ } import deriving._, compiletime._ import upickle.implicits.{MacrosCommon, ReadersVersionSpecific} -type IsInt[A <: Int] = A def getDefaultParamsImpl0[T](using Quotes, Type[T]): Map[String, Expr[AnyRef]] = import quotes.reflect._ - val unwrapped = TypeRepr.of[T] match{case AppliedType(p, v) => p case t => t} + val unwrapped = TypeRepr.of[T] match { + case AppliedType(p, v) => p + case t => t + } val sym = unwrapped.typeSymbol if (!sym.isClassDef) Map.empty @@ -60,27 +62,109 @@ def extractIgnoreUnknownKeysImpl[T](using Quotes, Type[T]): Expr[List[Boolean]] .toList ) +def extractFlatten[A](using Quotes)(sym: quotes.reflect.Symbol): Boolean = + import quotes.reflect._ + sym + .annotations + .exists(_.tpe =:= TypeRepr.of[upickle.implicits.flatten]) + inline def paramsCount[T]: Int = ${paramsCountImpl[T]} def paramsCountImpl[T](using Quotes, Type[T]) = { - Expr(fieldLabelsImpl0[T].size) + import quotes.reflect._ + val fields = allFields[T] + val count = fields.filter {case (_, _, _, _, flattenMap) => !flattenMap}.length + Expr(count) +} + +inline def allReaders[T, R[_]]: (AnyRef, Array[AnyRef]) = ${allReadersImpl[T, R]} +def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRef, Array[AnyRef])] = { + import quotes.reflect._ + val fields = allFields[T] + val (readerMap, readers) = fields.partitionMap { case (_, _, tpe, _, isFlattenMap) => + if (isFlattenMap) { + val valueTpe = tpe.typeArgs(1) + val readerTpe = TypeRepr.of[R].appliedTo(valueTpe) + val reader = readerTpe.asType match { + case '[t] => '{summonInline[t].asInstanceOf[AnyRef]} + } + Left(reader) + } + else { + val readerTpe = TypeRepr.of[R].appliedTo(tpe) + val reader = readerTpe.asType match { + case '[t] => '{summonInline[t].asInstanceOf[AnyRef]} + } + Right(reader) + } + } + Expr.ofTuple( + ( + readerMap.headOption.getOrElse('{null}.asInstanceOf[Expr[AnyRef]]), + '{${Expr.ofList(readers)}.toArray}, + ) + ) +} + +inline def allFieldsMappedName[T]: List[String] = ${allFieldsMappedNameImpl[T]} +def allFieldsMappedNameImpl[T](using Quotes, Type[T]): Expr[List[String]] = { + import quotes.reflect._ + Expr(allFields[T].map { case (_, label, _, _, _) => label }) } inline def storeDefaults[T](inline x: upickle.implicits.BaseCaseObjectContext): Unit = ${storeDefaultsImpl[T]('x)} def storeDefaultsImpl[T](x: Expr[upickle.implicits.BaseCaseObjectContext])(using Quotes, Type[T]) = { import quotes.reflect.* - - val statements = fieldLabelsImpl0[T] + val statements = allFields[T] + .filter(!_._5) .zipWithIndex - .map { case ((rawLabel, label), i) => - val defaults = getDefaultParamsImpl0[T] - if (defaults.contains(label)) '{${x}.storeValueIfNotFound(${Expr(i)}, ${defaults(label)})} - else '{} + .map { case ((_, _, _, default, _), i) => + default match { + case Some(defaultValue) => '{${x}.storeValueIfNotFound(${Expr(i)}, ${defaultValue})} + case None => '{} + } } Expr.block(statements, '{}) } -inline def fieldLabels[T]: List[(String, String)] = ${fieldLabelsImpl[T]} +def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String, quotes.reflect.TypeRepr, Option[Expr[Any]], Boolean)] = { + import quotes.reflect._ + + def loop(field: Symbol, label: String, classTypeRepr: TypeRepr, defaults: Map[String, Expr[Object]]): List[(Symbol, String, TypeRepr, Option[Expr[Any]], Boolean)] = { + val flatten = extractFlatten(field) + val substitutedTypeRepr = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) + val typeSymbol = substitutedTypeRepr.typeSymbol + if (flatten) { + if (isMap(substitutedTypeRepr)) { + (field, label, substitutedTypeRepr, defaults.get(label), true) :: Nil + } + else if (isCaseClass(typeSymbol)) { + typeSymbol.typeRef.dealias.asType match { + case '[t] => + fieldLabelsImpl0[t] + .flatMap { case (rawLabel, label) => + val newDefaults = getDefaultParamsImpl0[t] + val newClassTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, newClassTypeRepr, newDefaults) + } + case _ => + report.errorAndAbort(s"Unsupported type $typeSymbol for flattening") + } + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } + else { + (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil + } + } + + fieldLabelsImpl0[T] + .flatMap{ (rawLabel, label) => + val defaults = getDefaultParamsImpl0[T] + val classTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, classTypeRepr, defaults) + } +} + def fieldLabelsImpl0[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String)] = import quotes.reflect._ val fields: List[Symbol] = TypeRepr.of[T].typeSymbol @@ -96,16 +180,14 @@ def fieldLabelsImpl0[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, Str case None => (sym, sym.name) } -def fieldLabelsImpl[T](using Quotes, Type[T]): Expr[List[(String, String)]] = - Expr.ofList(fieldLabelsImpl0[T].map((a, b) => Expr((a.name, b)))) - inline def keyToIndex[T](inline x: String): Int = ${keyToIndexImpl[T]('x)} def keyToIndexImpl[T](x: Expr[String])(using Quotes, Type[T]): Expr[Int] = { import quotes.reflect.* + val fields = allFields[T].filter { case (_, _, _, _, isFlattenMap) => !isFlattenMap } val z = Match( x.asTerm, - fieldLabelsImpl0[T].map(_._2).zipWithIndex.map{(f, i) => - CaseDef(Literal(StringConstant(f)), None, Literal(IntConstant(i))) + fields.zipWithIndex.map{case ((_, label, _, _, _), i) => + CaseDef(Literal(StringConstant(label)), None, Literal(IntConstant(i))) } ++ Seq( CaseDef(Wildcard(), None, Literal(IntConstant(-1))) ) @@ -126,71 +208,138 @@ def serDfltVals(using quotes: Quotes)(thisOuter: Expr[upickle.core.Types with up case None => '{ ${ thisOuter }.serializeDefaults } } } + def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], v: Expr[T]) (using quotes: Quotes, t: Type[T]): Expr[Int] = import quotes.reflect.* + def loop(field: Symbol, label: String, classTypeRepr: TypeRepr, select: Select, defaults: Map[String, Expr[Object]]): List[Expr[Int]] = + val flatten = extractFlatten(field) + if (flatten) { + val subsitituted = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) + val typeSymbol = subsitituted.typeSymbol + if (isMap(subsitituted)) { + List( + '{${select.asExprOf[Map[_, _]]}.size} + ) + } + else if (isCaseClass(typeSymbol)) { + typeSymbol.typeRef.dealias.asType match { + case '[t] => + fieldLabelsImpl0[t] + .flatMap { case (rawLabel, label) => + val newDefaults = getDefaultParamsImpl0[t] + val newSelect = Select.unique(select, rawLabel.name) + val newClassTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) + } + case _ => + report.errorAndAbort("Unsupported type for flattening") + } + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } + else if (!defaults.contains(label)) List('{1}) + else { + val serDflt = serDfltVals(thisOuter, field, classTypeRepr.typeSymbol) + List( + '{if (${serDflt} || ${select.asExprOf[Any]} != ${defaults(label)}) 1 else 0} + ) + } + fieldLabelsImpl0[T] - .map{(rawLabel, label) => + .flatMap { (rawLabel, label) => val defaults = getDefaultParamsImpl0[T] - val select = Select.unique(v.asTerm, rawLabel.name).asExprOf[Any] - - if (!defaults.contains(label)) '{1} - else { - val serDflt = serDfltVals(thisOuter, rawLabel, TypeRepr.of[T].typeSymbol) - '{if (${serDflt} || ${select} != ${defaults(label)}) 1 else 0} - } + val select = Select.unique(v.asTerm, rawLabel.name) + val classTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, classTypeRepr, select, defaults) } .foldLeft('{0}) { case (prev, next) => '{$prev + $next} } -inline def checkErrorMissingKeysCount[T](): Long = - ${checkErrorMissingKeysCountImpl[T]()} - -def checkErrorMissingKeysCountImpl[T]()(using Quotes, Type[T]): Expr[Long] = - import quotes.reflect.* - val paramCount = fieldLabelsImpl0[T].size - if (paramCount <= 64) if (paramCount == 64) Expr(-1) else Expr((1L << paramCount) - 1) - else Expr(paramCount) - -inline def writeSnippets[R, T, WS <: Tuple](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, +inline def writeSnippets[R, T, W[_]](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, inline self: upickle.implicits.CaseClassReadWriters#CaseClassWriter[T], inline v: T, inline ctx: _root_.upickle.core.ObjVisitor[_, R]): Unit = - ${writeSnippetsImpl[R, T, WS]('thisOuter, 'self, 'v, 'ctx)} + ${writeSnippetsImpl[R, T, W]('thisOuter, 'self, 'v, 'ctx)} -def writeSnippetsImpl[R, T, WS <: Tuple](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], +def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], self: Expr[upickle.implicits.CaseClassReadWriters#CaseClassWriter[T]], v: Expr[T], ctx: Expr[_root_.upickle.core.ObjVisitor[_, R]]) - (using Quotes, Type[T], Type[R], Type[WS]): Expr[Unit] = + (using Quotes, Type[T], Type[R], Type[W]): Expr[Unit] = import quotes.reflect.* - Expr.block( - for (((rawLabel, label), i) <- fieldLabelsImpl0[T].zipWithIndex) yield { - - val tpe0 = TypeRepr.of[T].memberType(rawLabel).asType - tpe0 match - case '[tpe] => - val defaults = getDefaultParamsImpl0[T] - Literal(IntConstant(i)).tpe.asType match - case '[IsInt[index]] => - val select = Select.unique(v.asTerm, rawLabel.name).asExprOf[Any] - val snippet = '{ - ${self}.writeSnippetMappedName[R, tpe]( - ${ctx}, - ${thisOuter}.objectAttributeKeyWriteMap(${Expr(label)}), - summonInline[Tuple.Elem[WS, index]], - ${select}, - ) + def loop(field: Symbol, label: String, classTypeRepr: TypeRepr, select: Select, defaults: Map[String, Expr[Object]]): List[Expr[Any]] = + val flatten = extractFlatten(field) + val fieldTypeRepr = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) + val typeSymbol = fieldTypeRepr.typeSymbol + if (flatten) { + if (isMap(fieldTypeRepr)) { + val (keyTpe0, valueTpe0) = fieldTypeRepr.typeArgs match { + case key :: value :: Nil => (key, value) + case _ => report.errorAndAbort(s"Unsupported type ${typeSymbol} for flattening", v.asTerm.pos) } - if (!defaults.contains(label)) snippet - else { - val serDflt = serDfltVals(thisOuter, rawLabel, TypeRepr.of[T].typeSymbol) - '{if ($serDflt || ${select} != ${defaults(label)}) $snippet} + val writerTpe0 = TypeRepr.of[W].appliedTo(valueTpe0) + (keyTpe0.asType, valueTpe0.asType, writerTpe0.asType) match { + case ('[keyTpe], '[valueTpe], '[writerTpe])=> + val snippet = '{ + ${select.asExprOf[Map[keyTpe, valueTpe]]}.foreach { (k, v) => + ${self}.writeSnippetMappedName[R, valueTpe]( + ${ctx}, + k.toString, + summonInline[writerTpe], + v, + ) + } + } + List(snippet) + } + } + else if (isCaseClass(typeSymbol)) { + typeSymbol.typeRef.dealias.asType match { + case '[t] => + fieldLabelsImpl0[t] + .flatMap { case (rawLabel, label) => + val newDefaults = getDefaultParamsImpl0[t] + val newSelect = Select.unique(select, rawLabel.name) + val newClassTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) + } + case _ => + report.errorAndAbort("Unsupported type for flattening", v) } + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map", v.asTerm.pos) + } + else { + val tpe0 = fieldTypeRepr + val writerTpe0 = TypeRepr.of[W].appliedTo(tpe0) + (tpe0.asType, writerTpe0.asType) match + case ('[tpe], '[writerTpe]) => + val snippet = '{ + ${self}.writeSnippetMappedName[R, tpe]( + ${ctx}, + ${thisOuter}.objectAttributeKeyWriteMap(${Expr(label)}), + summonInline[writerTpe], + ${select.asExprOf[Any]}, + ) + } + List( + if (!defaults.contains(label)) snippet + else { + val serDflt = serDfltVals(thisOuter, field, classTypeRepr.typeSymbol) + '{if ($serDflt || ${select.asExprOf[Any]} != ${defaults(label)}) $snippet} + } + ) + } - }, + Expr.block( + fieldLabelsImpl0[T] + .flatMap { (rawLabel, label) => + val defaults = getDefaultParamsImpl0[T] + val select = Select.unique(v.asTerm, rawLabel.name) + val classTypeRepr = TypeRepr.of[T] + loop(rawLabel, label, classTypeRepr, select, defaults) + }, '{()} ) @@ -221,11 +370,19 @@ def tagKeyImpl[T](using Quotes, Type[T])(thisOuter: Expr[upickle.core.Types with case None => '{${thisOuter}.tagName} } -inline def applyConstructor[T](params: Array[Any]): T = ${ applyConstructorImpl[T]('params) } -def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]]): Expr[T] = +def substituteTypeArgs(using Quotes)(tpe: quotes.reflect.TypeRepr, subsitituted: quotes.reflect.TypeRepr): quotes.reflect.TypeRepr = { + import quotes.reflect._ + val constructorSym = tpe.typeSymbol.primaryConstructor + val constructorParamSymss = constructorSym.paramSymss + + val tparams0 = constructorParamSymss.flatten.filter(_.isType) + subsitituted.substituteTypes(tparams0 ,tpe.typeArgs) +} + +inline def applyConstructor[T](params: Array[Any], map: scala.collection.mutable.Map[String, Any]): T = ${ applyConstructorImpl[T]('params, 'map) } +def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]], map: Expr[scala.collection.mutable.Map[String, Any]]): Expr[T] = import quotes.reflect._ - def apply(typeApply: Option[List[TypeRepr]]) = { - val tpe = TypeRepr.of[T] + def apply(tpe: TypeRepr, typeArgs: List[TypeRepr], offset: Int): (Term, Int) = { val companion: Symbol = tpe.classSymbol.get.companionModule val constructorSym = tpe.typeSymbol.primaryConstructor val constructorParamSymss = constructorSym.paramSymss @@ -233,39 +390,64 @@ def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Arra val (tparams0, params0) = constructorParamSymss.flatten.partition(_.isType) val constructorTpe = tpe.memberType(constructorSym).widen - val rhs = params0.zipWithIndex.map { - case (sym0, i) => - val lhs = '{$params(${ Expr(i) })} + val (rhs, nextOffset) = params0.foldLeft((List.empty[Term], offset)) { case ((terms, i), sym0) => val tpe0 = constructorTpe.memberType(sym0) - - typeApply.map(tps => tpe0.substituteTypes(tparams0, tps)).getOrElse(tpe0) match { - case AnnotatedType(AppliedType(base, Seq(arg)), x) - if x.tpe =:= defn.RepeatedAnnot.typeRef => - arg.asType match { - case '[t] => - Typed( - lhs.asTerm, - TypeTree.of(using AppliedType(defn.RepeatedParamClass.typeRef, List(arg)).asType) - ) + val appliedTpe = tpe0.substituteTypes(tparams0, typeArgs) + val typeSymbol = appliedTpe.typeSymbol + val flatten = extractFlatten(sym0) + if (flatten) { + if (isMap(appliedTpe)) { + val keyTpe0 = appliedTpe.typeArgs.head + val valueTpe0 = appliedTpe.typeArgs(1) + (keyTpe0.asType, valueTpe0.asType) match { + case ('[keyTpe], '[valueTpe]) => + val typedMap = '{${map}.asInstanceOf[collection.mutable.Map[keyTpe, valueTpe]]}.asTerm + val term = Select.unique(typedMap, "toMap") + (term :: terms, i) } - case tpe => - tpe.asType match { - case '[t] => '{ $lhs.asInstanceOf[t] }.asTerm + } + else if (isCaseClass(typeSymbol)) { + typeSymbol.typeRef.dealias.asType match { + case '[t] => + val newTpe = TypeRepr.of[t] + val (term, nextOffset) = newTpe match { + case t: AppliedType => apply(newTpe, t.args, i) + case t: TypeRef => apply(newTpe, List.empty, i) + case t: TermRef => (Ref(t.classSymbol.get.companionModule), i) + } + (term :: terms, nextOffset) + case _ => + report.errorAndAbort(s"Unsupported type $typeSymbol for flattening") } + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } + else { + val lhs = '{$params(${ Expr(i) })} + val term = appliedTpe match { + case AnnotatedType(AppliedType(base, Seq(arg)), x) if x.tpe =:= defn.RepeatedAnnot.typeRef => + arg.asType match { + case '[t] => + Typed( + lhs.asTerm, + TypeTree.of(using AppliedType(defn.RepeatedParamClass.typeRef, List(arg)).asType) + ) + } + case tpe => + tpe.asType match { + case '[t] => '{ $lhs.asInstanceOf[t] }.asTerm + } + } + (term :: terms, i + 1) } - } - typeApply match{ - case None => Select.overloaded(Ref(companion), "apply", Nil, rhs).asExprOf[T] - case Some(args) => - Select.overloaded(Ref(companion), "apply", args, rhs).asExprOf[T] - } + (Select.overloaded(Ref(companion), "apply", typeArgs, rhs.reverse), nextOffset) } - TypeRepr.of[T] match{ - case t: AppliedType => apply(Some(t.args)) - case t: TypeRef => apply(None) + val tpe = TypeRepr.of[T] + tpe match{ + case t: AppliedType => apply(tpe, t.args, 0)._1.asExprOf[T] + case t: TypeRef => apply(tpe, List.empty, 0)._1.asExprOf[T] case t: TermRef => '{${Ref(t.classSymbol.get.companionModule).asExprOf[Any]}.asInstanceOf[T]} } @@ -389,3 +571,27 @@ def defineEnumVisitorsImpl[T0, T <: Tuple](prefix: Expr[Any], macroX: String)(us Block(allDefs.map(_._1), Ident(allDefs.head._2.termRef)).asExprOf[T0] +inline def validateFlattenAnnotation[T](): Unit = ${ validateFlattenAnnotationImpl[T] } +def validateFlattenAnnotationImpl[T](using Quotes, Type[T]): Expr[Unit] = + import quotes.reflect._ + val fields = allFields[T] + if (fields.count(_._5) > 1) { + report.errorAndAbort("Only one Map can be annotated with @upickle.implicits.flatten in the same level") + } + if (fields.map(_._2).distinct.length != fields.length) { + report.errorAndAbort("There are multiple fields with the same key") + } + if (fields.exists {case (_, _, tpe, _, isFlattenMap) => isFlattenMap && !(tpe.typeArgs.head.dealias =:= TypeRepr.of[String].dealias)}) { + report.errorAndAbort("The key type of a Map annotated with @flatten must be String.") + } + '{()} + +private def isMap(using Quotes)(tpe: quotes.reflect.TypeRepr): Boolean = { + import quotes.reflect._ + tpe.typeSymbol == TypeRepr.of[collection.immutable.Map[_, _]].typeSymbol +} + +private def isCaseClass(using Quotes)(typeSymbol: quotes.reflect.Symbol): Boolean = { + import quotes.reflect._ + typeSymbol.isClassDef && typeSymbol.flags.is(Flags.Case) +} diff --git a/upickle/implicits/src/upickle/implicits/ObjectContexts.scala b/upickle/implicits/src/upickle/implicits/ObjectContexts.scala index cac1c17e6..49f33f225 100644 --- a/upickle/implicits/src/upickle/implicits/ObjectContexts.scala +++ b/upickle/implicits/src/upickle/implicits/ObjectContexts.scala @@ -4,6 +4,9 @@ import upickle.core.ObjVisitor trait BaseCaseObjectContext { + var currentKey = "" + var storeToMap = false + def storeAggregatedValue(currentIndex: Int, v: Any): Unit def visitKey(index: Int) = _root_.upickle.core.StringVisitor @@ -21,10 +24,13 @@ abstract class CaseObjectContext[V](fieldCount: Int) extends ObjVisitor[Any, V] var found = 0L def visitValue(v: Any, index: Int): Unit = { - if (currentIndex != -1 && ((found & (1L << currentIndex)) == 0)) { + if ((currentIndex != -1) && ((found & (1L << currentIndex)) == 0)) { storeAggregatedValue(currentIndex, v) found |= (1L << currentIndex) } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } } def storeValueIfNotFound(i: Int, v: Any) = { @@ -53,10 +59,13 @@ abstract class HugeCaseObjectContext[V](fieldCount: Int) extends ObjVisitor[Any, var found = new Array[Long](fieldCount / 64 + 1) def visitValue(v: Any, index: Int): Unit = { - if (currentIndex != -1 && ((found(currentIndex / 64) & (1L << currentIndex)) == 0)) { + if ((currentIndex != -1) && ((found(currentIndex / 64) & (1L << currentIndex)) == 0)) { storeAggregatedValue(currentIndex, v) found(currentIndex / 64) |= (1L << currentIndex) } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } } def storeValueIfNotFound(i: Int, v: Any) = { diff --git a/upickle/implicits/src/upickle/implicits/key.scala b/upickle/implicits/src/upickle/implicits/key.scala index 36c013d71..2be787810 100644 --- a/upickle/implicits/src/upickle/implicits/key.scala +++ b/upickle/implicits/src/upickle/implicits/key.scala @@ -30,4 +30,16 @@ class serializeDefaults(s: Boolean) extends StaticAnnotation */ class allowUnknownKeys(b: Boolean) extends StaticAnnotation + +/** + * An annotation that, when applied to a field in a case class, flattens the fields of the + * annotated `case class` or `Map` into the parent case class during serialization. + * This means the fields will appear at the same level as the parent case class's fields + * rather than nested under the field name. During deserialization, these fields are + * grouped back into the annotated `case class` or `Map`. + * + * **Limitations**: + * - Only works with `Map` types that are subtypes of `Map[String, _]`. + * - Cannot flatten more than two `Map` instances in a same level. + */ class flatten extends StaticAnnotation diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 109781a16..9a70f0d76 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -37,6 +37,11 @@ object WrongTag { } +case class FlattenTwoMaps(@upickle.implicits.flatten map1: Map[String, String], @upickle.implicits.flatten map2: Map[String, String]) +case class ConflictingKeys(i: Int, @upickle.implicits.flatten cm: ConflictingMessage) +case class ConflictingMessage(i: Int) +case class MapWithNoneStringKey(@upickle.implicits.flatten map: Map[ConflictingMessage, String]) + object TaggedCustomSerializer{ sealed trait BooleanOrInt @@ -265,6 +270,9 @@ object FailureTests extends TestSuite { // compileError("""read[Array[Object]]("")""").msg // Make sure this doesn't hang the compiler =/ compileError("implicitly[upickle.default.Reader[Nothing]]") + compileError("upickle.default.macroRW[FlattenTwoMaps]") + compileError("upickle.default.macroRW[ConflictingKeys]") + compileError("upickle.default.macroRW[MapWithNoneStringKey]") } test("expWholeNumbers"){ upickle.default.read[Byte]("0e0") ==> 0.toByte diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 2c9736317..9217e15ac 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -145,25 +145,63 @@ object TagName{ implicit val fooRw: TagNamePickler.ReadWriter[Foo] = TagNamePickler.macroRW } -case class Pagination(limit: Int, offset: Int, total: Int) +object Flatten { + case class FlattenTest(i: Int, s: String, @upickle.implicits.flatten n: Nested, @upickle.implicits.flatten n2: Nested2) -object Pagination { - implicit val rw: RW[Pagination] = upickle.default.macroRW -} + object FlattenTest { + implicit val rw: RW[FlattenTest] = upickle.default.macroRW + } -case class Users(Ids: List[Int], @upickle.implicits.flatten pagination: Pagination) + case class Nested(d: Double, @upickle.implicits.flatten m: Map[String, Int]) -object Users { - implicit val rw: RW[Users] = upickle.default.macroRW -} + object Nested { + implicit val rw: RW[Nested] = upickle.default.macroRW + } + + case class Nested2(name: String) + + object Nested2 { + implicit val rw: RW[Nested2] = upickle.default.macroRW + } + + case class FlattenTestWithType[T](i: Int, @upickle.implicits.flatten t: T) + + object FlattenTestWithType { + // implicit def rw[T: RW]: RW[FlattenTestWithType[T]] = upickle.default.macroRW + implicit val rw: RW[FlattenTestWithType[Nested]] = upickle.default.macroRW + } + + case class InnerMost(a: String, b: Int) + + object InnerMost { + implicit val rw: RW[InnerMost] = upickle.default.macroRW + } + + case class Inner(@upickle.implicits.flatten innerMost: InnerMost, c: Boolean) + + object Inner { + implicit val rw: RW[Inner] = upickle.default.macroRW + } -case class PackageManifest( - name: String, - @upickle.implicits.flatten otherStuff: Map[String, ujson.Value] - ) + case class Outer(d: Double, @upickle.implicits.flatten inner: Inner) -object PackageManifest { - implicit val rw: RW[PackageManifest] = upickle.default.macroRW + object Outer { + implicit val rw: RW[Outer] = upickle.default.macroRW + } + + case class HasMap(@upickle.implicits.flatten map: Map[String, String], i: Int) + object HasMap { + implicit val rw: RW[HasMap] = upickle.default.macroRW + } + + case class FlattenWithDefault(i: Int, @upickle.implicits.flatten n: NestedWithDefault) + object FlattenWithDefault { + implicit val rw: RW[FlattenWithDefault] = upickle.default.macroRW + } + case class NestedWithDefault(k: Int = 100, l: String) + object NestedWithDefault { + implicit val rw: RW[NestedWithDefault] = upickle.default.macroRW + } } object MacroTests extends TestSuite { @@ -172,7 +210,7 @@ object MacroTests extends TestSuite { // case class A_(objects: Option[C_]); case class C_(nodes: Option[C_]) // implicitly[Reader[A_]] -// implicitly[upickle.old.Writer[upickle.MixedIn.Obj.ClsB]code] +// implicitly[upickle.old.Writer[upickle.MixedIn.Obj.ClsB]] // println(write(ADTs.ADTc(1, "lol", (1.1, 1.2)))) // implicitly[upickle.old.Writer[ADTs.ADTc]] @@ -904,9 +942,39 @@ object MacroTests extends TestSuite { } test("flatten"){ - val a = Users(List(1, 2, 3), Pagination(10, 20, 30)) - upickle.default.write[Users](a) ==> """{"Ids":[1,2,3],"limit":10,"offset":20,"total":30}""" + import Flatten._ + val a = FlattenTest(10, "test", Nested(3.0, Map("one" -> 1, "two" -> 2)), Nested2("hello")) + rw(a, """{"i":10,"s":"test","d":3,"one":1,"two":2,"name":"hello"}""") + } + + test("flattenTypeParam"){ + import Flatten._ + val a = FlattenTestWithType[Nested](10, Nested(5.0, Map("one" -> 1, "two" -> 2))) + rw(a, """{"i":10,"d":5,"one":1,"two":2}""") } + test("nestedFlatten") { + import Flatten._ + val value = Outer(1.1, Inner(InnerMost("test", 42), true)) + rw(value, """{"d":1.1,"a":"test","b":42,"c":true}""") + } + + test("flattenWithMap") { + import Flatten._ + val value = HasMap(Map("key1" -> "value1", "key2" -> "value2"), 10) + rw(value, """{"key1":"value1","key2":"value2","i":10}""") + } + + test("flattenEmptyMap") { + import Flatten._ + val value = HasMap(Map.empty, 10) + rw(value, """{"i":10}""") + } + + test("flattenWithDefaults") { + import Flatten._ + val value = FlattenWithDefault(10, NestedWithDefault(l = "default")) + rw(value, """{"i":10,"l":"default"}""") + } } } From b6d17b1267e033cba1f35e51bc4d7bfb7dbc81cc Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Sun, 8 Dec 2024 22:54:50 +0900 Subject: [PATCH 04/23] Use same approach for scala 2 & 3 --- .../upickle/implicits/internal/Macros2.scala | 379 +++++++++--------- 1 file changed, 195 insertions(+), 184 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 0682adb92..25b6cf894 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -21,10 +21,10 @@ object Macros2 { trait DeriveDefaults[M[_]] { val c: scala.reflect.macros.blackbox.Context - private def fail(tpe: c.Type, s: String) = c.abort(c.enclosingPosition, s) + private[upickle] def fail(tpe: c.Type, s: String) = c.abort(c.enclosingPosition, s) import c.universe._ - private def companionTree(tpe: c.Type): Tree = { + private[upickle] def companionTree(tpe: c.Type): Tree = { val companionSymbol = tpe.typeSymbol.companionSymbol if (companionSymbol == NoSymbol && tpe.typeSymbol.isClass) { @@ -37,7 +37,7 @@ object Macros2 { } else { val symTab = c.universe.asInstanceOf[reflect.internal.SymbolTable] val pre = tpe.asInstanceOf[symTab.Type].prefix.asInstanceOf[Type] - c.universe.treeBuild.mkAttributedRef(pre, companionSymbol) + c.universe.internal.gen.mkAttributedRef(pre, companionSymbol) } } @@ -55,10 +55,10 @@ object Macros2 { baseClsArgs = st.baseType(tpe.typeSymbol).asInstanceOf[TypeRef].args } yield { tpe match{ - case ExistentialType(_, TypeRef(pre, sym, args)) => + case ExistentialType(_, TypeRef(_, _, args)) => st.substituteTypes(baseClsArgs.map(_.typeSymbol), args) case ExistentialType(_, _) => st - case TypeRef(pre, sym, args) => + case TypeRef(_, _, args) => st.substituteTypes(baseClsArgs.map(_.typeSymbol), args) } } @@ -68,7 +68,7 @@ object Macros2 { val mod = tpe.typeSymbol.asClass.module val symTab = c.universe.asInstanceOf[reflect.internal.SymbolTable] val pre = tpe.asInstanceOf[symTab.Type].prefix.asInstanceOf[Type] - val mod2 = c.universe.treeBuild.mkAttributedRef(pre, mod) + val mod2 = c.universe.internal.gen.mkAttributedRef(pre, mod) annotate(tpe)(wrapObject(mod2)) @@ -77,7 +77,7 @@ object Macros2 { private[upickle] def mergeTrait(tagKey: Option[String], subtrees: Seq[Tree], subtypes: Seq[Type], targetType: c.Type): Tree private[upickle] def derive(tpe: c.Type) = { - if (tpe.typeSymbol.asClass.isTrait || (tpe.typeSymbol.asClass.isAbstractClass && !tpe.typeSymbol.isJava)) { + if (tpe.typeSymbol.asClass.isTrait || (tpe.typeSymbol.asClass.isAbstract && !tpe.typeSymbol.isJava)) { val derived = deriveTrait(tpe) derived } @@ -115,11 +115,9 @@ object Macros2 { weakTypeOf[M[_]](typeclass) match { case TypeRef(a, b, _) => - import compat._ - TypeRef(a, b, List(t)) + internal.typeRef(a, b, List(t)) case ExistentialType(_, TypeRef(a, b, _)) => - import compat._ - TypeRef(a, b, List(t)) + internal.typeRef(a, b, List(t)) case x => println("Dunno Wad Dis Typeclazz Is " + x) println(x) @@ -128,112 +126,86 @@ object Macros2 { } } - sealed trait Flatten + private def deriveClass(tpe: c.Type) = { + val fields = getFields(tpe) + // According to @retronym, this is necessary in order to force the + // default argument `apply$default$n` methods to be synthesized + val companion = companionTree(tpe) + companion.tpe.member(TermName("apply")).info - object Flatten { - case class Class(companion: Tree, fields: List[Field], varArgs: Boolean) extends Flatten - case object Map extends Flatten - case object None extends Flatten - } + validateFlattenAnnotation(fields) - case class Field( - name: String, - mappedName: String, - tpe: Type, - symbol: Symbol, - defaultValue: Option[Tree], - flatten: Flatten, - ) { - lazy val allFields: List[Field] = { - def loop(field: Field): List[Field] = - field.flatten match { - case Flatten.Class(_, fields, _) => fields.flatMap(loop) - case Flatten.Map => List(field) - case Flatten.None => List(field) - } - loop(this) - } + val derive = + // Otherwise, reading and writing are kinda identical + wrapCaseN( + fields, + targetType = tpe, + ) + + annotate(tpe)(derive) } - private def getFields(tpe: c.Type): (c.Tree, List[Field], Boolean) = { - def applyTypeArguments(t: c.Type ): c.Type = { - val typeParams = tpe.typeSymbol.asClass.typeParams - val typeArguments = tpe.normalize.asInstanceOf[TypeRef].args - if (t.typeSymbol != definitions.RepeatedParamClass) { - t.substituteTypes(typeParams, typeArguments) - } else { - val TypeRef(pref, sym, _) = typeOf[Seq[Int]] - internal.typeRef(pref, sym, t.asInstanceOf[TypeRef].args) + private def getFields(tpe: c.Type): List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)] = { + val params = getSymbols(tpe) + val fields = params.zipWithIndex.flatMap { case (param, i) => + val (mappedName, tpeOfField, defaultValue) = getSymbolDetail(param, i, tpe) + param.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { + case Some(_) => + if (isFlattableCollection(tpeOfField)) + List((param.name.toString, mappedName, tpeOfField, param, defaultValue, true)) + else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { + getFields(tpeOfField) + } + else fail(tpeOfField, s"Invalid type for flattening getField: $tpeOfField.") + case None => + List((param.name.toString, mappedName, tpeOfField, param, defaultValue, false)) } } + fields + } + private[upickle] def getSymbols(tpe: c.Type): List[Symbol] = { val companion = companionTree(tpe) //tickle the companion members -- Not doing this leads to unexpected runtime behavior //I wonder if there is an SI related to this? companion.tpe.members.foreach(_ => ()) + tpe.members.find(x => x.isMethod && x.asMethod.isPrimaryConstructor) match { + case Some(primaryConstructor) => primaryConstructor.asMethod.paramLists.flatten case None => fail(tpe, "Can't find primary constructor of " + tpe) - case Some(primaryConstructor) => - val params = primaryConstructor.asMethod.paramLists.flatten - val varArgs = params.lastOption.exists(_.typeSignature.typeSymbol == definitions.RepeatedParamClass) - val fields = params.zipWithIndex.map { case (param, i) => - val name = param.name.decodedName.toString - val mappedName = customKey(param).getOrElse(name) - val tpeOfField = applyTypeArguments(param.typeSignature) - val defaultValue = if (param.asTerm.isParamWithDefault) - Some(q"$companion.${TermName("apply$default$" + (i + 1))}") - else - None - val flatten = param.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { - case Some(_) => - if (tpeOfField.typeSymbol == typeOf[collection.immutable.Map[_, _]].typeSymbol) Flatten.Map - else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { - val (nestedCompanion, fields, nestedVarArgs) = getFields(tpeOfField) - Flatten.Class(nestedCompanion, fields, nestedVarArgs) - } - else fail(tpeOfField, - s"""Invalid type for flattening: $tpeOfField. - | Flatten only works on case classes and Maps""".stripMargin) - case None => - Flatten.None - } - Field(param.name.toString, mappedName, tpeOfField, param, defaultValue, flatten) - } - (companion, fields, varArgs) } } - private def deriveClass(tpe: c.Type) = { - val (companion, fields, varArgs) = getFields(tpe) - // According to @retronym, this is necessary in order to force the - // default argument `apply$default$n` methods to be synthesized - companion.tpe.member(TermName("apply")).info - - val allFields = fields.flatMap(_.allFields) - validateFlattenAnnotation(allFields) - - val derive = - // Otherwise, reading and writing are kinda identical - wrapCaseN( - companion, - fields, - varArgs, - targetType = tpe, - ) + private[upickle] def getSymbolDetail(symbol: Symbol, idx: Int, containingTpe: c.Type): (String, c.Type, Option[c.Tree]) = { + val name = symbol.name.decodedName.toString + val mappedName = customKey(symbol).getOrElse(name) + val companion = companionTree(containingTpe) + val tpe = applyTypeArguments(containingTpe, appliedTpe = symbol.typeSignature) + val defaultValue = if (symbol.asTerm.isParamWithDefault) + Some(q"$companion.${TermName("apply$default$" + (idx + 1))}") + else + None + (mappedName, tpe, defaultValue) + } - annotate(tpe)(derive) + private def applyTypeArguments(tpe: c.Type, appliedTpe: c.Type): c.Type = { + val typeParams = tpe.typeSymbol.asClass.typeParams + val typeArguments = tpe.typeArgs + if (appliedTpe.typeSymbol != definitions.RepeatedParamClass) { + appliedTpe.substituteTypes(typeParams, typeArguments) + } else { + val TypeRef(pref, sym, _) = typeOf[Seq[Int]] + internal.typeRef(pref, sym, appliedTpe.asInstanceOf[TypeRef].args) + } } - private def validateFlattenAnnotation(fields: List[Field]): Unit = { - if (fields.count(_.flatten == Flatten.Map) > 1) { - fail(NoType, "Only one Map can be annotated with @upickle.implicits.flatten in the same level") + private def validateFlattenAnnotation(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)]): Unit = { + if (fields.count { case(_, _, _, _, _, isCollection) => isCollection } > 1) { + fail(NoType, "Only one collection can be annotated with @upickle.implicits.flatten in the same level") } - if (fields.map(_.mappedName).distinct.length != fields.length) { + if (fields.map { case (_, mappedName, _, _, _, _) => mappedName }.distinct.length != fields.length) { fail(NoType, "There are multiple fields with the same key") } - if (fields.exists(field => field.flatten == Flatten.Map && !(field.tpe <:< typeOf[Map[String, _]]))) { - fail(NoType, "The key type of a Map annotated with @flatten must be String.") - } } /** If there is a sealed base class, annotate the derived tree in the JSON @@ -285,6 +257,10 @@ object Macros2 { } } + private[upickle] def isFlattableCollection(tpe: c.Type): Boolean = { + tpe.typeSymbol == typeOf[collection.immutable.Map[_, _]].typeSymbol + } + private def customKey(sym: c.Symbol): Option[String] = { sym.annotations .find(_.tpe == typeOf[key]) @@ -301,10 +277,8 @@ object Macros2 { private[upickle] def wrapObject(obj: Tree): Tree - private[upickle] def wrapCaseN(companion: Tree, - fields: List[Field], - varargs: Boolean, - targetType: c.Type): Tree + private[upickle] def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + targetType: c.Type): Tree } abstract class Reading[M[_]] extends DeriveDefaults[M] { @@ -312,51 +286,70 @@ object Macros2 { import c.universe._ def wrapObject(t: c.Tree) = q"new ${c.prefix}.SingletonReader($t)" - def wrapCaseN(companion: c.universe.Tree, fields: List[Field], varargs: Boolean, targetType: c.Type): c.universe.Tree = { + def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + targetType: c.Type): c.universe.Tree = { val allowUnknownKeysAnnotation = targetType.typeSymbol .annotations .find(_.tree.tpe == typeOf[upickle.implicits.allowUnknownKeys]) .flatMap(_.tree.children.tail.headOption) .map { case Literal(Constant(b)) => b.asInstanceOf[Boolean] } - val allFields = fields.flatMap(_.allFields).toArray.filter(_.flatten != Flatten.Map) - val (hasFlattenOnMap, valueTypeOfMap) = fields.flatMap(_.allFields).find(_.flatten == Flatten.Map) match { + val (mappedNames, types, defaultValues) = fields.toArray.filter { case (_, _, _, _, _, isCollection) => !isCollection }.map { + case (_, mappedName, tpe, _, defaultValue, _) => (mappedName, tpe, defaultValue) + }.unzip3 + val (hasFlattenOnMap, valueTypeOfMap) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { case Some(f) => - val TypeRef(_, _, _ :: valueType :: Nil) = f.tpe + val TypeRef(_, _, _ :: valueType :: Nil) = f._3 (true, valueType) case None => (false, NoType) } - val numberOfFields = allFields.length - val (localReaders, aggregates) = allFields.zipWithIndex.map { case (_, idx) => + val numberOfFields = mappedNames.length + val (localReaders, aggregates) = mappedNames.zipWithIndex.map { case (_, idx) => (TermName(s"localReader$idx"), TermName(s"aggregated$idx")) }.unzip - val fieldToId = allFields.zipWithIndex.toMap - def constructClass(companion: c.universe.Tree, fields: List[Field], varargs: Boolean): c.universe.Tree = - q""" - $companion.apply( - ..${ - fields.map { field => - field.flatten match { - case Flatten.Class(c, f, v) => constructClass(c, f, v) - case Flatten.Map => - val termName = TermName(s"aggregatedMap") - q"$termName.toMap" - case Flatten.None => - val idx = fieldToId(field) - val termName = TermName(s"aggregated$idx") - if (field == fields.last && varargs) q"$termName:_*" - else q"$termName" - } - } - } - ) - """ + def constructClass(constructedTpe: c.Type): c.universe.Tree = { + def loop(tpe: c.Type, offset: Int): (c.universe.Tree, Int) = { + val companion = companionTree(tpe) + val symbols = getSymbols(tpe) + val varArgs = symbols.lastOption.exists(_.typeSignature.typeSymbol == definitions.RepeatedParamClass) + val (terms, nextOffset) = + symbols.foldLeft((List.empty[Tree], offset)) { case ((terms, idx), symbol) => + val (_, tpeOfField, _) = getSymbolDetail(symbol, 0, tpe) + symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { + case Some(_) => + if (isFlattableCollection(tpeOfField)) { + val termName = TermName(s"aggregatedMap") + val term = q"$termName.toMap" + (term :: terms, idx) + } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { + val (term, nextIdx) = loop(tpeOfField, idx) + (term :: terms, nextIdx) + } + else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + case None => + val termName = TermName(s"aggregated$idx") + val term = if (symbol == symbols.last && varArgs) q"$termName:_*" else q"$termName" + (term :: terms, idx + 1) + } + } + val constructed = + q""" + $companion.apply( + ..${ + terms.reverse + } + ) + """ + (constructed, nextOffset) + } + loop(constructedTpe, 0)._1 + } q""" ..${ - for (i <- allFields.indices) - yield q"private[this] lazy val ${localReaders(i)} = implicitly[${c.prefix}.Reader[${allFields(i).tpe}]]" + for (i <- types.indices) + yield q"private[this] lazy val ${localReaders(i)} = implicitly[${c.prefix}.Reader[${types(i)}]]" } ..${ if (hasFlattenOnMap) @@ -368,8 +361,8 @@ object Macros2 { new ${c.prefix}.CaseClassReader[$targetType] { override def visitObject(length: Int, jsonableKeys: Boolean, index: Int) = new ${if (numberOfFields <= 64) tq"_root_.upickle.implicits.CaseObjectContext[$targetType]" else tq"_root_.upickle.implicits.HugeCaseObjectContext[$targetType]"}(${numberOfFields}) { ..${ - for (i <- allFields.indices) - yield q"private[this] var ${aggregates(i)}: ${allFields(i).tpe} = _" + for (i <- types.indices) + yield q"private[this] var ${aggregates(i)}: ${types(i)} = _" } ..${ if (hasFlattenOnMap) @@ -382,7 +375,7 @@ object Macros2 { def storeAggregatedValue(currentIndex: Int, v: Any): Unit = currentIndex match { case ..${ for (i <- aggregates.indices) - yield cq"$i => ${aggregates(i)} = v.asInstanceOf[${allFields(i).tpe}]" + yield cq"$i => ${aggregates(i)} = v.asInstanceOf[${types(i)}]" } case ..${ if (hasFlattenOnMap) @@ -397,8 +390,8 @@ object Macros2 { currentKey = ${c.prefix}.objectAttributeKeyReadMap(s.toString).toString currentIndex = currentKey match { case ..${ - for (i <- allFields.indices) - yield cq"${allFields(i).mappedName} => $i" + for (i <- mappedNames.indices) + yield cq"${mappedNames(i)} => $i" } case _ => ${ @@ -418,8 +411,8 @@ object Macros2 { def visitEnd(index: Int) = { ..${ - for(i <- allFields.indices if allFields(i).defaultValue.isDefined) - yield q"this.storeValueIfNotFound($i, ${allFields(i).defaultValue.get})" + for(i <- defaultValues.indices if defaultValues(i).isDefined) + yield q"this.storeValueIfNotFound($i, ${defaultValues(i).get})" } // Special-case 64 because java bit shifting ignores any RHS values above 63 @@ -428,10 +421,10 @@ object Macros2 { if (numberOfFields <= 64) q"this.checkErrorMissingKeys(${if (numberOfFields == 64) -1 else (1L << numberOfFields) - 1})" else q"this.checkErrorMissingKeys(${numberOfFields})" }) { - this.errorMissingKeys(${numberOfFields}, ${allFields.map(_.mappedName)}) + this.errorMissingKeys(${numberOfFields}, ${mappedNames}) } - ${constructClass(companion, fields, varargs)} + ${constructClass(targetType)} } def subVisitor: _root_.upickle.core.Visitor[_, _] = currentIndex match { @@ -443,7 +436,7 @@ object Macros2 { q"_root_.upickle.core.NoOpVisitor" } case ..${ - for (i <- allFields.indices) + for (i <- mappedNames.indices) yield cq"$i => ${localReaders(i)} " } case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) @@ -469,58 +462,76 @@ object Macros2 { def internal = q"${c.prefix}.Internal" - def wrapCaseN(companion: c.universe.Tree, fields: List[Field], varargs: Boolean, targetType: c.Type): c.universe.Tree = { - def serDfltVals(field: Field) = { - val b: Option[Boolean] = serializeDefaults(field.symbol).orElse(serializeDefaults(targetType.typeSymbol)) + def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + targetType: c.Type): c.universe.Tree = { + def serDfltVals(symbol: Symbol) = { + val b: Option[Boolean] = serializeDefaults(symbol).orElse(serializeDefaults(targetType.typeSymbol)) b match { case Some(b) => q"${b}" case None => q"${c.prefix}.serializeDefaults" } } - def write(field: Field, outer: c.universe.Tree): List[c.universe.Tree] = { - val select = Select(outer, TermName(field.name)) - field.flatten match { - case Flatten.Class(_, fields, _) => - fields.flatMap(write(_, select)) - case Flatten.Map => - val TypeRef(_, _, _ :: valueType :: Nil) = field.tpe - q""" - $select.foreach { case (key, value) => - this.writeSnippetMappedName[R, $valueType]( - ctx, - key.toString, - implicitly[${c.prefix}.Writer[$valueType]], - value - ) - } - """ :: Nil - case Flatten.None => - val snippet = - q""" - this.writeSnippetMappedName[R, ${field.tpe}]( - ctx, - ${c.prefix}.objectAttributeKeyWriteMap(${field.mappedName}), - implicitly[${c.prefix}.Writer[${field.tpe}]], - $select - ) - """ - val default = if (field.defaultValue.isEmpty) snippet - else q"""if (${serDfltVals(field)} || $select != ${field.defaultValue.get}) $snippet""" - default :: Nil + def write(tpe: c.Type, outer: c.universe.Tree): List[c.universe.Tree] = { + val symbols = getSymbols(tpe) + symbols.zipWithIndex.flatMap { case (symbol, i) => + val (mappedName, tpeOfField, defaultValue) = getSymbolDetail(symbol, i, tpe) + val select = Select(outer, TermName(symbol.name.toString)) + + symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { + case Some(_) => + if (isFlattableCollection(tpeOfField)) { + val TypeRef(_, _, _ :: valueType :: Nil) = tpeOfField + q""" + $select.foreach { case (key, value) => + this.writeSnippetMappedName[R, $valueType]( + ctx, + key.toString, + implicitly[${c.prefix}.Writer[$valueType]], + value + ) + } + """ :: Nil + } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { + write(tpeOfField, select) + } + else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + case None => + val snippet = + q""" + this.writeSnippetMappedName[R, ${tpeOfField}]( + ctx, + ${c.prefix}.objectAttributeKeyWriteMap(${mappedName}), + implicitly[${c.prefix}.Writer[${tpeOfField}]], + $select + ) + """ + val default = if (defaultValue.isEmpty) snippet + else q"""if (${serDfltVals(symbol)} || $select != ${defaultValue.get}) $snippet""" + default :: Nil + } } } - def getLength(field: Field, outer: c.universe.Tree): List[c.universe.Tree] = { - val select = Select(outer, TermName(field.name)) - field.flatten match { - case Flatten.Class(_, fields, _) => fields.flatMap(getLength(_, select)) - case Flatten.Map => q"${select}.size" :: Nil - case Flatten.None => - ( - if (field.defaultValue.isEmpty) q"1" - else q"""if (${serDfltVals(field)} || ${select} != ${field.defaultValue}.get) 1 else 0""" - ) :: Nil + def getLength(tpe: c.Type, outer: c.universe.Tree): List[c.universe.Tree] = { + val symbols = getSymbols(tpe) + symbols.zipWithIndex.flatMap { case (symbol, i) => + val (_, tpeOfField, defaultValue) = getSymbolDetail(symbol, i, tpe) + val select = Select(outer, TermName(symbol.name.toString)) + symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { + case Some(_) => + if (isFlattableCollection(tpeOfField)) { + q"${select}.size" :: Nil + } + else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { + getLength(tpeOfField, select) + } + else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + case None => + val snippet = if (defaultValue.isEmpty) q"1" + else q"""if (${serDfltVals(symbol)} || $select != ${defaultValue.get}) 1 else 0""" + snippet :: Nil + } } } @@ -528,7 +539,7 @@ object Macros2 { new ${c.prefix}.CaseClassWriter[$targetType]{ def length(v: $targetType) = { ${ - fields.flatMap(getLength(_, q"v")) + getLength(targetType, q"v") .foldLeft[Tree](q"0") { case (prev, next) => q"$prev + $next" } } } @@ -536,13 +547,13 @@ object Macros2 { if (v == null) out.visitNull(-1) else { val ctx = out.visitObject(length(v), true, -1) - ..${fields.flatMap(write(_, q"v"))} + ..${write(targetType, q"v")} ctx.visitEnd(-1) } } def writeToObject[R](ctx: _root_.upickle.core.ObjVisitor[_, R], v: $targetType): Unit = { - ..${fields.flatMap(write(_, q"v"))} + ..${write(targetType, q"v")} } } """ From 35f6851b0889ce2437b830e435061f71c80b45e4 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Sun, 8 Dec 2024 23:13:07 +0900 Subject: [PATCH 05/23] fix bench --- .../implicits/src-2/upickle/implicits/internal/Macros2.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 25b6cf894..271dd45a3 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -190,7 +190,7 @@ object Macros2 { private def applyTypeArguments(tpe: c.Type, appliedTpe: c.Type): c.Type = { val typeParams = tpe.typeSymbol.asClass.typeParams - val typeArguments = tpe.typeArgs + val typeArguments = tpe.dealias.asInstanceOf[TypeRef].args if (appliedTpe.typeSymbol != definitions.RepeatedParamClass) { appliedTpe.substituteTypes(typeParams, typeArguments) } else { From 77a882e751bb24ce9abc65ecba19cbf327880e26 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Tue, 10 Dec 2024 21:20:39 +0900 Subject: [PATCH 06/23] Move storeToMap to downstream trait --- .../upickle/implicits/internal/Macros2.scala | 31 +++++++++++++++++ .../src-3/upickle/implicits/Readers.scala | 33 ++++++++++++++++--- .../upickle/implicits/ObjectContexts.scala | 9 ----- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 271dd45a3..3b4a950c9 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -346,6 +346,32 @@ object Macros2 { loop(constructedTpe, 0)._1 } + val visitValueDef = + if (numberOfFields <= 64) + q""" + override def visitValue(v: Any, index: Int): Unit = { + if ((currentIndex != -1) && ((found & (1L << currentIndex)) == 0)) { + storeAggregatedValue(currentIndex, v) + found |= (1L << currentIndex) + } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } + } + """ + else + q""" + override def visitValue(v: Any, index: Int): Unit = { + if ((currentIndex != -1) && ((found(currentIndex / 64) & (1L << currentIndex)) == 0)) { + storeAggregatedValue(currentIndex, v) + found(currentIndex / 64) |= (1L << currentIndex) + } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } + } + """ + q""" ..${ for (i <- types.indices) @@ -360,6 +386,11 @@ object Macros2 { } new ${c.prefix}.CaseClassReader[$targetType] { override def visitObject(length: Int, jsonableKeys: Boolean, index: Int) = new ${if (numberOfFields <= 64) tq"_root_.upickle.implicits.CaseObjectContext[$targetType]" else tq"_root_.upickle.implicits.HugeCaseObjectContext[$targetType]"}(${numberOfFields}) { + var currentKey = "" + var storeToMap = false + + $visitValueDef + ..${ for (i <- types.indices) yield q"private[this] var ${aggregates(i)}: ${types(i)} = _" diff --git a/upickle/implicits/src-3/upickle/implicits/Readers.scala b/upickle/implicits/src-3/upickle/implicits/Readers.scala index aa93f4067..ab677be13 100644 --- a/upickle/implicits/src-3/upickle/implicits/Readers.scala +++ b/upickle/implicits/src-3/upickle/implicits/Readers.scala @@ -24,9 +24,11 @@ trait ReadersVersionSpecific def keyToIndex(x: String): Int def allKeysArray: Array[String] def storeDefaults(x: upickle.implicits.BaseCaseObjectContext): Unit - trait ObjectContext extends ObjVisitor[Any, T] with BaseCaseObjectContext{ + trait ObjectContext extends ObjVisitor[Any, T] with BaseCaseObjectContext { private val params = new Array[Any](paramCount) - private val map = scala.collection.mutable.Map.empty[String, Any] + private val map = scala.collection.mutable.Map.empty[String, Any] + private var currentKey = "" + protected var storeToMap = false def storeAggregatedValue(currentIndex: Int, v: Any): Unit = if (currentIndex == -1) { @@ -54,7 +56,7 @@ trait ReadersVersionSpecific if (hasFlattenOnMap) { storeToMap = true } else if (!allowUnknownKeys) { - throw new upickle.core.Abort("Unknown Key: " + currentKey.toString) + throw new upickle.core.Abort("Unknown Key: " + currentKey) } } @@ -68,11 +70,32 @@ trait ReadersVersionSpecific construct(params, map) } + override def visitObject(length: Int, jsonableKeys: Boolean, index: Int) = - if (paramCount <= 64) new CaseObjectContext[T](paramCount) with ObjectContext - else new HugeCaseObjectContext[T](paramCount) with ObjectContext + if (paramCount <= 64) new CaseObjectContext[T](paramCount) with ObjectContext { + override def visitValue(v: Any, index: Int): Unit = { + if ((currentIndex != -1) && ((found & (1L << currentIndex)) == 0)) { + storeAggregatedValue(currentIndex, v) + found |= (1L << currentIndex) + } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } + } + } + else new HugeCaseObjectContext[T](paramCount) with ObjectContext { + override def visitValue(v: Any, index: Int): Unit = { + if ((currentIndex != -1) && ((found(currentIndex / 64) & (1L << currentIndex)) == 0)) { + storeAggregatedValue(currentIndex, v) + found(currentIndex / 64) |= (1L << currentIndex) + } + else if (storeToMap) { + storeAggregatedValue(currentIndex, v) + } + } + } } inline def macroR[T](using m: Mirror.Of[T]): Reader[T] = inline m match { diff --git a/upickle/implicits/src/upickle/implicits/ObjectContexts.scala b/upickle/implicits/src/upickle/implicits/ObjectContexts.scala index 49f33f225..329b31c65 100644 --- a/upickle/implicits/src/upickle/implicits/ObjectContexts.scala +++ b/upickle/implicits/src/upickle/implicits/ObjectContexts.scala @@ -4,9 +4,6 @@ import upickle.core.ObjVisitor trait BaseCaseObjectContext { - var currentKey = "" - var storeToMap = false - def storeAggregatedValue(currentIndex: Int, v: Any): Unit def visitKey(index: Int) = _root_.upickle.core.StringVisitor @@ -28,9 +25,6 @@ abstract class CaseObjectContext[V](fieldCount: Int) extends ObjVisitor[Any, V] storeAggregatedValue(currentIndex, v) found |= (1L << currentIndex) } - else if (storeToMap) { - storeAggregatedValue(currentIndex, v) - } } def storeValueIfNotFound(i: Int, v: Any) = { @@ -63,9 +57,6 @@ abstract class HugeCaseObjectContext[V](fieldCount: Int) extends ObjVisitor[Any, storeAggregatedValue(currentIndex, v) found(currentIndex / 64) |= (1L << currentIndex) } - else if (storeToMap) { - storeAggregatedValue(currentIndex, v) - } } def storeValueIfNotFound(i: Int, v: Any) = { From a8c671a61b235dc063a02647ddbb67a5fa978c0c Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Tue, 10 Dec 2024 21:39:22 +0900 Subject: [PATCH 07/23] Pass bincompat --- .../upickle/implicits/internal/Macros.scala | 3 + .../src-3/upickle/implicits/Readers.scala | 52 +++++++++++++++- .../src-3/upickle/implicits/macros.scala | 60 +++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala index 3236716c3..53eda98d4 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -471,6 +471,8 @@ object Macros { q"${c.prefix}.Writer.merge[$targetType](..$subtree)" } } + + @deprecated("Use one's from Macros2.scala") def macroRImpl[T, R[_]](c0: scala.reflect.macros.blackbox.Context) (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[R[_]]): c0.Expr[R[T]] = { import c0.universe._ @@ -482,6 +484,7 @@ object Macros { c0.Expr[R[T]](res) } + @deprecated("Use one's from Macros2.scala") def macroWImpl[T, W[_]](c0: scala.reflect.macros.blackbox.Context) (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[W[_]]): c0.Expr[W[T]] = { import c0.universe._ diff --git a/upickle/implicits/src-3/upickle/implicits/Readers.scala b/upickle/implicits/src-3/upickle/implicits/Readers.scala index ab677be13..e6d15303d 100644 --- a/upickle/implicits/src-3/upickle/implicits/Readers.scala +++ b/upickle/implicits/src-3/upickle/implicits/Readers.scala @@ -13,7 +13,7 @@ trait ReadersVersionSpecific with Annotator with CaseClassReadWriters: - abstract class CaseClassReader3[T](paramCount: Int, + abstract class CaseClassReader3V2[T](paramCount: Int, missingKeyCount: Long, allowUnknownKeys: Boolean, construct: (Array[Any], scala.collection.mutable.Map[String, Any]) => T) extends CaseClassReader[T] { @@ -102,7 +102,7 @@ trait ReadersVersionSpecific case m: Mirror.ProductOf[T] => macros.validateFlattenAnnotation[T]() val paramCount = macros.paramsCount[T] - val reader = new CaseClassReader3[T]( + val reader = new CaseClassReader3V2[T]( paramCount, if (paramCount <= 64) if (paramCount == 64) -1 else (1L << paramCount) - 1 else paramCount, @@ -159,4 +159,52 @@ trait ReadersVersionSpecific implicit class ReaderExtension(r: Reader.type): inline def derived[T](using Mirror.Of[T]): Reader[T] = macroRAll[T] end ReaderExtension + + @deprecated + abstract class CaseClassReader3[T](paramCount: Int, + missingKeyCount: Long, + allowUnknownKeys: Boolean, + construct: Array[Any] => T) extends CaseClassReader[T] { + def visitors0: Product + lazy val visitors = visitors0 + def fromProduct(p: Product): T + def keyToIndex(x: String): Int + def allKeysArray: Array[String] + def storeDefaults(x: upickle.implicits.BaseCaseObjectContext): Unit + + trait ObjectContext extends ObjVisitor[Any, T] with BaseCaseObjectContext { + private val params = new Array[Any](paramCount) + + def storeAggregatedValue(currentIndex: Int, v: Any): Unit = params(currentIndex) = v + + def subVisitor: Visitor[_, _] = + if (currentIndex == -1) upickle.core.NoOpVisitor + else visitors.productElement(currentIndex).asInstanceOf[Visitor[_, _]] + + def visitKeyValue(v: Any): Unit = + val k = objectAttributeKeyReadMap(v.toString).toString + currentIndex = keyToIndex(k) + if (currentIndex == -1 && !allowUnknownKeys) { + throw new upickle.core.Abort("Unknown Key: " + k.toString) + } + + def visitEnd(index: Int): T = + storeDefaults(this) + + // Special-case 64 because java bit shifting ignores any RHS values above 63 + // https://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.19 + if (this.checkErrorMissingKeys(missingKeyCount)) + this.errorMissingKeys(paramCount, allKeysArray) + + construct(params) + } + + override def visitObject(length: Int, + jsonableKeys: Boolean, + index: Int) = + if (paramCount <= 64) new CaseObjectContext[T](paramCount) with ObjectContext + else new HugeCaseObjectContext[T](paramCount) with ObjectContext + } + + end ReadersVersionSpecific diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 0a65eba9c..426c36c08 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -595,3 +595,63 @@ private def isCaseClass(using Quotes)(typeSymbol: quotes.reflect.Symbol): Boolea import quotes.reflect._ typeSymbol.isClassDef && typeSymbol.flags.is(Flags.Case) } + +@deprecated +def fieldLabelsImpl[T](using Quotes, Type[T]): Expr[List[(String, String)]] = + Expr.ofList(fieldLabelsImpl0[T].map((a, b) => Expr((a.name, b)))) + +@deprecated +def checkErrorMissingKeysCountImpl[T]()(using Quotes, Type[T]): Expr[Long] = + import quotes.reflect.* + val paramCount = fieldLabelsImpl0[T].size + if (paramCount <= 64) if (paramCount == 64) Expr(-1) else Expr((1L << paramCount) - 1) + else Expr(paramCount) + +@deprecated +def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]]): Expr[T] = + import quotes.reflect._ + def apply(typeApply: Option[List[TypeRepr]]) = { + val tpe = TypeRepr.of[T] + val companion: Symbol = tpe.classSymbol.get.companionModule + val constructorSym = tpe.typeSymbol.primaryConstructor + val constructorParamSymss = constructorSym.paramSymss + + val (tparams0, params0) = constructorParamSymss.flatten.partition(_.isType) + val constructorTpe = tpe.memberType(constructorSym).widen + + val rhs = params0.zipWithIndex.map { + case (sym0, i) => + val lhs = '{ $params(${ Expr(i) }) } + val tpe0 = constructorTpe.memberType(sym0) + + typeApply.map(tps => tpe0.substituteTypes(tparams0, tps)).getOrElse(tpe0) match { + case AnnotatedType(AppliedType(base, Seq(arg)), x) + if x.tpe =:= defn.RepeatedAnnot.typeRef => + arg.asType match { + case '[t] => + Typed( + lhs.asTerm, + TypeTree.of(using AppliedType(defn.RepeatedParamClass.typeRef, List(arg)).asType) + ) + } + case tpe => + tpe.asType match { + case '[t] => '{ $lhs.asInstanceOf[t] }.asTerm + } + } + + } + + typeApply match { + case None => Select.overloaded(Ref(companion), "apply", Nil, rhs).asExprOf[T] + case Some(args) => + Select.overloaded(Ref(companion), "apply", args, rhs).asExprOf[T] + } + } + + TypeRepr.of[T] match { + case t: AppliedType => apply(Some(t.args)) + case t: TypeRef => apply(None) + case t: TermRef => '{ ${ Ref(t.classSymbol.get.companionModule).asExprOf[Any] }.asInstanceOf[T] } + } + From 6766ec23be9d032545a2a46373d9e1ffd281ba87 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Tue, 10 Dec 2024 21:58:07 +0900 Subject: [PATCH 08/23] remove calls to deprecated api --- .../upickle/implicits/internal/Macros2.scala | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 3b4a950c9..b8bc08b1f 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -21,7 +21,7 @@ object Macros2 { trait DeriveDefaults[M[_]] { val c: scala.reflect.macros.blackbox.Context - private[upickle] def fail(tpe: c.Type, s: String) = c.abort(c.enclosingPosition, s) + private[upickle] def fail(s: String) = c.abort(c.enclosingPosition, s) import c.universe._ private[upickle] def companionTree(tpe: c.Type): Tree = { @@ -33,7 +33,7 @@ object Macros2 { s"[[${clsSymbol.name}]]. This may be due to a bug in scalac (SI-7567) " + "that arises when a case class within a function is upickle. As a " + "workaround, move the declaration to the module-level." - fail(tpe, msg) + fail(msg) } else { val symTab = c.universe.asInstanceOf[reflect.internal.SymbolTable] val pre = tpe.asInstanceOf[symTab.Type].prefix.asInstanceOf[Type] @@ -71,7 +71,6 @@ object Macros2 { val mod2 = c.universe.internal.gen.mkAttributedRef(pre, mod) annotate(tpe)(wrapObject(mod2)) - } private[upickle] def mergeTrait(tagKey: Option[String], subtrees: Seq[Tree], subtypes: Seq[Type], targetType: c.Type): Tree @@ -89,14 +88,14 @@ object Macros2 { val clsSymbol = tpe.typeSymbol.asClass if (!clsSymbol.isSealed) { - fail(tpe, s"[error] The referenced trait [[${clsSymbol.name}]] must be sealed.") + fail(s"[error] The referenced trait [[${clsSymbol.name}]] must be sealed.") }else if (clsSymbol.knownDirectSubclasses.filter(!_.toString.contains("")).isEmpty) { val msg = s"The referenced trait [[${clsSymbol.name}]] does not have any sub-classes. This may " + "happen due to a limitation of scalac (SI-7046). To work around this, " + "try manually specifying the sealed trait picklers as described in " + "https://com-lihaoyi.github.io/upickle/#ManualSealedTraitPicklers" - fail(tpe, msg) + fail(msg) }else{ val tagKey = customKey(clsSymbol) val subTypes = fleshedOutSubtypes(tpe).toSeq.sortBy(_.typeSymbol.fullName) @@ -156,7 +155,7 @@ object Macros2 { else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { getFields(tpeOfField) } - else fail(tpeOfField, s"Invalid type for flattening getField: $tpeOfField.") + else fail(s"Invalid type for flattening getField: $tpeOfField.") case None => List((param.name.toString, mappedName, tpeOfField, param, defaultValue, false)) } @@ -172,7 +171,7 @@ object Macros2 { tpe.members.find(x => x.isMethod && x.asMethod.isPrimaryConstructor) match { case Some(primaryConstructor) => primaryConstructor.asMethod.paramLists.flatten - case None => fail(tpe, "Can't find primary constructor of " + tpe) + case None => fail("Can't find primary constructor of " + tpe) } } @@ -201,10 +200,10 @@ object Macros2 { private def validateFlattenAnnotation(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)]): Unit = { if (fields.count { case(_, _, _, _, _, isCollection) => isCollection } > 1) { - fail(NoType, "Only one collection can be annotated with @upickle.implicits.flatten in the same level") + fail("Only one collection can be annotated with @upickle.implicits.flatten in the same level") } if (fields.map { case (_, mappedName, _, _, _, _) => mappedName }.distinct.length != fields.length) { - fail(NoType, "There are multiple fields with the same key") + fail("There are multiple fields with the same key") } } @@ -221,7 +220,7 @@ object Macros2 { sealedParents, customKey, (_: c.Symbol).name.toString, - fail(tpe, _), + fail, ) val sealedClassSymbol: Option[Symbol] = sealedParents.find(_ == tpe.typeSymbol) @@ -263,15 +262,15 @@ object Macros2 { private def customKey(sym: c.Symbol): Option[String] = { sym.annotations - .find(_.tpe == typeOf[key]) - .flatMap(_.scalaArgs.headOption) + .find(_.tree.tpe == typeOf[key]) + .flatMap(_.tree.children.tail.headOption) .map{case Literal(Constant(s)) => s.toString} } private[upickle] def serializeDefaults(sym: c.Symbol): Option[Boolean] = { sym.annotations - .find(_.tpe == typeOf[upickle.implicits.serializeDefaults]) - .flatMap(_.scalaArgs.headOption) + .find(_.tree.tpe == typeOf[upickle.implicits.serializeDefaults]) + .flatMap(_.tree.children.tail.headOption) .map{case Literal(Constant(s)) => s.asInstanceOf[Boolean]} } @@ -326,7 +325,7 @@ object Macros2 { val (term, nextIdx) = loop(tpeOfField, idx) (term :: terms, nextIdx) } - else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + else fail(s"Invalid type for flattening: $tpeOfField.") case None => val termName = TermName(s"aggregated$idx") val term = if (symbol == symbols.last && varArgs) q"$termName:_*" else q"$termName" @@ -526,7 +525,7 @@ object Macros2 { } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { write(tpeOfField, select) } - else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + else fail(s"Invalid type for flattening: $tpeOfField.") case None => val snippet = q""" @@ -557,7 +556,7 @@ object Macros2 { else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { getLength(tpeOfField, select) } - else fail(tpeOfField, s"Invalid type for flattening: $tpeOfField.") + else fail(s"Invalid type for flattening: $tpeOfField.") case None => val snippet = if (defaultValue.isEmpty) q"1" else q"""if (${serDfltVals(symbol)} || $select != ${defaultValue.get}) 1 else 0""" From a2470b1b618bc45096d2bac4c2aaa4912ec23f0a Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Wed, 11 Dec 2024 09:45:41 +0900 Subject: [PATCH 09/23] polish --- .../implicits/src-2/upickle/implicits/internal/Macros.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala index 53eda98d4..211f0aba9 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros.scala @@ -472,7 +472,7 @@ object Macros { } } - @deprecated("Use one's from Macros2.scala") + @deprecated("Use Macros2 instead") def macroRImpl[T, R[_]](c0: scala.reflect.macros.blackbox.Context) (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[R[_]]): c0.Expr[R[T]] = { import c0.universe._ @@ -484,7 +484,7 @@ object Macros { c0.Expr[R[T]](res) } - @deprecated("Use one's from Macros2.scala") + @deprecated("Use Macros2 instead") def macroWImpl[T, W[_]](c0: scala.reflect.macros.blackbox.Context) (implicit e1: c0.WeakTypeTag[T], e2: c0.WeakTypeTag[W[_]]): c0.Expr[W[T]] = { import c0.universe._ From 28d4f5786910f2372ea7bde7c60194b300decea6 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Wed, 11 Dec 2024 22:25:20 +0900 Subject: [PATCH 10/23] Support map with non-string key --- .../upickle/implicits/internal/Macros2.scala | 28 ++++++++++--------- upickle/test/src/upickle/MacroTests.scala | 18 ++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index b8bc08b1f..bd47d96cb 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -150,7 +150,7 @@ object Macros2 { val (mappedName, tpeOfField, defaultValue) = getSymbolDetail(param, i, tpe) param.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => - if (isFlattableCollection(tpeOfField)) + if (isCollectionFlattenable(tpeOfField)) List((param.name.toString, mappedName, tpeOfField, param, defaultValue, true)) else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { getFields(tpeOfField) @@ -256,7 +256,7 @@ object Macros2 { } } - private[upickle] def isFlattableCollection(tpe: c.Type): Boolean = { + private[upickle] def isCollectionFlattenable(tpe: c.Type): Boolean = { tpe.typeSymbol == typeOf[collection.immutable.Map[_, _]].typeSymbol } @@ -296,16 +296,17 @@ object Macros2 { val (mappedNames, types, defaultValues) = fields.toArray.filter { case (_, _, _, _, _, isCollection) => !isCollection }.map { case (_, mappedName, tpe, _, defaultValue, _) => (mappedName, tpe, defaultValue) }.unzip3 - val (hasFlattenOnMap, valueTypeOfMap) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { + val (hasFlattenOnMap, keyTypeOfMap, valueTypeOfMap) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { case Some(f) => - val TypeRef(_, _, _ :: valueType :: Nil) = f._3 - (true, valueType) - case None => (false, NoType) + val TypeRef(_, _, keyType :: valueType :: Nil) = f._3 + (true, keyType, valueType) + case None => (false, NoType, NoType) } val numberOfFields = mappedNames.length val (localReaders, aggregates) = mappedNames.zipWithIndex.map { case (_, idx) => (TermName(s"localReader$idx"), TermName(s"aggregated$idx")) }.unzip + val readKey = if (keyTypeOfMap =:= typeOf[String]) q"currentKey" else q"read(currentKey)(implicitly[${c.prefix}.Reader[$keyTypeOfMap]])" def constructClass(constructedTpe: c.Type): c.universe.Tree = { def loop(tpe: c.Type, offset: Int): (c.universe.Tree, Int) = { @@ -317,7 +318,7 @@ object Macros2 { val (_, tpeOfField, _) = getSymbolDetail(symbol, 0, tpe) symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => - if (isFlattableCollection(tpeOfField)) { + if (isCollectionFlattenable(tpeOfField)) { val termName = TermName(s"aggregatedMap") val term = q"$termName.toMap" (term :: terms, idx) @@ -397,7 +398,7 @@ object Macros2 { ..${ if (hasFlattenOnMap) List( - q"private[this] lazy val aggregatedMap: scala.collection.mutable.ListBuffer[(String, $valueTypeOfMap)] = scala.collection.mutable.ListBuffer.empty", + q"private[this] lazy val aggregatedMap: scala.collection.mutable.ListBuffer[($keyTypeOfMap, $valueTypeOfMap)] = scala.collection.mutable.ListBuffer.empty", ) else Nil } @@ -409,7 +410,7 @@ object Macros2 { } case ..${ if (hasFlattenOnMap) - List(cq"-1 => aggregatedMap += currentKey -> v.asInstanceOf[$valueTypeOfMap]") + List(cq"-1 => aggregatedMap += $readKey -> v.asInstanceOf[$valueTypeOfMap]") else Nil } case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) @@ -510,13 +511,14 @@ object Macros2 { symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => - if (isFlattableCollection(tpeOfField)) { - val TypeRef(_, _, _ :: valueType :: Nil) = tpeOfField + if (isCollectionFlattenable(tpeOfField)) { + val TypeRef(_, _, keyType :: valueType :: Nil) = tpeOfField + val writeKey = if (keyType =:= typeOf[String]) q"key" else q"upickle.default.write(key)(implicitly[${c.prefix}.Writer[$keyType]])" q""" $select.foreach { case (key, value) => this.writeSnippetMappedName[R, $valueType]( ctx, - key.toString, + $writeKey, implicitly[${c.prefix}.Writer[$valueType]], value ) @@ -550,7 +552,7 @@ object Macros2 { val select = Select(outer, TermName(symbol.name.toString)) symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => - if (isFlattableCollection(tpeOfField)) { + if (isCollectionFlattenable(tpeOfField)) { q"${select}.size" :: Nil } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 9217e15ac..527f4dbd2 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -202,6 +202,15 @@ object Flatten { object NestedWithDefault { implicit val rw: RW[NestedWithDefault] = upickle.default.macroRW } + + case class KeyClass(id: Int, name: String) + object KeyClass { + implicit val rw: RW[KeyClass] = upickle.default.macroRW + } + case class FlattenWithKey(@upickle.implicits.flatten n: Map[KeyClass, String]) + object FlattenWithKey { + implicit val rw: RW[FlattenWithKey] = upickle.default.macroRW + } } object MacroTests extends TestSuite { @@ -969,6 +978,7 @@ object MacroTests extends TestSuite { import Flatten._ val value = HasMap(Map.empty, 10) rw(value, """{"i":10}""") + write(1) } test("flattenWithDefaults") { @@ -976,5 +986,13 @@ object MacroTests extends TestSuite { val value = FlattenWithDefault(10, NestedWithDefault(l = "default")) rw(value, """{"i":10,"l":"default"}""") } + + test("flattenWithKey") { + import Flatten._ + val value = FlattenWithKey(Map(KeyClass(1, "a") -> "value1", KeyClass(2, "b") -> "value2")) + println(write(KeyClass(1, "a"))) + println(write(value)) + rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") + } } } From b8c186f6a70ff7fceeef112786105d84f99ed58c Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Wed, 11 Dec 2024 23:11:00 +0900 Subject: [PATCH 11/23] flatten collection --- .../upickle/implicits/internal/Macros2.scala | 44 +++++++++++-------- upickle/test/src/upickle/MacroTests.scala | 11 +++++ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index bd47d96cb..8eed06879 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -257,9 +257,17 @@ object Macros2 { } private[upickle] def isCollectionFlattenable(tpe: c.Type): Boolean = { - tpe.typeSymbol == typeOf[collection.immutable.Map[_, _]].typeSymbol + (tpe <:< typeOf[collection.Map[_, _]] + || tpe <:< typeOf[Iterable[(_, _)]]) } + private[upickle] def extractKeyValueTypes(tpe: c.Type): (Symbol, c.Type, c.Type) = + tpe match { + case TypeRef(_, collection, keyType :: valueType :: Nil) => (collection, keyType, valueType) + case TypeRef(_, collection, TypeRef(_, _, keyType :: valueType :: Nil) :: Nil) => (collection, keyType, valueType) + case _ => fail(s"Fail to extract key value from $tpe") + } + private def customKey(sym: c.Symbol): Option[String] = { sym.annotations .find(_.tree.tpe == typeOf[key]) @@ -296,17 +304,17 @@ object Macros2 { val (mappedNames, types, defaultValues) = fields.toArray.filter { case (_, _, _, _, _, isCollection) => !isCollection }.map { case (_, mappedName, tpe, _, defaultValue, _) => (mappedName, tpe, defaultValue) }.unzip3 - val (hasFlattenOnMap, keyTypeOfMap, valueTypeOfMap) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { + val (hasFlattenOnCollection, collection, keyTypeOfCollection, valueTypeOfCollection) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { case Some(f) => - val TypeRef(_, _, keyType :: valueType :: Nil) = f._3 - (true, keyType, valueType) - case None => (false, NoType, NoType) + val (collection, key, value) = extractKeyValueTypes(f._3) + (true, collection, key, value) + case None => (false, NoSymbol, NoType, NoType) } val numberOfFields = mappedNames.length val (localReaders, aggregates) = mappedNames.zipWithIndex.map { case (_, idx) => (TermName(s"localReader$idx"), TermName(s"aggregated$idx")) }.unzip - val readKey = if (keyTypeOfMap =:= typeOf[String]) q"currentKey" else q"read(currentKey)(implicitly[${c.prefix}.Reader[$keyTypeOfMap]])" + val readKey = if (keyTypeOfCollection =:= typeOf[String]) q"currentKey" else q"read(currentKey)(implicitly[${c.prefix}.Reader[$keyTypeOfCollection]])" def constructClass(constructedTpe: c.Type): c.universe.Tree = { def loop(tpe: c.Type, offset: Int): (c.universe.Tree, Int) = { @@ -319,8 +327,8 @@ object Macros2 { symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => if (isCollectionFlattenable(tpeOfField)) { - val termName = TermName(s"aggregatedMap") - val term = q"$termName.toMap" + val termName = TermName(s"aggregatedCollection") + val term = q"${Ident(TermName(collection.name.toString))}.from($termName)" (term :: terms, idx) } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { val (term, nextIdx) = loop(tpeOfField, idx) @@ -378,9 +386,9 @@ object Macros2 { yield q"private[this] lazy val ${localReaders(i)} = implicitly[${c.prefix}.Reader[${types(i)}]]" } ..${ - if (hasFlattenOnMap) + if (hasFlattenOnCollection) List( - q"private[this] lazy val localReaderMap = implicitly[${c.prefix}.Reader[$valueTypeOfMap]]", + q"private[this] lazy val localReaderCollection = implicitly[${c.prefix}.Reader[$valueTypeOfCollection]]", ) else Nil } @@ -396,9 +404,9 @@ object Macros2 { yield q"private[this] var ${aggregates(i)}: ${types(i)} = _" } ..${ - if (hasFlattenOnMap) + if (hasFlattenOnCollection) List( - q"private[this] lazy val aggregatedMap: scala.collection.mutable.ListBuffer[($keyTypeOfMap, $valueTypeOfMap)] = scala.collection.mutable.ListBuffer.empty", + q"private[this] lazy val aggregatedCollection: scala.collection.mutable.ListBuffer[($keyTypeOfCollection, $valueTypeOfCollection)] = scala.collection.mutable.ListBuffer.empty", ) else Nil } @@ -409,8 +417,8 @@ object Macros2 { yield cq"$i => ${aggregates(i)} = v.asInstanceOf[${types(i)}]" } case ..${ - if (hasFlattenOnMap) - List(cq"-1 => aggregatedMap += $readKey -> v.asInstanceOf[$valueTypeOfMap]") + if (hasFlattenOnCollection) + List(cq"-1 => aggregatedCollection += $readKey -> v.asInstanceOf[$valueTypeOfCollection]") else Nil } case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) @@ -426,7 +434,7 @@ object Macros2 { } case _ => ${ - (allowUnknownKeysAnnotation, hasFlattenOnMap) match { + (allowUnknownKeysAnnotation, hasFlattenOnCollection) match { case (_, true) => q"storeToMap = true; -1" case (None, false) => q""" @@ -461,8 +469,8 @@ object Macros2 { def subVisitor: _root_.upickle.core.Visitor[_, _] = currentIndex match { case -1 => ${ - if (hasFlattenOnMap) - q"localReaderMap" + if (hasFlattenOnCollection) + q"localReaderCollection" else q"_root_.upickle.core.NoOpVisitor" } @@ -512,7 +520,7 @@ object Macros2 { symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => if (isCollectionFlattenable(tpeOfField)) { - val TypeRef(_, _, keyType :: valueType :: Nil) = tpeOfField + val (_, keyType, valueType) = extractKeyValueTypes(tpeOfField) val writeKey = if (keyType =:= typeOf[String]) q"key" else q"upickle.default.write(key)(implicitly[${c.prefix}.Writer[$keyType]])" q""" $select.foreach { case (key, value) => diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 527f4dbd2..f1393c036 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -211,6 +211,11 @@ object Flatten { object FlattenWithKey { implicit val rw: RW[FlattenWithKey] = upickle.default.macroRW } + + case class FlattenSeq(@upickle.implicits.flatten n: Seq[(String, Int)]) + object FlattenSeq { + implicit val rw: RW[FlattenSeq] = upickle.default.macroRW + } } object MacroTests extends TestSuite { @@ -994,5 +999,11 @@ object MacroTests extends TestSuite { println(write(value)) rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") } + + test("flattenSeq") { + import Flatten._ + val value = FlattenSeq(Seq("a" -> 1, "b" -> 2)) + rw(value, """{"a":1,"b":2}""") + } } } From 1fb37954d1525bbf56d5cb90f4ff9d0cb8646e8a Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Thu, 12 Dec 2024 22:19:52 +0900 Subject: [PATCH 12/23] wip --- .../upickle/implicits/internal/Macros2.scala | 9 ++++----- upickle/test/src/upickle/MacroTests.scala | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 8eed06879..ca9feea30 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -256,10 +256,8 @@ object Macros2 { } } - private[upickle] def isCollectionFlattenable(tpe: c.Type): Boolean = { - (tpe <:< typeOf[collection.Map[_, _]] - || tpe <:< typeOf[Iterable[(_, _)]]) - } + private[upickle] def isCollectionFlattenable(tpe: c.Type): Boolean = + tpe <:< typeOf[Iterable[(_, _)]] private[upickle] def extractKeyValueTypes(tpe: c.Type): (Symbol, c.Type, c.Type) = tpe match { @@ -328,7 +326,8 @@ object Macros2 { case Some(_) => if (isCollectionFlattenable(tpeOfField)) { val termName = TermName(s"aggregatedCollection") - val term = q"${Ident(TermName(collection.name.toString))}.from($termName)" + val fullyQualifiedCollection = c.parse(collection.fullName) + val term = q"$fullyQualifiedCollection($termName.toList :_*)" (term :: terms, idx) } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { val (term, nextIdx) = loop(tpeOfField, idx) diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index f1393c036..486720cc9 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -216,6 +216,15 @@ object Flatten { object FlattenSeq { implicit val rw: RW[FlattenSeq] = upickle.default.macroRW } + + case class ValueClass(value: Double) + object ValueClass { + implicit val rw: RW[ValueClass] = upickle.default.macroRW + } + case class Collection(@upickle.implicits.flatten n: scala.collection.mutable.LinkedHashMap[KeyClass, ValueClass]) + object Collection { + implicit val rw: RW[Collection] = upickle.default.macroRW + } } object MacroTests extends TestSuite { @@ -995,8 +1004,6 @@ object MacroTests extends TestSuite { test("flattenWithKey") { import Flatten._ val value = FlattenWithKey(Map(KeyClass(1, "a") -> "value1", KeyClass(2, "b") -> "value2")) - println(write(KeyClass(1, "a"))) - println(write(value)) rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") } @@ -1005,5 +1012,11 @@ object MacroTests extends TestSuite { val value = FlattenSeq(Seq("a" -> 1, "b" -> 2)) rw(value, """{"a":1,"b":2}""") } + + test("flattenLinkedHashMap") { + import Flatten._ + val value = Collection(scala.collection.mutable.LinkedHashMap(KeyClass(1, "a") -> ValueClass(3.0), KeyClass(2, "b") -> ValueClass(4.0))) + rw(value, """{"{\"id\":1,\"name\":\"a\"}":{"value":3},"{\"id\":2,\"name\":\"b\"}":{"value":4}}""") + } } } From a07839dc9b4c0c7a9ebffac2f32861d55e4751f8 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Wed, 18 Dec 2024 22:12:36 +0900 Subject: [PATCH 13/23] Support flatten iterable --- .../upickle/implicits/internal/Macros2.scala | 41 ++++++++++----- .../src-3/upickle/implicits/Readers.scala | 10 ++-- .../src-3/upickle/implicits/macros.scala | 51 +++++++++--------- .../src/upickle/implicits/MacrosCommon.scala | 2 +- .../implicits/src/upickle/implicits/key.scala | 8 +-- upickle/test/src/upickle/FailureTests.scala | 3 ++ upickle/test/src/upickle/MacroTests.scala | 52 +++++++++---------- 7 files changed, 92 insertions(+), 75 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index ca9feea30..820fc9b50 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -133,11 +133,18 @@ object Macros2 { companion.tpe.member(TermName("apply")).info validateFlattenAnnotation(fields) + val (mappedNames, types, defaultValues) = fields.toArray.filter { case (_, _, _, _, _, isCollection) => !isCollection }.map { + case (_, mappedName, tpe, _, defaultValue, _) => (mappedName, tpe, defaultValue) + }.unzip3 + val collectionFlattened = fields.find { case (_, _, _, _, _, isCollection) => isCollection }.map(_._3) val derive = // Otherwise, reading and writing are kinda identical wrapCaseN( - fields, + mappedNames, + types, + defaultValues, + collectionFlattened, targetType = tpe, ) @@ -155,7 +162,7 @@ object Macros2 { else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { getFields(tpeOfField) } - else fail(s"Invalid type for flattening getField: $tpeOfField.") + else fail(s"Invalid type for flattening: $tpeOfField.") case None => List((param.name.toString, mappedName, tpeOfField, param, defaultValue, false)) } @@ -257,7 +264,7 @@ object Macros2 { } private[upickle] def isCollectionFlattenable(tpe: c.Type): Boolean = - tpe <:< typeOf[Iterable[(_, _)]] + tpe <:< typeOf[Iterable[(String, _)]] private[upickle] def extractKeyValueTypes(tpe: c.Type): (Symbol, c.Type, c.Type) = tpe match { @@ -282,7 +289,10 @@ object Macros2 { private[upickle] def wrapObject(obj: Tree): Tree - private[upickle] def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + private[upickle] def wrapCaseN(mappedNames: Array[String], + types: Array[c.Type], + defaultValues: Array[Option[c.Tree]], + collectionFlattened: Option[c.Type], targetType: c.Type): Tree } @@ -291,7 +301,10 @@ object Macros2 { import c.universe._ def wrapObject(t: c.Tree) = q"new ${c.prefix}.SingletonReader($t)" - def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + def wrapCaseN(mappedNames: Array[String], + types: Array[c.Type], + defaultValues: Array[Option[c.Tree]], + collectionFlattened: Option[c.Type], targetType: c.Type): c.universe.Tree = { val allowUnknownKeysAnnotation = targetType.typeSymbol .annotations @@ -299,12 +312,9 @@ object Macros2 { .flatMap(_.tree.children.tail.headOption) .map { case Literal(Constant(b)) => b.asInstanceOf[Boolean] } - val (mappedNames, types, defaultValues) = fields.toArray.filter { case (_, _, _, _, _, isCollection) => !isCollection }.map { - case (_, mappedName, tpe, _, defaultValue, _) => (mappedName, tpe, defaultValue) - }.unzip3 - val (hasFlattenOnCollection, collection, keyTypeOfCollection, valueTypeOfCollection) = fields.find { case (_, _, _, _, _, isCollection) => isCollection } match { - case Some(f) => - val (collection, key, value) = extractKeyValueTypes(f._3) + val (hasFlattenOnCollection, _, keyTypeOfCollection, valueTypeOfCollection) = collectionFlattened match { + case Some(tpe) => + val (collection, key, value) = extractKeyValueTypes(tpe) (true, collection, key, value) case None => (false, NoSymbol, NoType, NoType) } @@ -326,8 +336,8 @@ object Macros2 { case Some(_) => if (isCollectionFlattenable(tpeOfField)) { val termName = TermName(s"aggregatedCollection") - val fullyQualifiedCollection = c.parse(collection.fullName) - val term = q"$fullyQualifiedCollection($termName.toList :_*)" + val companion = companionTree(tpeOfField) + val term = q"$companion($termName.toList :_*)" (term :: terms, idx) } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { val (term, nextIdx) = loop(tpeOfField, idx) @@ -500,7 +510,10 @@ object Macros2 { def internal = q"${c.prefix}.Internal" - def wrapCaseN(fields: List[(String, String, c.Type, Symbol, Option[c.Tree], Boolean)], + def wrapCaseN(mappedNames: Array[String], + types: Array[c.Type], + defaultValues: Array[Option[c.Tree]], + collectionFlattened: Option[c.Type], targetType: c.Type): c.universe.Tree = { def serDfltVals(symbol: Symbol) = { val b: Option[Boolean] = serializeDefaults(symbol).orElse(serializeDefaults(targetType.typeSymbol)) diff --git a/upickle/implicits/src-3/upickle/implicits/Readers.scala b/upickle/implicits/src-3/upickle/implicits/Readers.scala index e6d15303d..2de2135a2 100644 --- a/upickle/implicits/src-3/upickle/implicits/Readers.scala +++ b/upickle/implicits/src-3/upickle/implicits/Readers.scala @@ -16,7 +16,7 @@ trait ReadersVersionSpecific abstract class CaseClassReader3V2[T](paramCount: Int, missingKeyCount: Long, allowUnknownKeys: Boolean, - construct: (Array[Any], scala.collection.mutable.Map[String, Any]) => T) extends CaseClassReader[T] { + construct: (Array[Any], scala.collection.mutable.ListBuffer[(String, Any)]) => T) extends CaseClassReader[T] { def visitors0: (AnyRef, Array[AnyRef]) lazy val (visitorMap, visitors) = visitors0 @@ -26,14 +26,14 @@ trait ReadersVersionSpecific def storeDefaults(x: upickle.implicits.BaseCaseObjectContext): Unit trait ObjectContext extends ObjVisitor[Any, T] with BaseCaseObjectContext { private val params = new Array[Any](paramCount) - private val map = scala.collection.mutable.Map.empty[String, Any] + private val collection = scala.collection.mutable.ListBuffer.empty[(String, Any)] private var currentKey = "" protected var storeToMap = false def storeAggregatedValue(currentIndex: Int, v: Any): Unit = if (currentIndex == -1) { if (storeToMap) { - map(currentKey) = v + collection += (currentKey -> v) } } else { params(currentIndex) = v @@ -68,7 +68,7 @@ trait ReadersVersionSpecific if (this.checkErrorMissingKeys(missingKeyCount)) this.errorMissingKeys(paramCount, allKeysArray) - construct(params, map) + construct(params, collection) } override def visitObject(length: Int, @@ -107,7 +107,7 @@ trait ReadersVersionSpecific if (paramCount <= 64) if (paramCount == 64) -1 else (1L << paramCount) - 1 else paramCount, macros.extractIgnoreUnknownKeys[T]().headOption.getOrElse(this.allowUnknownKeys), - (params: Array[Any], map :scala.collection.mutable.Map[String ,Any]) => macros.applyConstructor[T](params, map) + (params: Array[Any], collection :scala.collection.mutable.ListBuffer[(String ,Any)]) => macros.applyConstructor[T](params, collection) ){ override def visitors0 = macros.allReaders[T, Reader] override def keyToIndex(x: String): Int = macros.keyToIndex[T](x) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 426c36c08..aacc97e6c 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -82,7 +82,7 @@ def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRef, Array val fields = allFields[T] val (readerMap, readers) = fields.partitionMap { case (_, _, tpe, _, isFlattenMap) => if (isFlattenMap) { - val valueTpe = tpe.typeArgs(1) + val (_, _, valueTpe) = extractKeyValueTypes(tpe) val readerTpe = TypeRepr.of[R].appliedTo(valueTpe) val reader = readerTpe.asType match { case '[t] => '{summonInline[t].asInstanceOf[AnyRef]} @@ -135,7 +135,7 @@ def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String, qu val substitutedTypeRepr = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) val typeSymbol = substitutedTypeRepr.typeSymbol if (flatten) { - if (isMap(substitutedTypeRepr)) { + if (isCollectionFlattenable(substitutedTypeRepr)) { (field, label, substitutedTypeRepr, defaults.get(label), true) :: Nil } else if (isCaseClass(typeSymbol)) { @@ -218,9 +218,9 @@ def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.implicits if (flatten) { val subsitituted = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) val typeSymbol = subsitituted.typeSymbol - if (isMap(subsitituted)) { + if (isCollectionFlattenable(subsitituted)) { List( - '{${select.asExprOf[Map[_, _]]}.size} + '{${select.asExprOf[Iterable[(_, _)]]}.size} ) } else if (isCaseClass(typeSymbol)) { @@ -274,16 +274,13 @@ def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types with upickl val fieldTypeRepr = substituteTypeArgs(classTypeRepr, subsitituted = classTypeRepr.memberType(field)) val typeSymbol = fieldTypeRepr.typeSymbol if (flatten) { - if (isMap(fieldTypeRepr)) { - val (keyTpe0, valueTpe0) = fieldTypeRepr.typeArgs match { - case key :: value :: Nil => (key, value) - case _ => report.errorAndAbort(s"Unsupported type ${typeSymbol} for flattening", v.asTerm.pos) - } + if (isCollectionFlattenable(fieldTypeRepr)) { + val (_, keyTpe0, valueTpe0) = extractKeyValueTypes(fieldTypeRepr) val writerTpe0 = TypeRepr.of[W].appliedTo(valueTpe0) (keyTpe0.asType, valueTpe0.asType, writerTpe0.asType) match { case ('[keyTpe], '[valueTpe], '[writerTpe])=> val snippet = '{ - ${select.asExprOf[Map[keyTpe, valueTpe]]}.foreach { (k, v) => + ${select.asExprOf[Iterable[(keyTpe, valueTpe)]]}.foreach { (k, v) => ${self}.writeSnippetMappedName[R, valueTpe]( ${ctx}, k.toString, @@ -308,7 +305,7 @@ def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types with upickl case _ => report.errorAndAbort("Unsupported type for flattening", v) } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map", v.asTerm.pos) + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a subtype of Iterable", v.asTerm.pos) } else { val tpe0 = fieldTypeRepr @@ -379,8 +376,8 @@ def substituteTypeArgs(using Quotes)(tpe: quotes.reflect.TypeRepr, subsitituted: subsitituted.substituteTypes(tparams0 ,tpe.typeArgs) } -inline def applyConstructor[T](params: Array[Any], map: scala.collection.mutable.Map[String, Any]): T = ${ applyConstructorImpl[T]('params, 'map) } -def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]], map: Expr[scala.collection.mutable.Map[String, Any]]): Expr[T] = +inline def applyConstructor[T](params: Array[Any], collection: scala.collection.mutable.ListBuffer[(String, Any)]): T = ${ applyConstructorImpl[T]('params, 'collection) } +def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]], collection: Expr[scala.collection.mutable.ListBuffer[(String, Any)]]): Expr[T] = import quotes.reflect._ def apply(tpe: TypeRepr, typeArgs: List[TypeRepr], offset: Int): (Term, Int) = { val companion: Symbol = tpe.classSymbol.get.companionModule @@ -396,13 +393,13 @@ def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Arra val typeSymbol = appliedTpe.typeSymbol val flatten = extractFlatten(sym0) if (flatten) { - if (isMap(appliedTpe)) { - val keyTpe0 = appliedTpe.typeArgs.head - val valueTpe0 = appliedTpe.typeArgs(1) + if (isCollectionFlattenable(appliedTpe)) { + val (_, keyTpe0, valueTpe0) = extractKeyValueTypes(appliedTpe) (keyTpe0.asType, valueTpe0.asType) match { case ('[keyTpe], '[valueTpe]) => - val typedMap = '{${map}.asInstanceOf[collection.mutable.Map[keyTpe, valueTpe]]}.asTerm - val term = Select.unique(typedMap, "toMap") + val typedCollection = '{${collection}.asInstanceOf[scala.collection.mutable.ListBuffer[(keyTpe, valueTpe)]]}.asTerm + val companionSym = typeSymbol.companionModule + val term = Select.overloaded(Ref(companionSym), "from", appliedTpe.typeArgs, List(typedCollection)) (term :: terms, i) } } @@ -576,19 +573,16 @@ def validateFlattenAnnotationImpl[T](using Quotes, Type[T]): Expr[Unit] = import quotes.reflect._ val fields = allFields[T] if (fields.count(_._5) > 1) { - report.errorAndAbort("Only one Map can be annotated with @upickle.implicits.flatten in the same level") + report.errorAndAbort("Only one collection can be annotated with @upickle.implicits.flatten in the same level") } if (fields.map(_._2).distinct.length != fields.length) { report.errorAndAbort("There are multiple fields with the same key") } - if (fields.exists {case (_, _, tpe, _, isFlattenMap) => isFlattenMap && !(tpe.typeArgs.head.dealias =:= TypeRepr.of[String].dealias)}) { - report.errorAndAbort("The key type of a Map annotated with @flatten must be String.") - } '{()} -private def isMap(using Quotes)(tpe: quotes.reflect.TypeRepr): Boolean = { +private def isCollectionFlattenable(using Quotes)(tpe: quotes.reflect.TypeRepr): Boolean = { import quotes.reflect._ - tpe.typeSymbol == TypeRepr.of[collection.immutable.Map[_, _]].typeSymbol + tpe <:< TypeRepr.of[Iterable[(String, _)]] } private def isCaseClass(using Quotes)(typeSymbol: quotes.reflect.Symbol): Boolean = { @@ -596,6 +590,14 @@ private def isCaseClass(using Quotes)(typeSymbol: quotes.reflect.Symbol): Boolea typeSymbol.isClassDef && typeSymbol.flags.is(Flags.Case) } +private def extractKeyValueTypes(using Quotes)(tpe: quotes.reflect.TypeRepr): (quotes.reflect.TypeRepr, quotes.reflect.TypeRepr, quotes.reflect.TypeRepr) = + import quotes.reflect._ + tpe match { + case AppliedType(tycon, keyType :: valueType :: Nil) => (tycon, keyType, valueType) + case AppliedType(tycon, AppliedType(_, keyType :: valueType :: Nil) :: Nil) => (tycon, keyType, valueType) + case _ => report.errorAndAbort(s"Fail to extract key value from $tpe") + } + @deprecated def fieldLabelsImpl[T](using Quotes, Type[T]): Expr[List[(String, String)]] = Expr.ofList(fieldLabelsImpl0[T].map((a, b) => Expr((a.name, b)))) @@ -654,4 +656,3 @@ def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Arra case t: TypeRef => apply(None) case t: TermRef => '{ ${ Ref(t.classSymbol.get.companionModule).asExprOf[Any] }.asInstanceOf[T] } } - diff --git a/upickle/implicits/src/upickle/implicits/MacrosCommon.scala b/upickle/implicits/src/upickle/implicits/MacrosCommon.scala index f30350dde..a6646fb78 100644 --- a/upickle/implicits/src/upickle/implicits/MacrosCommon.scala +++ b/upickle/implicits/src/upickle/implicits/MacrosCommon.scala @@ -1,6 +1,6 @@ package upickle.implicits -trait MacrosCommon extends upickle.core.Config with upickle.core.Types{ +trait MacrosCommon extends upickle.core.Config with upickle.core.Types { val outerThis = this } diff --git a/upickle/implicits/src/upickle/implicits/key.scala b/upickle/implicits/src/upickle/implicits/key.scala index 2be787810..c5c6ad3da 100644 --- a/upickle/implicits/src/upickle/implicits/key.scala +++ b/upickle/implicits/src/upickle/implicits/key.scala @@ -33,13 +33,13 @@ class allowUnknownKeys(b: Boolean) extends StaticAnnotation /** * An annotation that, when applied to a field in a case class, flattens the fields of the - * annotated `case class` or `Map` into the parent case class during serialization. + * annotated `case class` or `Iterable` into the parent case class during serialization. * This means the fields will appear at the same level as the parent case class's fields * rather than nested under the field name. During deserialization, these fields are - * grouped back into the annotated `case class` or `Map`. + * grouped back into the annotated `case class` or `Iterable`. * * **Limitations**: - * - Only works with `Map` types that are subtypes of `Map[String, _]`. - * - Cannot flatten more than two `Map` instances in a same level. + * - Only works with collections type that are subtypes of `Iterable[(String, _)]`. + * - Cannot flatten more than two collections in a same level. */ class flatten extends StaticAnnotation diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 9a70f0d76..0da8d22ef 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -40,6 +40,9 @@ object WrongTag { case class FlattenTwoMaps(@upickle.implicits.flatten map1: Map[String, String], @upickle.implicits.flatten map2: Map[String, String]) case class ConflictingKeys(i: Int, @upickle.implicits.flatten cm: ConflictingMessage) case class ConflictingMessage(i: Int) +object ConflictingMessage { + implicit def rw: upickle.default.ReadWriter[ConflictingMessage] = upickle.default.macroRW +} case class MapWithNoneStringKey(@upickle.implicits.flatten map: Map[ConflictingMessage, String]) object TaggedCustomSerializer{ diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 486720cc9..e0c4de4e1 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -203,28 +203,28 @@ object Flatten { implicit val rw: RW[NestedWithDefault] = upickle.default.macroRW } - case class KeyClass(id: Int, name: String) - object KeyClass { - implicit val rw: RW[KeyClass] = upickle.default.macroRW - } - case class FlattenWithKey(@upickle.implicits.flatten n: Map[KeyClass, String]) - object FlattenWithKey { - implicit val rw: RW[FlattenWithKey] = upickle.default.macroRW - } + // case class KeyClass(id: Int, name: String) + // object KeyClass { + // implicit val rw: RW[KeyClass] = upickle.default.macroRW + // } + // case class FlattenWithKey(@upickle.implicits.flatten n: Map[KeyClass, String]) + // object FlattenWithKey { + // implicit val rw: RW[FlattenWithKey] = upickle.default.macroRW + // } case class FlattenSeq(@upickle.implicits.flatten n: Seq[(String, Int)]) object FlattenSeq { implicit val rw: RW[FlattenSeq] = upickle.default.macroRW } - case class ValueClass(value: Double) - object ValueClass { - implicit val rw: RW[ValueClass] = upickle.default.macroRW - } - case class Collection(@upickle.implicits.flatten n: scala.collection.mutable.LinkedHashMap[KeyClass, ValueClass]) - object Collection { - implicit val rw: RW[Collection] = upickle.default.macroRW - } + // case class ValueClass(value: Double) + // object ValueClass { + // implicit val rw: RW[ValueClass] = upickle.default.macroRW + // } + // case class Collection(@upickle.implicits.flatten n: scala.collection.mutable.LinkedHashMap[KeyClass, ValueClass]) + // object Collection { + // implicit val rw: RW[Collection] = upickle.default.macroRW + // } } object MacroTests extends TestSuite { @@ -1001,11 +1001,11 @@ object MacroTests extends TestSuite { rw(value, """{"i":10,"l":"default"}""") } - test("flattenWithKey") { - import Flatten._ - val value = FlattenWithKey(Map(KeyClass(1, "a") -> "value1", KeyClass(2, "b") -> "value2")) - rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") - } + // test("flattenWithKey") { + // import Flatten._ + // val value = FlattenWithKey(Map(KeyClass(1, "a") -> "value1", KeyClass(2, "b") -> "value2")) + // rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") + // } test("flattenSeq") { import Flatten._ @@ -1013,10 +1013,10 @@ object MacroTests extends TestSuite { rw(value, """{"a":1,"b":2}""") } - test("flattenLinkedHashMap") { - import Flatten._ - val value = Collection(scala.collection.mutable.LinkedHashMap(KeyClass(1, "a") -> ValueClass(3.0), KeyClass(2, "b") -> ValueClass(4.0))) - rw(value, """{"{\"id\":1,\"name\":\"a\"}":{"value":3},"{\"id\":2,\"name\":\"b\"}":{"value":4}}""") - } + // test("flattenLinkedHashMap") { + // import Flatten._ + // val value = Collection(scala.collection.mutable.LinkedHashMap(KeyClass(1, "a") -> ValueClass(3.0), KeyClass(2, "b") -> ValueClass(4.0))) + // rw(value, """{"{\"id\":1,\"name\":\"a\"}":{"value":3},"{\"id\":2,\"name\":\"b\"}":{"value":4}}""") + // } } } From 30240a9f9f87e9acc956303b63963888aa2cd56a Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Fri, 20 Dec 2024 22:42:53 +0900 Subject: [PATCH 14/23] private --- .../upickle/implicits/internal/Macros2.scala | 6 +- .../src-3/upickle/implicits/macros.scala | 85 ++++++++++--------- upickle/test/src/upickle/MacroTests.scala | 2 +- 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 820fc9b50..082cb7c91 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -19,7 +19,7 @@ import language.existentials @nowarn("cat=deprecation") object Macros2 { - trait DeriveDefaults[M[_]] { + private[upickle] trait DeriveDefaults[M[_]] { val c: scala.reflect.macros.blackbox.Context private[upickle] def fail(s: String) = c.abort(c.enclosingPosition, s) @@ -296,7 +296,7 @@ object Macros2 { targetType: c.Type): Tree } - abstract class Reading[M[_]] extends DeriveDefaults[M] { + private[upickle] abstract class Reading[M[_]] extends DeriveDefaults[M] { val c: scala.reflect.macros.blackbox.Context import c.universe._ def wrapObject(t: c.Tree) = q"new ${c.prefix}.SingletonReader($t)" @@ -503,7 +503,7 @@ object Macros2 { } } - abstract class Writing[M[_]] extends DeriveDefaults[M] { + private[upickle] abstract class Writing[M[_]] extends DeriveDefaults[M] { val c: scala.reflect.macros.blackbox.Context import c.universe._ def wrapObject(obj: c.Tree) = q"new ${c.prefix}.SingletonWriter($obj)" diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index aacc97e6c..a2e6e46b2 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -4,7 +4,7 @@ import scala.quoted.{ given, _ } import deriving._, compiletime._ import upickle.implicits.{MacrosCommon, ReadersVersionSpecific} -def getDefaultParamsImpl0[T](using Quotes, Type[T]): Map[String, Expr[AnyRef]] = +private def getDefaultParamsImpl0[T](using Quotes, Type[T]): Map[String, Expr[AnyRef]] = import quotes.reflect._ val unwrapped = TypeRepr.of[T] match { case AppliedType(p, v) => p @@ -36,21 +36,21 @@ def getDefaultParamsImpl0[T](using Quotes, Type[T]): Map[String, Expr[AnyRef]] = names.zip(idents.map(_.asExpr).map(e => '{$e.asInstanceOf[AnyRef]})).toMap -def extractKey[A](using Quotes)(sym: quotes.reflect.Symbol): Option[String] = +private def extractKey[A](using Quotes)(sym: quotes.reflect.Symbol): Option[String] = import quotes.reflect._ sym .annotations .find(_.tpe =:= TypeRepr.of[upickle.implicits.key]) .map{case Apply(_, Literal(StringConstant(s)) :: Nil) => s} -def extractSerializeDefaults[A](using quotes: Quotes)(sym: quotes.reflect.Symbol): Option[Boolean] = +private def extractSerializeDefaults[A](using quotes: Quotes)(sym: quotes.reflect.Symbol): Option[Boolean] = import quotes.reflect._ sym .annotations .find(_.tpe =:= TypeRepr.of[upickle.implicits.serializeDefaults]) .map{case Apply(_, Literal(BooleanConstant(s)) :: Nil) => s} -inline def extractIgnoreUnknownKeys[T](): List[Boolean] = ${extractIgnoreUnknownKeysImpl[T]} +private[upickle] inline def extractIgnoreUnknownKeys[T](): List[Boolean] = ${extractIgnoreUnknownKeysImpl[T]} def extractIgnoreUnknownKeysImpl[T](using Quotes, Type[T]): Expr[List[Boolean]] = import quotes.reflect._ Expr.ofList( @@ -62,13 +62,13 @@ def extractIgnoreUnknownKeysImpl[T](using Quotes, Type[T]): Expr[List[Boolean]] .toList ) -def extractFlatten[A](using Quotes)(sym: quotes.reflect.Symbol): Boolean = +private def extractFlatten[A](using Quotes)(sym: quotes.reflect.Symbol): Boolean = import quotes.reflect._ sym .annotations .exists(_.tpe =:= TypeRepr.of[upickle.implicits.flatten]) -inline def paramsCount[T]: Int = ${paramsCountImpl[T]} +private[upickle] inline def paramsCount[T]: Int = ${paramsCountImpl[T]} def paramsCountImpl[T](using Quotes, Type[T]) = { import quotes.reflect._ val fields = allFields[T] @@ -76,8 +76,8 @@ def paramsCountImpl[T](using Quotes, Type[T]) = { Expr(count) } -inline def allReaders[T, R[_]]: (AnyRef, Array[AnyRef]) = ${allReadersImpl[T, R]} -def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRef, Array[AnyRef])] = { +private[upickle] inline def allReaders[T, R[_]]: (AnyRef, Array[AnyRef]) = ${allReadersImpl[T, R]} +private def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRef, Array[AnyRef])] = { import quotes.reflect._ val fields = allFields[T] val (readerMap, readers) = fields.partitionMap { case (_, _, tpe, _, isFlattenMap) => @@ -105,14 +105,14 @@ def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRef, Array ) } -inline def allFieldsMappedName[T]: List[String] = ${allFieldsMappedNameImpl[T]} -def allFieldsMappedNameImpl[T](using Quotes, Type[T]): Expr[List[String]] = { +private[upickle] inline def allFieldsMappedName[T]: List[String] = ${allFieldsMappedNameImpl[T]} +private def allFieldsMappedNameImpl[T](using Quotes, Type[T]): Expr[List[String]] = { import quotes.reflect._ Expr(allFields[T].map { case (_, label, _, _, _) => label }) } -inline def storeDefaults[T](inline x: upickle.implicits.BaseCaseObjectContext): Unit = ${storeDefaultsImpl[T]('x)} -def storeDefaultsImpl[T](x: Expr[upickle.implicits.BaseCaseObjectContext])(using Quotes, Type[T]) = { +private[upickle] inline def storeDefaults[T](inline x: upickle.implicits.BaseCaseObjectContext): Unit = ${storeDefaultsImpl[T]('x)} +private def storeDefaultsImpl[T](x: Expr[upickle.implicits.BaseCaseObjectContext])(using Quotes, Type[T]) = { import quotes.reflect.* val statements = allFields[T] .filter(!_._5) @@ -127,7 +127,7 @@ def storeDefaultsImpl[T](x: Expr[upickle.implicits.BaseCaseObjectContext])(using Expr.block(statements, '{}) } -def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String, quotes.reflect.TypeRepr, Option[Expr[Any]], Boolean)] = { +private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String, quotes.reflect.TypeRepr, Option[Expr[Any]], Boolean)] = { import quotes.reflect._ def loop(field: Symbol, label: String, classTypeRepr: TypeRepr, defaults: Map[String, Expr[Object]]): List[(Symbol, String, TypeRepr, Option[Expr[Any]], Boolean)] = { @@ -165,7 +165,7 @@ def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String, qu } } -def fieldLabelsImpl0[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String)] = +private def fieldLabelsImpl0[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, String)] = import quotes.reflect._ val fields: List[Symbol] = TypeRepr.of[T].typeSymbol .primaryConstructor @@ -180,8 +180,8 @@ def fieldLabelsImpl0[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, Str case None => (sym, sym.name) } -inline def keyToIndex[T](inline x: String): Int = ${keyToIndexImpl[T]('x)} -def keyToIndexImpl[T](x: Expr[String])(using Quotes, Type[T]): Expr[Int] = { +private[upickle] inline def keyToIndex[T](inline x: String): Int = ${keyToIndexImpl[T]('x)} +private def keyToIndexImpl[T](x: Expr[String])(using Quotes, Type[T]): Expr[Int] = { import quotes.reflect.* val fields = allFields[T].filter { case (_, _, _, _, isFlattenMap) => !isFlattenMap } val z = Match( @@ -196,11 +196,11 @@ def keyToIndexImpl[T](x: Expr[String])(using Quotes, Type[T]): Expr[Int] = { z.asExpr.asInstanceOf[Expr[Int]] } -inline def writeLength[T](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, +private[upickle] inline def writeLength[T](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, inline v: T): Int = ${writeLengthImpl[T]('thisOuter, 'v)} -def serDfltVals(using quotes: Quotes)(thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], +private def serDfltVals(using quotes: Quotes)(thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], argSym: quotes.reflect.Symbol, targetType: quotes.reflect.Symbol): Expr[Boolean] = { extractSerializeDefaults(argSym).orElse(extractSerializeDefaults(targetType)) match { @@ -209,7 +209,7 @@ def serDfltVals(using quotes: Quotes)(thisOuter: Expr[upickle.core.Types with up } } -def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], +private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], v: Expr[T]) (using quotes: Quotes, t: Type[T]): Expr[Int] = import quotes.reflect.* @@ -255,13 +255,13 @@ def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.implicits } .foldLeft('{0}) { case (prev, next) => '{$prev + $next} } -inline def writeSnippets[R, T, W[_]](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, +private[upickle] inline def writeSnippets[R, T, W[_]](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon, inline self: upickle.implicits.CaseClassReadWriters#CaseClassWriter[T], inline v: T, inline ctx: _root_.upickle.core.ObjVisitor[_, R]): Unit = ${writeSnippetsImpl[R, T, W]('thisOuter, 'self, 'v, 'ctx)} -def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], +private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon], self: Expr[upickle.implicits.CaseClassReadWriters#CaseClassWriter[T]], v: Expr[T], ctx: Expr[_root_.upickle.core.ObjVisitor[_, R]]) @@ -345,12 +345,12 @@ private def sealedHierarchyParents[T](using Quotes, Type[T]): List[quotes.reflec TypeRepr.of[T].baseClasses.filter(_.flags.is(Flags.Sealed)) -inline def isMemberOfSealedHierarchy[T]: Boolean = ${ isMemberOfSealedHierarchyImpl[T] } -def isMemberOfSealedHierarchyImpl[T](using Quotes, Type[T]): Expr[Boolean] = +private[upickle] inline def isMemberOfSealedHierarchy[T]: Boolean = ${ isMemberOfSealedHierarchyImpl[T] } +private def isMemberOfSealedHierarchyImpl[T](using Quotes, Type[T]): Expr[Boolean] = Expr(sealedHierarchyParents[T].nonEmpty) -inline def tagKey[T](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon): String = ${ tagKeyImpl[T]('thisOuter) } -def tagKeyImpl[T](using Quotes, Type[T])(thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon]): Expr[String] = +private[upickle] inline def tagKey[T](inline thisOuter: upickle.core.Types with upickle.implicits.MacrosCommon): String = ${ tagKeyImpl[T]('thisOuter) } +private def tagKeyImpl[T](using Quotes, Type[T])(thisOuter: Expr[upickle.core.Types with upickle.implicits.MacrosCommon]): Expr[String] = import quotes.reflect._ // `case object`s extend from `Mirror`, which is `sealed` and will never have a `@key` annotation @@ -367,7 +367,7 @@ def tagKeyImpl[T](using Quotes, Type[T])(thisOuter: Expr[upickle.core.Types with case None => '{${thisOuter}.tagName} } -def substituteTypeArgs(using Quotes)(tpe: quotes.reflect.TypeRepr, subsitituted: quotes.reflect.TypeRepr): quotes.reflect.TypeRepr = { +private def substituteTypeArgs(using Quotes)(tpe: quotes.reflect.TypeRepr, subsitituted: quotes.reflect.TypeRepr): quotes.reflect.TypeRepr = { import quotes.reflect._ val constructorSym = tpe.typeSymbol.primaryConstructor val constructorParamSymss = constructorSym.paramSymss @@ -376,8 +376,8 @@ def substituteTypeArgs(using Quotes)(tpe: quotes.reflect.TypeRepr, subsitituted: subsitituted.substituteTypes(tparams0 ,tpe.typeArgs) } -inline def applyConstructor[T](params: Array[Any], collection: scala.collection.mutable.ListBuffer[(String, Any)]): T = ${ applyConstructorImpl[T]('params, 'collection) } -def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]], collection: Expr[scala.collection.mutable.ListBuffer[(String, Any)]]): Expr[T] = +private[upickle] inline def applyConstructor[T](params: Array[Any], collection: scala.collection.mutable.ListBuffer[(String, Any)]): T = ${ applyConstructorImpl[T]('params, 'collection) } +private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Array[Any]], collection: Expr[scala.collection.mutable.ListBuffer[(String, Any)]]): Expr[T] = import quotes.reflect._ def apply(tpe: TypeRepr, typeArgs: List[TypeRepr], offset: Int): (Term, Int) = { val companion: Symbol = tpe.classSymbol.get.companionModule @@ -448,11 +448,11 @@ def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: Expr[Arra case t: TermRef => '{${Ref(t.classSymbol.get.companionModule).asExprOf[Any]}.asInstanceOf[T]} } -inline def tagName[T]: String = ${ tagNameImpl[T] } -def tagNameImpl[T](using Quotes, Type[T]): Expr[String] = +private[upickle] inline def tagName[T]: String = ${ tagNameImpl[T] } +private def tagNameImpl[T](using Quotes, Type[T]): Expr[String] = tagNameImpl0(identity) -def tagNameImpl0[T](transform: String => String)(using Quotes, Type[T]): Expr[String] = +private def tagNameImpl0[T](transform: String => String)(using Quotes, Type[T]): Expr[String] = import quotes.reflect._ val sym = TypeTree.of[T].symbol @@ -485,8 +485,9 @@ def tagNameImpl0[T](transform: String => String)(using Quotes, Type[T]): Expr[St transform(TypeTree.of[T].tpe.typeSymbol.fullName.filter(_ != '$')) } ) -inline def shortTagName[T]: String = ${ shortTagNameImpl[T] } -def shortTagNameImpl[T](using Quotes, Type[T]): Expr[String] = + +private[upickle] inline def shortTagName[T]: String = ${ shortTagNameImpl[T] } +private def shortTagNameImpl[T](using Quotes, Type[T]): Expr[String] = import quotes.reflect._ val sealedClassSymbol = if (TypeRepr.of[T].baseClasses.contains(TypeRepr.of[T].typeSymbol)) Some(TypeRepr.of[T].typeSymbol.fullName.split('.')) @@ -504,13 +505,13 @@ def shortTagNameImpl[T](using Quotes, Type[T]): Expr[String] = tagNameImpl0(_.split('.').drop(identicalSegmentCount).mkString(".")) -inline def isSingleton[T]: Boolean = ${ isSingletonImpl[T] } -def isSingletonImpl[T](using Quotes, Type[T]): Expr[Boolean] = +private[upickle] inline def isSingleton[T]: Boolean = ${ isSingletonImpl[T] } +private def isSingletonImpl[T](using Quotes, Type[T]): Expr[Boolean] = import quotes.reflect._ Expr(TypeRepr.of[T].typeSymbol.flags.is(Flags.Module) || TypeRepr.of[T].isSingleton) -inline def getSingleton[T]: T = ${ getSingletonImpl[T] } -def getSingletonImpl[T](using Quotes, Type[T]): Expr[T] = +private[upickle] inline def getSingleton[T]: T = ${ getSingletonImpl[T] } +private def getSingletonImpl[T](using Quotes, Type[T]): Expr[T] = import quotes.reflect._ TypeRepr.of[T] match{ @@ -519,9 +520,9 @@ def getSingletonImpl[T](using Quotes, Type[T]): Expr[T] = } -inline def defineEnumReaders[T0, T <: Tuple](prefix: Any): T0 = ${ defineEnumVisitorsImpl[T0, T]('prefix, "macroR") } -inline def defineEnumWriters[T0, T <: Tuple](prefix: Any): T0 = ${ defineEnumVisitorsImpl[T0, T]('prefix, "macroW") } -def defineEnumVisitorsImpl[T0, T <: Tuple](prefix: Expr[Any], macroX: String)(using Quotes, Type[T0], Type[T]): Expr[T0] = +private[upickle] inline def defineEnumReaders[T0, T <: Tuple](prefix: Any): T0 = ${ defineEnumVisitorsImpl[T0, T]('prefix, "macroR") } +private[upickle] inline def defineEnumWriters[T0, T <: Tuple](prefix: Any): T0 = ${ defineEnumVisitorsImpl[T0, T]('prefix, "macroW") } +private def defineEnumVisitorsImpl[T0, T <: Tuple](prefix: Expr[Any], macroX: String)(using Quotes, Type[T0], Type[T]): Expr[T0] = import quotes.reflect._ def handleType(tpe: TypeRepr, name: String, skipTrait: Boolean): Option[(ValDef, Symbol)] = { @@ -568,8 +569,8 @@ def defineEnumVisitorsImpl[T0, T <: Tuple](prefix: Expr[Any], macroX: String)(us Block(allDefs.map(_._1), Ident(allDefs.head._2.termRef)).asExprOf[T0] -inline def validateFlattenAnnotation[T](): Unit = ${ validateFlattenAnnotationImpl[T] } -def validateFlattenAnnotationImpl[T](using Quotes, Type[T]): Expr[Unit] = +private[upickle] inline def validateFlattenAnnotation[T](): Unit = ${ validateFlattenAnnotationImpl[T] } +private def validateFlattenAnnotationImpl[T](using Quotes, Type[T]): Expr[Unit] = import quotes.reflect._ val fields = allFields[T] if (fields.count(_._5) > 1) { diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index e0c4de4e1..50edf0ca6 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -167,7 +167,7 @@ object Flatten { case class FlattenTestWithType[T](i: Int, @upickle.implicits.flatten t: T) object FlattenTestWithType { - // implicit def rw[T: RW]: RW[FlattenTestWithType[T]] = upickle.default.macroRW + // implicit def rw[T: RW]: RW[FlattenTestWithType[T]] = upickle.default.macroRW implicit val rw: RW[FlattenTestWithType[Nested]] = upickle.default.macroRW } From d3fb86c6d00d558481d674e442dcee55201791c2 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Sat, 28 Dec 2024 15:36:08 +0900 Subject: [PATCH 15/23] polish --- .../upickle/implicits/internal/Macros2.scala | 8 ++-- upickle/test/src/upickle/MacroTests.scala | 42 ++++++------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 082cb7c91..72a5b8e27 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -322,7 +322,6 @@ object Macros2 { val (localReaders, aggregates) = mappedNames.zipWithIndex.map { case (_, idx) => (TermName(s"localReader$idx"), TermName(s"aggregated$idx")) }.unzip - val readKey = if (keyTypeOfCollection =:= typeOf[String]) q"currentKey" else q"read(currentKey)(implicitly[${c.prefix}.Reader[$keyTypeOfCollection]])" def constructClass(constructedTpe: c.Type): c.universe.Tree = { def loop(tpe: c.Type, offset: Int): (c.universe.Tree, Int) = { @@ -427,7 +426,7 @@ object Macros2 { } case ..${ if (hasFlattenOnCollection) - List(cq"-1 => aggregatedCollection += $readKey -> v.asInstanceOf[$valueTypeOfCollection]") + List(cq"-1 => aggregatedCollection += currentKey -> v.asInstanceOf[$valueTypeOfCollection]") else Nil } case _ => throw new java.lang.IndexOutOfBoundsException(currentIndex.toString) @@ -532,13 +531,12 @@ object Macros2 { symbol.annotations.find(_.tree.tpe =:= typeOf[flatten]) match { case Some(_) => if (isCollectionFlattenable(tpeOfField)) { - val (_, keyType, valueType) = extractKeyValueTypes(tpeOfField) - val writeKey = if (keyType =:= typeOf[String]) q"key" else q"upickle.default.write(key)(implicitly[${c.prefix}.Writer[$keyType]])" + val (_, _, valueType) = extractKeyValueTypes(tpeOfField) q""" $select.foreach { case (key, value) => this.writeSnippetMappedName[R, $valueType]( ctx, - $writeKey, + key, implicitly[${c.prefix}.Writer[$valueType]], value ) diff --git a/upickle/test/src/upickle/MacroTests.scala b/upickle/test/src/upickle/MacroTests.scala index 50edf0ca6..80060fac1 100644 --- a/upickle/test/src/upickle/MacroTests.scala +++ b/upickle/test/src/upickle/MacroTests.scala @@ -167,7 +167,6 @@ object Flatten { case class FlattenTestWithType[T](i: Int, @upickle.implicits.flatten t: T) object FlattenTestWithType { - // implicit def rw[T: RW]: RW[FlattenTestWithType[T]] = upickle.default.macroRW implicit val rw: RW[FlattenTestWithType[Nested]] = upickle.default.macroRW } @@ -203,28 +202,19 @@ object Flatten { implicit val rw: RW[NestedWithDefault] = upickle.default.macroRW } - // case class KeyClass(id: Int, name: String) - // object KeyClass { - // implicit val rw: RW[KeyClass] = upickle.default.macroRW - // } - // case class FlattenWithKey(@upickle.implicits.flatten n: Map[KeyClass, String]) - // object FlattenWithKey { - // implicit val rw: RW[FlattenWithKey] = upickle.default.macroRW - // } - case class FlattenSeq(@upickle.implicits.flatten n: Seq[(String, Int)]) object FlattenSeq { implicit val rw: RW[FlattenSeq] = upickle.default.macroRW } - // case class ValueClass(value: Double) - // object ValueClass { - // implicit val rw: RW[ValueClass] = upickle.default.macroRW - // } - // case class Collection(@upickle.implicits.flatten n: scala.collection.mutable.LinkedHashMap[KeyClass, ValueClass]) - // object Collection { - // implicit val rw: RW[Collection] = upickle.default.macroRW - // } + case class ValueClass(value: Double) + object ValueClass { + implicit val rw: RW[ValueClass] = upickle.default.macroRW + } + case class Collection(@upickle.implicits.flatten n: scala.collection.mutable.LinkedHashMap[String, ValueClass]) + object Collection { + implicit val rw: RW[Collection] = upickle.default.macroRW + } } object MacroTests extends TestSuite { @@ -1001,22 +991,16 @@ object MacroTests extends TestSuite { rw(value, """{"i":10,"l":"default"}""") } - // test("flattenWithKey") { - // import Flatten._ - // val value = FlattenWithKey(Map(KeyClass(1, "a") -> "value1", KeyClass(2, "b") -> "value2")) - // rw(value, """{"{\"id\":1,\"name\":\"a\"}":"value1","{\"id\":2,\"name\":\"b\"}":"value2"}""") - // } - test("flattenSeq") { import Flatten._ val value = FlattenSeq(Seq("a" -> 1, "b" -> 2)) rw(value, """{"a":1,"b":2}""") } - // test("flattenLinkedHashMap") { - // import Flatten._ - // val value = Collection(scala.collection.mutable.LinkedHashMap(KeyClass(1, "a") -> ValueClass(3.0), KeyClass(2, "b") -> ValueClass(4.0))) - // rw(value, """{"{\"id\":1,\"name\":\"a\"}":{"value":3},"{\"id\":2,\"name\":\"b\"}":{"value":4}}""") - // } + test("flattenLinkedHashMap") { + import Flatten._ + val value = Collection(scala.collection.mutable.LinkedHashMap("a" -> ValueClass(3.0), "b" -> ValueClass(4.0))) + rw(value, """{"a":{"value":3},"b":{"value":4}}""") + } } } From 2eb1ab8c0e22cbbb90e02533ee4d3883605c6016 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Sat, 28 Dec 2024 16:50:28 +0900 Subject: [PATCH 16/23] More tests --- .../upickle/implicits/internal/Macros2.scala | 8 +++-- .../src-3/upickle/implicits/macros.scala | 9 +++-- upickle/test/src/upickle/FailureTests.scala | 35 ++++++++++++++----- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 72a5b8e27..5c0d84c10 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -209,8 +209,12 @@ object Macros2 { if (fields.count { case(_, _, _, _, _, isCollection) => isCollection } > 1) { fail("Only one collection can be annotated with @upickle.implicits.flatten in the same level") } - if (fields.map { case (_, mappedName, _, _, _, _) => mappedName }.distinct.length != fields.length) { - fail("There are multiple fields with the same key") + val duplicatedKeys = fields.map { case (_, mappedName, _, _, _, _) => mappedName }.groupBy(identity).collect { case (x, List(_, _, _*)) => x } + if (duplicatedKeys.nonEmpty) { + fail( + s"""There are multiple fields with the same key. + |Following keys are duplicated: ${duplicatedKeys.mkString(", ")}. + |""".stripMargin) } } diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index a2e6e46b2..2a97a2cdd 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -576,8 +576,13 @@ private def validateFlattenAnnotationImpl[T](using Quotes, Type[T]): Expr[Unit] if (fields.count(_._5) > 1) { report.errorAndAbort("Only one collection can be annotated with @upickle.implicits.flatten in the same level") } - if (fields.map(_._2).distinct.length != fields.length) { - report.errorAndAbort("There are multiple fields with the same key") + val duplicatedKeys = fields.map { case (_, mappedName, _, _, _) => mappedName }.groupBy(identity).collect { case (x, List(_, _, _*)) => x } + if (duplicatedKeys.nonEmpty) { + report.errorAndAbort( + s"""There are multiple fields with the same key. + |Following keys are duplicated: ${duplicatedKeys.mkString(", ")}. + |""".stripMargin + ) } '{()} diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 0da8d22ef..44b7cd2b5 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -37,13 +37,25 @@ object WrongTag { } -case class FlattenTwoMaps(@upickle.implicits.flatten map1: Map[String, String], @upickle.implicits.flatten map2: Map[String, String]) -case class ConflictingKeys(i: Int, @upickle.implicits.flatten cm: ConflictingMessage) -case class ConflictingMessage(i: Int) -object ConflictingMessage { - implicit def rw: upickle.default.ReadWriter[ConflictingMessage] = upickle.default.macroRW +object FlattenTest { + case class UnsupportedType(@upickle.implicits.flatten i: Int) + case class UnsupportedCollectionType(@upickle.implicits.flatten x: Seq[(Seq[Int], Int)]) + + case class FlattenTwoMaps(@upickle.implicits.flatten map1: Map[String, String], @upickle.implicits.flatten map2: Map[String, String]) + + case class ConflictingKeys(i: Int, @upickle.implicits.flatten cm: ConflictingMessage) + + case class ConflictingMessage(i: Int) + + object ConflictingMessage { + implicit def rw: upickle.default.ReadWriter[ConflictingMessage] = upickle.default.macroRW + } + + case class MapWithNoneStringKey(@upickle.implicits.flatten map: Map[ConflictingMessage, String]) + case class FlattenTwoCaseClasses(@upickle.implicits.flatten m1: Message1, @upickle.implicits.flatten m2: Message2) + case class Message1(x: Int) + case class Message2(x: String) } -case class MapWithNoneStringKey(@upickle.implicits.flatten map: Map[ConflictingMessage, String]) object TaggedCustomSerializer{ @@ -273,9 +285,14 @@ object FailureTests extends TestSuite { // compileError("""read[Array[Object]]("")""").msg // Make sure this doesn't hang the compiler =/ compileError("implicitly[upickle.default.Reader[Nothing]]") - compileError("upickle.default.macroRW[FlattenTwoMaps]") - compileError("upickle.default.macroRW[ConflictingKeys]") - compileError("upickle.default.macroRW[MapWithNoneStringKey]") + } + test("flattenAnnotation") { + compileError("upickle.default.macroRW[FlattenTest.FlattenTwoMaps]") + compileError("upickle.default.macroRW[FlattenTest.ConflictingKeys]") + compileError("upickle.default.macroRW[FlattenTest.MapWithNoneStringKey]") + compileError("upickle.default.macroRW[FlattenTest.UnsupportedType]") + compileError("upickle.default.macroRW[FlattenTest.UnsupportedCollectionType]") + compileError("upickle.default.macroRW[FlattenTest.FlattenTwoCaseClasses]") } test("expWholeNumbers"){ upickle.default.read[Byte]("0e0") ==> 0.toByte From e4071f6b9a1967d39dababacf417480cce05b6f5 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Sat, 28 Dec 2024 20:10:54 +0900 Subject: [PATCH 17/23] Throw an error when a key collision occurs while writing --- .../upickle/implicits/internal/Macros2.scala | 18 ++++++++++++------ .../src-3/upickle/implicits/macros.scala | 17 ++++++++++++----- upickle/test/src/upickle/FailureTests.scala | 13 +++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala index 5c0d84c10..997821446 100644 --- a/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala +++ b/upickle/implicits/src-2/upickle/implicits/internal/Macros2.scala @@ -537,13 +537,17 @@ object Macros2 { if (isCollectionFlattenable(tpeOfField)) { val (_, _, valueType) = extractKeyValueTypes(tpeOfField) q""" + val collisions = allKeys.intersect($select.map(_._1).toSet) + if (collisions.nonEmpty) { + throw new Exception("Key collision detected for the following keys: " + collisions.mkString(", ")) + } $select.foreach { case (key, value) => - this.writeSnippetMappedName[R, $valueType]( - ctx, - key, - implicitly[${c.prefix}.Writer[$valueType]], - value - ) + this.writeSnippetMappedName[R, $valueType]( + ctx, + key, + implicitly[${c.prefix}.Writer[$valueType]], + value + ) } """ :: Nil } else if (tpeOfField.typeSymbol.isClass && tpeOfField.typeSymbol.asClass.isCaseClass) { @@ -591,6 +595,8 @@ object Macros2 { q""" new ${c.prefix}.CaseClassWriter[$targetType]{ + private lazy val allKeys = Set[String](..${mappedNames.toList.map(name => Literal(Constant(name)))}) + def length(v: $targetType) = { ${ getLength(targetType, q"v") diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 2a97a2cdd..5563d590a 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -275,12 +275,19 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit val typeSymbol = fieldTypeRepr.typeSymbol if (flatten) { if (isCollectionFlattenable(fieldTypeRepr)) { - val (_, keyTpe0, valueTpe0) = extractKeyValueTypes(fieldTypeRepr) + val (_, _, valueTpe0) = extractKeyValueTypes(fieldTypeRepr) + val allKeysExpr: Expr[Set[String]] = classTypeRepr.asType match { + case '[t] => Expr(allFields[t].map(_._2).toSet) + } val writerTpe0 = TypeRepr.of[W].appliedTo(valueTpe0) - (keyTpe0.asType, valueTpe0.asType, writerTpe0.asType) match { - case ('[keyTpe], '[valueTpe], '[writerTpe])=> + (valueTpe0.asType, writerTpe0.asType) match { + case ('[valueTpe], '[writerTpe])=> val snippet = '{ - ${select.asExprOf[Iterable[(keyTpe, valueTpe)]]}.foreach { (k, v) => + val collisions = ${select.asExprOf[Iterable[(String, valueTpe)]]}.map(_._1).toSet.intersect(${allKeysExpr}) + if (collisions.nonEmpty) { + throw new Exception("Key collision detected for the following keys: " + collisions.mkString(", ")) + } + ${select.asExprOf[Iterable[(String, valueTpe)]]}.foreach { (k, v) => ${self}.writeSnippetMappedName[R, valueTpe]( ${ctx}, k.toString, @@ -557,7 +564,7 @@ private def defineEnumVisitorsImpl[T0, T <: Tuple](prefix: Expr[Any], macroX: St def getDefs(t: TypeRepr, defs: List[(ValDef, Symbol)]): List[(ValDef, Symbol)] = { t match{ - case AppliedType(prefix, args) => + case AppliedType(_, args) => val defAndSymbol = handleType(args(0), "x" + defs.size, skipTrait = true) getDefs(args(1), defAndSymbol.toList ::: defs) case _ if t =:= TypeRepr.of[EmptyTuple] => defs diff --git a/upickle/test/src/upickle/FailureTests.scala b/upickle/test/src/upickle/FailureTests.scala index 44b7cd2b5..9eeab65cc 100644 --- a/upickle/test/src/upickle/FailureTests.scala +++ b/upickle/test/src/upickle/FailureTests.scala @@ -55,6 +55,10 @@ object FlattenTest { case class FlattenTwoCaseClasses(@upickle.implicits.flatten m1: Message1, @upickle.implicits.flatten m2: Message2) case class Message1(x: Int) case class Message2(x: String) + case class RuntimeCollision(x: Int, @upickle.implicits.flatten m: Seq[(String, Int)]) + object RuntimeCollision { + implicit def rw: upickle.default.ReadWriter[RuntimeCollision] = upickle.default.macroRW + } } object TaggedCustomSerializer{ @@ -294,6 +298,15 @@ object FailureTests extends TestSuite { compileError("upickle.default.macroRW[FlattenTest.UnsupportedCollectionType]") compileError("upickle.default.macroRW[FlattenTest.FlattenTwoCaseClasses]") } + test("runtimeCollision") { + import upickle.default._ + + val error = intercept[Exception] { + upickle.default.write(FlattenTest.RuntimeCollision(1, Seq("x" -> 3))) + } + + assert(error.getMessage.startsWith("Key collision")) + } test("expWholeNumbers"){ upickle.default.read[Byte]("0e0") ==> 0.toByte upickle.default.read[Short]("0e0") ==> 0 From 379aab37746dad5782f54411bdccd96296687f9e Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 21:54:31 +0900 Subject: [PATCH 18/23] Fix error messages --- .../implicits/src-3/upickle/implicits/macros.scala | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 5563d590a..c8ea40cbf 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -147,10 +147,8 @@ private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, St val newClassTypeRepr = TypeRepr.of[T] loop(rawLabel, label, newClassTypeRepr, newDefaults) } - case _ => - report.errorAndAbort(s"Unsupported type $typeSymbol for flattening") } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") } else { (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil @@ -233,10 +231,8 @@ private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.i val newClassTypeRepr = TypeRepr.of[T] loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } - case _ => - report.errorAndAbort("Unsupported type for flattening") } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } else report.errorAndAbort(s"Invalid type for flattening ${typeSymbol}") } else if (!defaults.contains(label)) List('{1}) else { @@ -309,8 +305,6 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit val newClassTypeRepr = TypeRepr.of[T] loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } - case _ => - report.errorAndAbort("Unsupported type for flattening", v) } } else report.errorAndAbort(s"${typeSymbol} is not a case class or a subtype of Iterable", v.asTerm.pos) } @@ -420,10 +414,8 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E case t: TermRef => (Ref(t.classSymbol.get.companionModule), i) } (term :: terms, nextOffset) - case _ => - report.errorAndAbort(s"Unsupported type $typeSymbol for flattening") } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a immutable.Map") + } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") } else { val lhs = '{$params(${ Expr(i) })} From fc02f238ac4210858a3db2bb35c780bd442dc3bb Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 21:57:10 +0900 Subject: [PATCH 19/23] address reviews --- .../implicits/src-3/upickle/implicits/macros.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index c8ea40cbf..3839b59a4 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -82,7 +82,7 @@ private def allReadersImpl[T, R[_]](using Quotes, Type[T], Type[R]): Expr[(AnyRe val fields = allFields[T] val (readerMap, readers) = fields.partitionMap { case (_, _, tpe, _, isFlattenMap) => if (isFlattenMap) { - val (_, _, valueTpe) = extractKeyValueTypes(tpe) + val (_, valueTpe) = extractKeyValueTypes(tpe) val readerTpe = TypeRepr.of[R].appliedTo(valueTpe) val reader = readerTpe.asType match { case '[t] => '{summonInline[t].asInstanceOf[AnyRef]} @@ -271,7 +271,7 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit val typeSymbol = fieldTypeRepr.typeSymbol if (flatten) { if (isCollectionFlattenable(fieldTypeRepr)) { - val (_, _, valueTpe0) = extractKeyValueTypes(fieldTypeRepr) + val (_, valueTpe0) = extractKeyValueTypes(fieldTypeRepr) val allKeysExpr: Expr[Set[String]] = classTypeRepr.asType match { case '[t] => Expr(allFields[t].map(_._2).toSet) } @@ -395,7 +395,7 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E val flatten = extractFlatten(sym0) if (flatten) { if (isCollectionFlattenable(appliedTpe)) { - val (_, keyTpe0, valueTpe0) = extractKeyValueTypes(appliedTpe) + val (keyTpe0, valueTpe0) = extractKeyValueTypes(appliedTpe) (keyTpe0.asType, valueTpe0.asType) match { case ('[keyTpe], '[valueTpe]) => val typedCollection = '{${collection}.asInstanceOf[scala.collection.mutable.ListBuffer[(keyTpe, valueTpe)]]}.asTerm @@ -595,11 +595,12 @@ private def isCaseClass(using Quotes)(typeSymbol: quotes.reflect.Symbol): Boolea typeSymbol.isClassDef && typeSymbol.flags.is(Flags.Case) } -private def extractKeyValueTypes(using Quotes)(tpe: quotes.reflect.TypeRepr): (quotes.reflect.TypeRepr, quotes.reflect.TypeRepr, quotes.reflect.TypeRepr) = +// extract key value types from collections like Map[K, V] or Iterable[(K, V)] +private def extractKeyValueTypes(using Quotes)(tpe: quotes.reflect.TypeRepr): (quotes.reflect.TypeRepr, quotes.reflect.TypeRepr) = import quotes.reflect._ tpe match { - case AppliedType(tycon, keyType :: valueType :: Nil) => (tycon, keyType, valueType) - case AppliedType(tycon, AppliedType(_, keyType :: valueType :: Nil) :: Nil) => (tycon, keyType, valueType) + case AppliedType(_, keyType :: valueType :: Nil) => (keyType, valueType) + case AppliedType(_, AppliedType(_, keyType :: valueType :: Nil) :: Nil) => (keyType, valueType) case _ => report.errorAndAbort(s"Fail to extract key value from $tpe") } From 64a504803151219fae8b54972ffb89987fb80f28 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 21:59:18 +0900 Subject: [PATCH 20/23] do not abort --- upickle/implicits/src-3/upickle/implicits/macros.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 3839b59a4..2655e1fef 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -148,7 +148,7 @@ private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, St loop(rawLabel, label, newClassTypeRepr, newDefaults) } } - } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") + } else report.error(s"Invalid type for flattening: ${typeSymbol}"); Nil } else { (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil @@ -232,7 +232,7 @@ private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.i loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"Invalid type for flattening ${typeSymbol}") + } else report.error(s"Invalid type for flattening ${typeSymbol}"); Nil } else if (!defaults.contains(label)) List('{1}) else { @@ -415,7 +415,7 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E } (term :: terms, nextOffset) } - } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") + } else report.error(s"Invalid type for flattening: ${typeSymbol}"); (terms, i) } else { val lhs = '{$params(${ Expr(i) })} From 20760f959f8d5efc8845859f74d5c73abf50e305 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 22:17:26 +0900 Subject: [PATCH 21/23] fix --- upickle/implicits/src-3/upickle/implicits/macros.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 2655e1fef..52fd70ec5 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -148,7 +148,7 @@ private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, St loop(rawLabel, label, newClassTypeRepr, newDefaults) } } - } else report.error(s"Invalid type for flattening: ${typeSymbol}"); Nil + } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") } else { (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil @@ -232,7 +232,7 @@ private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.i loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.error(s"Invalid type for flattening ${typeSymbol}"); Nil + } else report.errorAndAbort(s"Invalid type for flattening ${typeSymbol}") } else if (!defaults.contains(label)) List('{1}) else { @@ -306,7 +306,7 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a subtype of Iterable", v.asTerm.pos) + } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}", v.asTerm.pos) } else { val tpe0 = fieldTypeRepr @@ -415,7 +415,7 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E } (term :: terms, nextOffset) } - } else report.error(s"Invalid type for flattening: ${typeSymbol}"); (terms, i) + } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") } else { val lhs = '{$params(${ Expr(i) })} From 2b40f83bfc3716a525cc9cc74de2613981ee1f0b Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 22:19:51 +0900 Subject: [PATCH 22/23] refine error messages --- upickle/implicits/src-3/upickle/implicits/macros.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 52fd70ec5..285c3100f 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -148,7 +148,7 @@ private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, St loop(rawLabel, label, newClassTypeRepr, newDefaults) } } - } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") } else { (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil @@ -232,7 +232,7 @@ private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.i loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"Invalid type for flattening ${typeSymbol}") + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") } else if (!defaults.contains(label)) List('{1}) else { @@ -306,7 +306,7 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}", v.asTerm.pos) + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]", v.asTerm.pos) } else { val tpe0 = fieldTypeRepr @@ -415,7 +415,7 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E } (term :: terms, nextOffset) } - } else report.errorAndAbort(s"Invalid type for flattening: ${typeSymbol}") + } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") } else { val lhs = '{$params(${ Expr(i) })} From 966022dd648b246dab61eceb23d4656938a315f6 Mon Sep 17 00:00:00 2001 From: Yoonjae Jeon Date: Mon, 6 Jan 2025 22:46:22 +0900 Subject: [PATCH 23/23] Do not abort --- .../src-3/upickle/implicits/macros.scala | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/upickle/implicits/src-3/upickle/implicits/macros.scala b/upickle/implicits/src-3/upickle/implicits/macros.scala index 285c3100f..801294720 100644 --- a/upickle/implicits/src-3/upickle/implicits/macros.scala +++ b/upickle/implicits/src-3/upickle/implicits/macros.scala @@ -148,7 +148,10 @@ private def allFields[T](using Quotes, Type[T]): List[(quotes.reflect.Symbol, St loop(rawLabel, label, newClassTypeRepr, newDefaults) } } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + } else { + report.error(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + Nil + } } else { (field, label, substitutedTypeRepr, defaults.get(label), false) :: Nil @@ -232,7 +235,10 @@ private def writeLengthImpl[T](thisOuter: Expr[upickle.core.Types with upickle.i loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + } else { + report.error(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + Nil + } } else if (!defaults.contains(label)) List('{1}) else { @@ -306,7 +312,10 @@ private def writeSnippetsImpl[R, T, W[_]](thisOuter: Expr[upickle.core.Types wit loop(rawLabel, label, newClassTypeRepr, newSelect, newDefaults) } } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]", v.asTerm.pos) + } else { + report.error(s"${typeSymbol} is not a case class or a Iterable[(String, _)]", v.asTerm.pos) + Nil + } } else { val tpe0 = fieldTypeRepr @@ -415,7 +424,10 @@ private def applyConstructorImpl[T](using quotes: Quotes, t0: Type[T])(params: E } (term :: terms, nextOffset) } - } else report.errorAndAbort(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + } else { + report.error(s"${typeSymbol} is not a case class or a Iterable[(String, _)]") + (terms, i + 1) + } } else { val lhs = '{$params(${ Expr(i) })}