From 61f857cf3f0006ed5eff4fafb8d90e45ec7bb6d9 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Mon, 27 Nov 2023 14:19:48 +0100 Subject: [PATCH 1/7] Add source file attribute to TASTY files This introduces the first non-boolean attribute to the TASTY format. We Split the tags into a set of boolean tags and string tags to handle This new kind of attribute. We keep some tags as unknown to allow for additional kinds of tags in the future. --- .../dotc/core/tasty/AttributePickler.scala | 10 +++- .../dotc/core/tasty/AttributeUnpickler.scala | 12 ++++- .../tools/dotc/core/tasty/Attributes.scala | 12 ++++- .../tools/dotc/core/tasty/TastyPrinter.scala | 4 ++ .../dotty/tools/dotc/transform/Pickler.scala | 4 ++ tasty/src/dotty/tools/tasty/TastyFormat.scala | 54 +++++++++++-------- 6 files changed, 69 insertions(+), 27 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala index d7c268a6b4da..3fae2f1e85c9 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala @@ -12,12 +12,18 @@ object AttributePickler: pickler: TastyPickler, buf: TastyBuffer ): Unit = - if attributes.booleanTags.nonEmpty then - pickler.newSection(AttributesSection, buf) + pickler.newSection(AttributesSection, buf) for tag <- attributes.booleanTags do + assert(tag < TastyFormat.firstStringAttrTag, "Not a boolean attribute tag: " + tag) buf.writeByte(tag) + assert(attributes.stringTagValues.exists(_._1 == TastyFormat.SOURCEFILEattr)) + for (tag, value) <- attributes.stringTagValues do + assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag) + buf.writeByte(tag) + buf.writeUtf8(value) + end pickleAttributes end AttributePickler diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala index 485635dceeb3..eebfc24af3d8 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala @@ -3,6 +3,7 @@ package core.tasty import scala.language.unsafeNulls import scala.collection.immutable.BitSet +import scala.collection.immutable.TreeMap import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer} @@ -11,11 +12,18 @@ class AttributeUnpickler(reader: TastyReader): lazy val attributes: Attributes = { val booleanTags = BitSet.newBuilder + val stringTagValue = List.newBuilder[(Int, String)] while !isAtEnd do - booleanTags += readByte() + val tag = readByte() + if tag < TastyFormat.firstStringAttrTag then + booleanTags += tag + else if tag < TastyFormat.firstUnassignedAttrTag then + stringTagValue += tag -> readUtf8() + else + assert(false, "unknown attribute tag: " + tag) - new Attributes(booleanTags.result()) + new Attributes(booleanTags.result(), stringTagValue.result()) } end AttributeUnpickler diff --git a/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala b/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala index 0697f39f6dab..9e7c62ea9b5d 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala @@ -3,9 +3,11 @@ package dotty.tools.dotc.core.tasty import dotty.tools.tasty.TastyFormat.* import scala.collection.immutable.BitSet +import scala.collection.immutable.TreeMap class Attributes private[tasty]( private[tasty] val booleanTags: BitSet, + private[tasty] val stringTagValues: List[(Int, String)], ) { def scala2StandardLibrary: Boolean = booleanTags(SCALA2STANDARDLIBRARYattr) def explicitNulls: Boolean = booleanTags(EXPLICITNULLSattr) @@ -13,10 +15,12 @@ class Attributes private[tasty]( def withPureFuns: Boolean = booleanTags(WITHPUREFUNSattr) def isJava: Boolean = booleanTags(JAVAattr) def isOutline: Boolean = booleanTags(OUTLINEattr) + def sourceFile: Option[String] = stringTagValues.find(_._1 == SOURCEFILEattr).map(_._2) } object Attributes: def apply( + sourceFile: String, scala2StandardLibrary: Boolean, explicitNulls: Boolean, captureChecked: Boolean, @@ -31,8 +35,12 @@ object Attributes: if withPureFuns then booleanTags += WITHPUREFUNSattr if isJava then booleanTags += JAVAattr if isOutline then booleanTags += OUTLINEattr - new Attributes(booleanTags.result()) + + val stringTagValues = List.newBuilder[(Int, String)] + stringTagValues += SOURCEFILEattr -> sourceFile + + new Attributes(booleanTags.result(), stringTagValues.result()) end apply val empty: Attributes = - new Attributes(BitSet.empty) + new Attributes(BitSet.empty, Nil) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index b2094bd8a94a..e9e5b1b9347c 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -233,6 +233,10 @@ class TastyPrinter(bytes: Array[Byte]) { for tag <- attributes.booleanTags do sb.append(" ").append(attributeTagToString(tag)).append("\n") + for (tag, value) <- attributes.stringTagValues do + sb.append(" ").append(attributeTagToString(tag)) + .append(" ").append(value).append("\n") + sb.result } } diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 2dc1599365cd..1b5c7214e1e0 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -96,12 +96,16 @@ class Pickler extends Phase { do if ctx.settings.YtestPickler.value then beforePickling(cls) = tree.show + val sourceRelativePath = + val reference = ctx.settings.sourceroot.value + util.SourceFile.relativePath(unit.source, reference) val isJavaAttr = unit.isJava // we must always set JAVAattr when pickling Java sources if isJavaAttr then // assert that Java sources didn't reach Pickler without `-Yjava-tasty`. assert(ctx.settings.YjavaTasty.value, "unexpected Java source file without -Yjava-tasty") val isOutline = isJavaAttr // TODO: later we may want outline for Scala sources too val attributes = Attributes( + sourceFile = sourceRelativePath, scala2StandardLibrary = ctx.settings.YcompileScala2Library.value, explicitNulls = ctx.settings.YexplicitNulls.value, captureChecked = Feature.ccEnabled, diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index 6ceb82f011f4..193539006621 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -236,11 +236,11 @@ Note: The signature of a SELECTin or TERMREFin node is the signature of the sele Note: Tree tags are grouped into 5 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. ```none - Category 1 (tags 1-59) : tag - Category 2 (tags 60-89) : tag Nat - Category 3 (tags 90-109) : tag AST - Category 4 (tags 110-127): tag Nat AST - Category 5 (tags 128-255): tag Length + Tree Category 1 (tags 1-59) : tag + Tree Category 2 (tags 60-89) : tag Nat + Tree Category 3 (tags 90-109) : tag AST + Tree Category 4 (tags 110-127): tag Nat AST + Tree Category 5 (tags 128-255): tag Length ``` Standard-Section: "Positions" LinesSizes Assoc* @@ -278,6 +278,13 @@ Standard Section: "Attributes" Attribute* WITHPUREFUNSattr JAVAattr OUTLINEattr + SOURCEFILEattr Utf8 +``` + +Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. +```none + Attribute Category 1 (tags 1-64) : tag + Attribute Category 2 (tags 65-128): tag UTF8 ``` **************************************************************************************/ @@ -441,9 +448,9 @@ object TastyFormat { final val SOURCE = 4 - // AST tags - // Cat. 1: tag + // AST tags + // Tree Cat. 1: tag final val firstSimpleTreeTag = UNITconst // final val ??? = 1 final val UNITconst = 2 @@ -492,8 +499,8 @@ object TastyFormat { final val EMPTYCLAUSE = 45 final val SPLITCLAUSE = 46 - // Cat. 2: tag Nat - + // Tree Cat. 2: tag Nat + final val firstNatTreeTag = SHAREDterm final val SHAREDterm = 60 final val SHAREDtype = 61 final val TERMREFdirect = 62 @@ -512,8 +519,8 @@ object TastyFormat { final val IMPORTED = 75 final val RENAMED = 76 - // Cat. 3: tag AST - + // Tree Cat. 3: tag AST + final val firstASTTreeTag = THIS final val THIS = 90 final val QUALTHIS = 91 final val CLASSconst = 92 @@ -531,8 +538,8 @@ object TastyFormat { final val ELIDED = 104 - // Cat. 4: tag Nat AST - + // Tree Cat. 4: tag Nat AST + final val firstNatASTTreeTag = IDENT final val IDENT = 110 final val IDENTtpt = 111 final val SELECT = 112 @@ -544,8 +551,8 @@ object TastyFormat { final val SELFDEF = 118 final val NAMEDARG = 119 - // Cat. 5: tag Length ... - + // Tree Cat. 5: tag Length ... + final val firstLengthTreeTag = PACKAGE final val PACKAGE = 128 final val VALDEF = 129 final val DEFDEF = 130 @@ -607,14 +614,10 @@ object TastyFormat { final val HOLE = 255 - final val firstNatTreeTag = SHAREDterm - final val firstASTTreeTag = THIS - final val firstNatASTTreeTag = IDENT - final val firstLengthTreeTag = PACKAGE - - // Attributes tags + // Attr Cat. 1: tag + final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr final val SCALA2STANDARDLIBRARYattr = 1 final val EXPLICITNULLSattr = 2 final val CAPTURECHECKEDattr = 3 @@ -622,6 +625,14 @@ object TastyFormat { final val JAVAattr = 5 final val OUTLINEattr = 6 + // Attr Cat. 2: tag UTF8 + final val firstStringAttrTag = SOURCEFILEattr + final val SOURCEFILEattr = 128 + + // Attr Cat. 3: unassigned + final val firstUnassignedAttrTag = 129 + + /** Useful for debugging */ def isLegalTag(tag: Int): Boolean = firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE || @@ -845,6 +856,7 @@ object TastyFormat { case WITHPUREFUNSattr => "WITHPUREFUNSattr" case JAVAattr => "JAVAattr" case OUTLINEattr => "OUTLINEattr" + case SOURCEFILEattr => "SOURCEFILEattr" } /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry. From a1dc803180616109897f1f33310a0bf150aa6e6e Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 28 Nov 2023 09:44:06 +0100 Subject: [PATCH 2/7] Use name table for string attributes --- .../tools/dotc/core/tasty/AttributePickler.scala | 3 ++- .../tools/dotc/core/tasty/AttributeUnpickler.scala | 7 +++++-- .../tools/dotc/core/tasty/DottyUnpickler.scala | 2 +- .../dotty/tools/dotc/core/tasty/NameBuffer.scala | 4 ++++ .../dotty/tools/dotc/core/tasty/TastyPrinter.scala | 14 ++++++++------ tasty/src/dotty/tools/tasty/TastyFormat.scala | 5 +++-- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala index 3fae2f1e85c9..6874c3fd51f2 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala @@ -21,8 +21,9 @@ object AttributePickler: assert(attributes.stringTagValues.exists(_._1 == TastyFormat.SOURCEFILEattr)) for (tag, value) <- attributes.stringTagValues do assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag) + val utf8Ref = pickler.nameBuffer.utf8Index(value) buf.writeByte(tag) - buf.writeUtf8(value) + buf.writeNat(utf8Ref.index) end pickleAttributes diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala index eebfc24af3d8..f8779cf9703c 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala @@ -6,8 +6,9 @@ import scala.collection.immutable.BitSet import scala.collection.immutable.TreeMap import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer} +import dotty.tools.dotc.core.tasty.TastyUnpickler.NameTable -class AttributeUnpickler(reader: TastyReader): +class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable): import reader._ lazy val attributes: Attributes = { @@ -19,7 +20,9 @@ class AttributeUnpickler(reader: TastyReader): if tag < TastyFormat.firstStringAttrTag then booleanTags += tag else if tag < TastyFormat.firstUnassignedAttrTag then - stringTagValue += tag -> readUtf8() + val utf8Ref = readNameRef() + val value = nameAtRef(utf8Ref).toString + stringTagValue += tag -> value else assert(false, "unknown attribute tag: " + tag) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index 8753017cc82f..21592a6ef7e6 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -39,7 +39,7 @@ object DottyUnpickler { class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler = - new AttributeUnpickler(reader) + new AttributeUnpickler(reader, nameAtRef) } } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/NameBuffer.scala b/compiler/src/dotty/tools/dotc/core/tasty/NameBuffer.scala index 5e2aee33859c..076c37435478 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/NameBuffer.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/NameBuffer.scala @@ -49,6 +49,10 @@ class NameBuffer extends TastyBuffer(10000) { } } + def utf8Index(value: String): NameRef = + import Decorators.toTermName + nameIndex(value.toTermName) + private inline def withLength(inline op: Unit, lengthWidth: Int = 1): Unit = { val lengthAddr = currentAddr var i = 0 diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index e9e5b1b9347c..db24cccff432 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -228,14 +228,16 @@ class TastyPrinter(bytes: Array[Byte]) { import dotty.tools.tasty.TastyFormat.* def unpickle(reader: TastyReader, tastyName: NameTable): Unit = { import reader.* - val attributes = new AttributeUnpickler(reader).attributes sb.append(s"\n\nAttributes (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n") - - for tag <- attributes.booleanTags do - sb.append(" ").append(attributeTagToString(tag)).append("\n") - for (tag, value) <- attributes.stringTagValues do + while !isAtEnd do + val tag = readByte() sb.append(" ").append(attributeTagToString(tag)) - .append(" ").append(value).append("\n") + if tag < firstStringAttrTag then () + else if tag < firstUnassignedAttrTag then + val utf8Ref = readNameRef() + val value = nameAtRef(utf8Ref).toString + sb.append(nameStr(s" ${utf8Ref.index} [$value]")) + sb.append("\n") sb.result } } diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index 193539006621..bc4178f22c96 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -50,6 +50,7 @@ Macro-format: // If positive, this is a NameRef for the fully qualified name of a term parameter. NameRef = Nat // ordinal number of name in name table, starting from 1. + Utf8Ref = Nat // ordinal number of UTF8 in name table, starting from 1. ``` Note: Unqualified names in the name table are strings. The context decides whether a name is @@ -278,13 +279,13 @@ Standard Section: "Attributes" Attribute* WITHPUREFUNSattr JAVAattr OUTLINEattr - SOURCEFILEattr Utf8 + SOURCEFILEattr Utf8Ref ``` Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. ```none Attribute Category 1 (tags 1-64) : tag - Attribute Category 2 (tags 65-128): tag UTF8 + Attribute Category 2 (tags 65-128): tag Utf8Ref ``` **************************************************************************************/ From 135e97ff4373cc29516a016de3674a4b1087cbbe Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Mon, 27 Nov 2023 14:28:58 +0100 Subject: [PATCH 3/7] Use source file from TASTy attribute when available --- compiler/src/dotty/tools/dotc/core/Symbols.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/core/Symbols.scala b/compiler/src/dotty/tools/dotc/core/Symbols.scala index 9a206261c744..ddddaf9b07fb 100644 --- a/compiler/src/dotty/tools/dotc/core/Symbols.scala +++ b/compiler/src/dotty/tools/dotc/core/Symbols.scala @@ -486,6 +486,12 @@ object Symbols extends SymUtils { mySource = ctx.getSource(file) else mySource = defn.patchSource(this) + if !mySource.exists then + val compUnitInfo = compilationUnitInfo + if compUnitInfo != null then + compUnitInfo.tastyInfo.flatMap(_.attributes.sourceFile) match + case Some(path) => mySource = ctx.getSource(path) + case _ => if !mySource.exists then mySource = atPhaseNoLater(flattenPhase) { denot.topLevelClass.unforcedAnnotation(defn.SourceFileAnnot) match From 64f1d11d5a6539711aa26c5629e072076c61d55d Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 30 Nov 2023 09:14:56 +0100 Subject: [PATCH 4/7] Fix missing CompilationUnit on class symbol loaded in TreeUnpickler Also refactor accesses to the tasty attributes to get them directly from the compilation unit info instead of the unpickler. --- .../dotty/tools/dotc/core/SymbolLoaders.scala | 17 ++--------- .../dotc/core/tasty/DottyUnpickler.scala | 28 ++++++++++++------- .../tools/dotc/core/tasty/TreeUnpickler.scala | 26 ++++++++--------- .../tools/dotc/fromtasty/ReadTasty.scala | 1 + .../tools/dotc/quoted/PickledQuotes.scala | 3 +- .../dotty/tools/dotc/transform/Pickler.scala | 2 +- .../dotc/core/tasty/CommentPicklingTest.scala | 3 +- 7 files changed, 40 insertions(+), 40 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index 6319956e2a81..75c610b29140 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -420,20 +420,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader { private val unpickler: tasty.DottyUnpickler = handleUnpicklingExceptions: val tastyBytes = tastyFile.toByteArray - new tasty.DottyUnpickler(tastyBytes) // reads header and name table - - val compilationUnitInfo: CompilationUnitInfo | Null = - val tastyHeader = unpickler.unpickler.header - val tastyVersion = TastyVersion( - tastyHeader.majorVersion, - tastyHeader.minorVersion, - tastyHeader.experimentalVersion, - ) - val attributes = unpickler.tastyAttributes - new CompilationUnitInfo( - tastyFile, - tastyInfo = Some(TastyInfo(tastyVersion, attributes)), - ) + new tasty.DottyUnpickler(tastyFile, tastyBytes) // reads header and name table + + val compilationUnitInfo: CompilationUnitInfo | Null = unpickler.compilationUnitInfo def description(using Context): String = "TASTy file " + tastyFile.toString diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index 21592a6ef7e6..4f083b09b015 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -16,15 +16,17 @@ import dotty.tools.tasty.TastyReader import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection} import dotty.tools.tasty.TastyVersion +import dotty.tools.io.AbstractFile + object DottyUnpickler { /** Exception thrown if classfile is corrupted */ class BadSignature(msg: String) extends RuntimeException(msg) - class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], attributeUnpickler: Option[AttributeUnpickler]) + class TreeSectionUnpickler(compilationUnitInfo: CompilationUnitInfo, posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler]) extends SectionUnpickler[TreeUnpickler](ASTsSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler = - new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, attributeUnpickler) + new TreeUnpickler(reader, nameAtRef, compilationUnitInfo, posUnpickler, commentUnpickler) } class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) { @@ -44,21 +46,28 @@ object DottyUnpickler { } /** A class for unpickling Tasty trees and symbols. + * @param tastyFile tasty file from which we unpickle (used for CompilationUnitInfo) * @param bytes the bytearray containing the Tasty file from which we unpickle * @param mode the tasty file contains package (TopLevel), an expression (Term) or a type (TypeTree) */ -class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLevel) extends ClassfileParser.Embedded with tpd.TreeProvider { +class DottyUnpickler(tastyFile: AbstractFile, bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLevel) extends ClassfileParser.Embedded with tpd.TreeProvider { import tpd.* import DottyUnpickler.* val unpickler: TastyUnpickler = new TastyUnpickler(bytes) + + val tastyAttributes: Attributes = + unpickler.unpickle(new AttributesSectionUnpickler) + .map(_.attributes).getOrElse(Attributes.empty) + val compilationUnitInfo: CompilationUnitInfo = + import unpickler.header.{majorVersion, minorVersion, experimentalVersion} + val tastyVersion = TastyVersion(majorVersion, minorVersion, experimentalVersion) + val tastyInfo = TastyInfo(tastyVersion, tastyAttributes) + new CompilationUnitInfo(tastyFile, Some(tastyInfo)) + private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler) private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler) - private val attributeUnpicklerOpt = unpickler.unpickle(new AttributesSectionUnpickler) - private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get - - def tastyAttributes: Attributes = - attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes.empty) + private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get /** Enter all toplevel classes and objects into their scopes * @param roots a set of SymDenotations that should be overwritten by unpickling @@ -69,9 +78,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe protected def treeSectionUnpickler( posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler], - attributeUnpicklerOpt: Option[AttributeUnpickler] ): TreeSectionUnpickler = - new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt) + new TreeSectionUnpickler(compilationUnitInfo, posUnpicklerOpt, commentUnpicklerOpt) protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index d4271d5bffaf..f68c1afcb5d5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -49,15 +49,16 @@ import scala.compiletime.uninitialized /** Unpickler for typed trees * @param reader the reader from which to unpickle + * @param compilationUnitInfo the compilation unit info of the TASTy * @param posUnpicklerOpt the unpickler for positions, if it exists * @param commentUnpicklerOpt the unpickler for comments, if it exists * @param attributeUnpicklerOpt the unpickler for attributes, if it exists */ class TreeUnpickler(reader: TastyReader, nameAtRef: NameTable, + compilationUnitInfo: CompilationUnitInfo, posUnpicklerOpt: Option[PositionUnpickler], - commentUnpicklerOpt: Option[CommentUnpickler], - attributeUnpicklerOpt: Option[AttributeUnpickler]) { + commentUnpicklerOpt: Option[CommentUnpickler]) { import TreeUnpickler.* import tpd.* @@ -92,22 +93,21 @@ class TreeUnpickler(reader: TastyReader, /** The root owner tree. See `OwnerTree` class definition. Set by `enterTopLevel`. */ private var ownerTree: OwnerTree = uninitialized + /** TASTy attributes */ + private val attributes: Attributes = compilationUnitInfo.tastyInfo.get.attributes + /** Was unpickled class compiled with capture checks? */ - private val withCaptureChecks: Boolean = - attributeUnpicklerOpt.exists(_.attributes.captureChecked) + private val withCaptureChecks: Boolean = attributes.captureChecked - private val unpicklingScala2Library = - attributeUnpicklerOpt.exists(_.attributes.scala2StandardLibrary) + private val unpicklingScala2Library = attributes.scala2StandardLibrary /** This dependency was compiled with explicit nulls enabled */ // TODO Use this to tag the symbols of this dependency as compiled with explicit nulls (see use of unpicklingScala2Library). - private val explicitNulls = - attributeUnpicklerOpt.exists(_.attributes.explicitNulls) + private val explicitNulls = attributes.explicitNulls - private val unpicklingJava = - attributeUnpicklerOpt.exists(_.attributes.isJava) + private val unpicklingJava = attributes.isJava - private val isOutline = attributeUnpicklerOpt.exists(_.attributes.isOutline) + private val isOutline = attributes.isOutline private def registerSym(addr: Addr, sym: Symbol) = symAtAddr(addr) = sym @@ -636,8 +636,8 @@ class TreeUnpickler(reader: TastyReader, rootd.symbol case _ => val completer = adjustIfModule(new Completer(subReader(start, end))) - if (isClass) - newClassSymbol(ctx.owner, name.asTypeName, flags, completer, privateWithin, coord) + if isClass then + newClassSymbol(ctx.owner, name.asTypeName, flags, completer, privateWithin, coord, compilationUnitInfo) else newSymbol(ctx.owner, name, flags, completer, privateWithin, coord) } diff --git a/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala b/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala index 9cf2421c3bda..74010b3f64d1 100644 --- a/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala +++ b/compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala @@ -11,6 +11,7 @@ import Denotations.staticRef import NameOps.* import ast.Trees.Tree import Phases.Phase +import core.tasty.Attributes /** Load trees from TASTY files */ class ReadTasty extends Phase { diff --git a/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala b/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala index 6a030c424f08..8ebd1f6973f2 100644 --- a/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala +++ b/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala @@ -21,6 +21,7 @@ import scala.quoted.runtime.impl.* import scala.collection.mutable import QuoteUtils.* +import dotty.tools.io.NoAbstractFile object PickledQuotes { import tpd.* @@ -268,7 +269,7 @@ object PickledQuotes { quotePickling.println(s"**** unpickling quote from TASTY\n${TastyPrinter.showContents(bytes, ctx.settings.color.value == "never")}") val mode = if (isType) UnpickleMode.TypeTree else UnpickleMode.Term - val unpickler = new DottyUnpickler(bytes, mode) + val unpickler = new DottyUnpickler(NoAbstractFile, bytes, mode) unpickler.enter(Set.empty) val tree = unpickler.tree diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 1b5c7214e1e0..1eaf0a58fda2 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -235,7 +235,7 @@ class Pickler extends Phase { ctx.initialize() val unpicklers = for ((cls, (unit, bytes)) <- pickledBytes) yield { - val unpickler = new DottyUnpickler(bytes) + val unpickler = new DottyUnpickler(unit.source.file, bytes) unpickler.enter(roots = Set.empty) cls -> (unit, unpickler) } diff --git a/compiler/test/dotty/tools/dotc/core/tasty/CommentPicklingTest.scala b/compiler/test/dotty/tools/dotc/core/tasty/CommentPicklingTest.scala index 45b807f73079..1fc374811a0e 100644 --- a/compiler/test/dotty/tools/dotc/core/tasty/CommentPicklingTest.scala +++ b/compiler/test/dotty/tools/dotc/core/tasty/CommentPicklingTest.scala @@ -18,6 +18,7 @@ import dotty.tools.vulpix.TestConfiguration import org.junit.Test import org.junit.Assert.{assertEquals, assertFalse, fail} +import dotty.tools.io.AbstractFile class CommentPicklingTest { @@ -116,7 +117,7 @@ class CommentPicklingTest { implicit val ctx: Context = setup(args, initCtx).map(_._2).getOrElse(initCtx) ctx.initialize() val trees = files.flatMap { f => - val unpickler = new DottyUnpickler(f.toByteArray()) + val unpickler = new DottyUnpickler(AbstractFile.getFile(f.jpath), f.toByteArray()) unpickler.enter(roots = Set.empty) unpickler.rootTrees(using ctx) } From 0ea9e526a1b13780ead5b4f3f9b775d3501f0077 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 30 Nov 2023 10:48:39 +0100 Subject: [PATCH 5/7] Prepare for `@SourceFile` removal The only missing step is in the TODO added in this commit. --- compiler/src/dotty/tools/dotc/core/Annotations.scala | 3 --- compiler/src/dotty/tools/dotc/transform/PostTyper.scala | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Annotations.scala b/compiler/src/dotty/tools/dotc/core/Annotations.scala index 43c753458f6e..ac02baa429b4 100644 --- a/compiler/src/dotty/tools/dotc/core/Annotations.scala +++ b/compiler/src/dotty/tools/dotc/core/Annotations.scala @@ -244,9 +244,6 @@ object Annotations { } else None } - - def makeSourceFile(path: String, span: Span)(using Context): Annotation = - apply(defn.SourceFileAnnot, Literal(Constant(path)), span) } @sharable val EmptyAnnotation = Annotation(EmptyTree) diff --git a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala index 35db363cd864..3b6c3f9bff96 100644 --- a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala +++ b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala @@ -417,12 +417,13 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase => if illegalRefs.nonEmpty then report.error( em"The type of a class parent cannot refer to constructor parameters, but ${parent.tpe} refers to ${illegalRefs.map(_.name.show).mkString(",")}", parent.srcPos) - // Add SourceFile annotation to top-level classes if sym.owner.is(Package) then + // Add SourceFile annotation to top-level classes + // TODO remove this annotation once the reference compiler uses the TASTy source file attribute. if ctx.compilationUnit.source.exists && sym != defn.SourceFileAnnot then val reference = ctx.settings.sourceroot.value val relativePath = util.SourceFile.relativePath(ctx.compilationUnit.source, reference) - sym.addAnnotation(Annotation.makeSourceFile(relativePath, tree.span)) + sym.addAnnotation(Annotation(defn.SourceFileAnnot, Literal(Constants.Constant(relativePath)), tree.span)) else if !sym.is(Param) && !sym.owner.isOneOf(AbstractOrTrait) then Checking.checkGoodBounds(tree.symbol) From e777aeddd1aa0b68e557044207acae8497afcd90 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 30 Nov 2023 13:52:54 +0100 Subject: [PATCH 6/7] Shrink reserved TASTy attribute categories --- .../dotc/core/tasty/AttributePickler.scala | 8 +++--- .../dotc/core/tasty/AttributeUnpickler.scala | 6 ++--- .../tools/dotc/core/tasty/TastyPrinter.scala | 4 +-- tasty/src/dotty/tools/tasty/TastyFormat.scala | 26 ++++++++++++------- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala index 6874c3fd51f2..5aea55bce4de 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala @@ -3,7 +3,7 @@ package dotty.tools.dotc.core.tasty import dotty.tools.dotc.ast.{tpd, untpd} import dotty.tools.tasty.TastyBuffer -import dotty.tools.tasty.TastyFormat, TastyFormat.AttributesSection +import dotty.tools.tasty.TastyFormat.* object AttributePickler: @@ -15,12 +15,12 @@ object AttributePickler: pickler.newSection(AttributesSection, buf) for tag <- attributes.booleanTags do - assert(tag < TastyFormat.firstStringAttrTag, "Not a boolean attribute tag: " + tag) + assert(isBooleanAttrTag(tag), "Not a boolean attribute tag: " + tag) buf.writeByte(tag) - assert(attributes.stringTagValues.exists(_._1 == TastyFormat.SOURCEFILEattr)) + assert(attributes.stringTagValues.exists(_._1 == SOURCEFILEattr)) for (tag, value) <- attributes.stringTagValues do - assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag) + assert(isStringAttrTag(tag), "Not a string attribute tag: " + tag) val utf8Ref = pickler.nameBuffer.utf8Index(value) buf.writeByte(tag) buf.writeNat(utf8Ref.index) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala index f8779cf9703c..405f110dbca1 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala @@ -5,7 +5,7 @@ import scala.language.unsafeNulls import scala.collection.immutable.BitSet import scala.collection.immutable.TreeMap -import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer} +import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}, TastyFormat.{isBooleanAttrTag, isStringAttrTag} import dotty.tools.dotc.core.tasty.TastyUnpickler.NameTable class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable): @@ -17,9 +17,9 @@ class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable): while !isAtEnd do val tag = readByte() - if tag < TastyFormat.firstStringAttrTag then + if isBooleanAttrTag(tag) then booleanTags += tag - else if tag < TastyFormat.firstUnassignedAttrTag then + else if isStringAttrTag(tag) then val utf8Ref = readNameRef() val value = nameAtRef(utf8Ref).toString stringTagValue += tag -> value diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index db24cccff432..a74607dbc9d5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -232,8 +232,8 @@ class TastyPrinter(bytes: Array[Byte]) { while !isAtEnd do val tag = readByte() sb.append(" ").append(attributeTagToString(tag)) - if tag < firstStringAttrTag then () - else if tag < firstUnassignedAttrTag then + if isBooleanAttrTag(tag) then () + else if isStringAttrTag(tag) then val utf8Ref = readNameRef() val value = nameAtRef(utf8Ref).toString sb.append(nameStr(s" ${utf8Ref.index} [$value]")) diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index bc4178f22c96..e1a8cf1d50ae 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -282,10 +282,13 @@ Standard Section: "Attributes" Attribute* SOURCEFILEattr Utf8Ref ``` -Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. +Note: Attribute tags are grouped into categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. + Unassigned categories can be used to extend and existing category or to add new kinds of attributes ```none - Attribute Category 1 (tags 1-64) : tag - Attribute Category 2 (tags 65-128): tag Utf8Ref + Attribute Category 1 (tags 1-32) : tag + Attribute Category 2 (tags 33-128): // not assigned yet + Attribute Category 3 (tags 129-160): tag Utf8Ref + Attribute Category 4 (tags 161-255): // not assigned yet ``` **************************************************************************************/ @@ -617,8 +620,8 @@ object TastyFormat { // Attributes tags - // Attr Cat. 1: tag - final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr + // Attribute Category 1 (tags 1-32) : tag + def isBooleanAttrTag(tag: Int): Boolean = 1 <= tag && tag <= 32 final val SCALA2STANDARDLIBRARYattr = 1 final val EXPLICITNULLSattr = 2 final val CAPTURECHECKEDattr = 3 @@ -626,12 +629,15 @@ object TastyFormat { final val JAVAattr = 5 final val OUTLINEattr = 6 - // Attr Cat. 2: tag UTF8 - final val firstStringAttrTag = SOURCEFILEattr - final val SOURCEFILEattr = 128 + // Attribute Category 2 (tags 33-128): unassigned - // Attr Cat. 3: unassigned - final val firstUnassignedAttrTag = 129 + // Attribute Category 3 (tags 129-160): tag Utf8Ref + def isStringAttrTag(tag: Int): Boolean = 129 <= tag && tag <= 160 + final val SOURCEFILEattr = 129 + + // Attribute Category 4 (tags 161-255): unassigned + + // end of Attributes tags /** Useful for debugging */ From e942cd0bd8e22577b7946fbfa25a59193182d588 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 1 Dec 2023 15:53:18 +0100 Subject: [PATCH 7/7] Ensure no duplicated and ordered TASTy attributes. --- .../dotty/tools/dotc/core/tasty/AttributePickler.scala | 8 ++++++++ .../dotty/tools/dotc/core/tasty/AttributeUnpickler.scala | 6 ++++++ tasty/src/dotty/tools/tasty/TastyFormat.scala | 1 + 3 files changed, 15 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala index 5aea55bce4de..cd2756f4c752 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala @@ -14,13 +14,21 @@ object AttributePickler: ): Unit = pickler.newSection(AttributesSection, buf) + var lastTag = -1 + def assertTagOrder(tag: Int): Unit = + assert(tag != lastTag, s"duplicate attribute tag: $tag") + assert(tag > lastTag, s"attribute tags are not ordered: $tag after $lastTag") + lastTag = tag + for tag <- attributes.booleanTags do assert(isBooleanAttrTag(tag), "Not a boolean attribute tag: " + tag) + assertTagOrder(tag) buf.writeByte(tag) assert(attributes.stringTagValues.exists(_._1 == SOURCEFILEattr)) for (tag, value) <- attributes.stringTagValues do assert(isStringAttrTag(tag), "Not a string attribute tag: " + tag) + assertTagOrder(tag) val utf8Ref = pickler.nameBuffer.utf8Index(value) buf.writeByte(tag) buf.writeNat(utf8Ref.index) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala index 405f110dbca1..43a2bea27216 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala @@ -15,6 +15,7 @@ class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable): val booleanTags = BitSet.newBuilder val stringTagValue = List.newBuilder[(Int, String)] + var lastTag = -1 while !isAtEnd do val tag = readByte() if isBooleanAttrTag(tag) then @@ -26,6 +27,11 @@ class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable): else assert(false, "unknown attribute tag: " + tag) + assert(tag != lastTag, s"duplicate attribute tag: $tag") + assert(tag > lastTag, s"attribute tags are not ordered: $tag after $lastTag") + lastTag = tag + end while + new Attributes(booleanTags.result(), stringTagValue.result()) } diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index e1a8cf1d50ae..adadd3267aac 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -281,6 +281,7 @@ Standard Section: "Attributes" Attribute* OUTLINEattr SOURCEFILEattr Utf8Ref ``` +Attribute tags cannot be repeated in an attribute section. Attributes are ordered by the tag ordinal. Note: Attribute tags are grouped into categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. Unassigned categories can be used to extend and existing category or to add new kinds of attributes