Skip to content

Commit

Permalink
Add attributes section to TASTy and use it for Stdlib TASTy (#18599)
Browse files Browse the repository at this point in the history
The `Attributes` section contains a list `Attribute` tags.

At this point, the only Scala 2 TASTy is the one from the standard
library. To know that a TASTy contains the definitions of the standard
library, we add the `SCALA2STANDARDLIBRARYattr` attribute to the TASTy
file. We mark all unpickled classes from those TASTy files with `Scala2x
| Scala2Tasty`.

Attributes will also be used to mark files that were compiled with
explicit nulls using the `EXPLICITNULLSattr` attribute.
  • Loading branch information
nicolasstucki authored Nov 6, 2023
2 parents 8c64fed + 16966fd commit 319e865
Show file tree
Hide file tree
Showing 12 changed files with 152 additions and 17 deletions.
4 changes: 2 additions & 2 deletions compiler/src/dotty/tools/dotc/core/Flags.scala
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,8 @@ object Flags {
*/
val (_, StableRealizable @ _, _) = newFlags(24, "<stable>")

/** A case parameter accessor */
val (_, CaseAccessor @ _, _) = newFlags(25, "<caseaccessor>")
/** A case parameter accessor / an unpickled Scala 2 TASTy (only for Scala 2 stdlib) */
val (_, CaseAccessor @ _, Scala2Tasty @ _) = newFlags(25, "<caseaccessor>", "<scala-2-tasty>")

/** A Scala 2x super accessor / an unpickled Scala 2.x class */
val (SuperParamAliasOrScala2x @ _, SuperParamAlias @ _, Scala2x @ _) = newFlags(26, "<super-param-alias>", "<scala-2.x>")
Expand Down
26 changes: 26 additions & 0 deletions compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package dotty.tools.dotc.core.tasty

import dotty.tools.dotc.ast.{tpd, untpd}

import dotty.tools.tasty.TastyBuffer
import dotty.tools.tasty.TastyFormat, TastyFormat.AttributesSection

import java.nio.charset.StandardCharsets

object AttributePickler:

def pickleAttributes(
attributes: Attributes,
pickler: TastyPickler,
buf: TastyBuffer
): Unit =
if attributes.scala2StandardLibrary || attributes.explicitNulls then // or any other attribute is set
pickler.newSection(AttributesSection, buf)
// Pickle attributes
if attributes.scala2StandardLibrary then buf.writeNat(TastyFormat.SCALA2STANDARDLIBRARYattr)
if attributes.explicitNulls then buf.writeNat(TastyFormat.EXPLICITNULLSattr)
end if

end pickleAttributes

end AttributePickler
33 changes: 33 additions & 0 deletions compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package dotty.tools.dotc
package core.tasty

import scala.language.unsafeNulls

import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}

import java.nio.charset.StandardCharsets

class AttributeUnpickler(reader: TastyReader):
import reader._

lazy val attributeTags: List[Int] =
val listBuilder = List.newBuilder[Int]
while !isAtEnd do listBuilder += readNat()
listBuilder.result()

lazy val attributes: Attributes = {
var scala2StandardLibrary = false
var explicitNulls = false
for attributeTag <- attributeTags do
attributeTag match
case TastyFormat.SCALA2STANDARDLIBRARYattr => scala2StandardLibrary = true
case TastyFormat.EXPLICITNULLSattr => explicitNulls = true
case attribute =>
assert(false, "Unexpected attribute value: " + attribute)
Attributes(
scala2StandardLibrary,
explicitNulls,
)
}

end AttributeUnpickler
6 changes: 6 additions & 0 deletions compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package dotty.tools.dotc.core.tasty

class Attributes(
val scala2StandardLibrary: Boolean,
val explicitNulls: Boolean,
)
21 changes: 15 additions & 6 deletions compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,17 @@ import Names.SimpleName
import TreeUnpickler.UnpickleMode

import dotty.tools.tasty.TastyReader
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection}
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}

object DottyUnpickler {

/** Exception thrown if classfile is corrupted */
class BadSignature(msg: String) extends RuntimeException(msg)

class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler])
class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], attributeUnpickler: Option[AttributeUnpickler])
extends SectionUnpickler[TreeUnpickler](ASTsSection) {
def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler =
new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler)
new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, attributeUnpickler)
}

class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) {
Expand All @@ -35,6 +35,10 @@ object DottyUnpickler {
def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
new CommentUnpickler(reader)
}
class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
new AttributeUnpickler(reader)
}
}

/** A class for unpickling Tasty trees and symbols.
Expand All @@ -48,16 +52,21 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
val unpickler: TastyUnpickler = new TastyUnpickler(bytes)
private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler)
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get
private val attributeUnpicklerOpt = unpickler.unpickle(new AttributesSectionUnpickler)
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get

/** Enter all toplevel classes and objects into their scopes
* @param roots a set of SymDenotations that should be overwritten by unpickling
*/
def enter(roots: Set[SymDenotation])(using Context): Unit =
treeUnpickler.enter(roots)

protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler =
new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)
protected def treeSectionUnpickler(
posUnpicklerOpt: Option[PositionUnpickler],
commentUnpicklerOpt: Option[CommentUnpickler],
attributeUnpicklerOpt: Option[AttributeUnpickler]
): TreeSectionUnpickler =
new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)

protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode)

Expand Down
2 changes: 2 additions & 0 deletions compiler/src/dotty/tools/dotc/core/tasty/ScratchData.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ class ScratchData:
val pickledIndices = new mutable.BitSet

val commentBuffer = new TastyBuffer(5000)
val attributeBuffer = new TastyBuffer(32)

def reset() =
assert(delta ne delta1)
assert(delta.length == delta1.length)
positionBuffer.reset()
pickledIndices.clear()
commentBuffer.reset()
attributeBuffer.reset()

26 changes: 21 additions & 5 deletions compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import Contexts.*, Decorators.*
import Names.Name
import TastyUnpickler.*
import util.Spans.offsetToInt
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection}
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}
import java.nio.file.{Files, Paths}
import dotty.tools.io.{JarArchive, Path}

Expand Down Expand Up @@ -84,14 +84,16 @@ class TastyPrinter(bytes: Array[Byte]) {
case Some(s) => sb.append(s)
case _ =>
}
sb.append("\n\n")
unpickle(new PositionSectionUnpickler) match {
case Some(s) => sb.append(s)
case Some(s) => sb.append("\n\n").append(s)
case _ =>
}
sb.append("\n\n")
unpickle(new CommentSectionUnpickler) match {
case Some(s) => sb.append(s)
case Some(s) => sb.append("\n\n").append(s)
case _ =>
}
unpickle(new AttributesSectionUnpickler) match {
case Some(s) => sb.append("\n\n").append(s)
case _ =>
}
sb.result
Expand Down Expand Up @@ -222,6 +224,20 @@ class TastyPrinter(bytes: Array[Byte]) {
}
}

class AttributesSectionUnpickler extends SectionUnpickler[String](AttributesSection) {
import dotty.tools.tasty.TastyFormat.attributeTagToString
private val sb: StringBuilder = new StringBuilder

def unpickle(reader: TastyReader, tastyName: NameTable): String = {
sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}")
val attributeTags = new AttributeUnpickler(reader).attributeTags
sb.append(s" attributes bytes:\n")
for attributeTag <- attributeTags do
sb.append(" ").append(attributeTagToString(attributeTag)).append("\n")
sb.result
}
}

protected def nameStr(str: String): String = str
protected def treeStr(str: String): String = str
protected def lengthStr(str: String): String = str
Expand Down
15 changes: 13 additions & 2 deletions compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ import scala.compiletime.uninitialized
* @param reader the reader from which to unpickle
* @param posUnpicklerOpt the unpickler for positions, if it exists
* @param commentUnpicklerOpt the unpickler for comments, if it exists
* @param attributeUnpicklerOpt the unpickler for attributes, if it exists
*/
class TreeUnpickler(reader: TastyReader,
nameAtRef: NameTable,
posUnpicklerOpt: Option[PositionUnpickler],
commentUnpicklerOpt: Option[CommentUnpickler]) {
commentUnpicklerOpt: Option[CommentUnpickler],
attributeUnpicklerOpt: Option[AttributeUnpickler]) {
import TreeUnpickler.*
import tpd.*

Expand Down Expand Up @@ -97,6 +99,14 @@ class TreeUnpickler(reader: TastyReader,
/** Was unpickled class compiled with capture checks? */
private var withCaptureChecks: Boolean = false

private val unpicklingScala2Library =
attributeUnpicklerOpt.exists(_.attributes.scala2StandardLibrary)

/** This dependency was compiled with explicit nulls enabled */
// TODO Use this to tag the symbols of this dependency as compiled with explicit nulls (see use of unpicklingScala2Library).
private val explicitNulls =
attributeUnpicklerOpt.exists(_.attributes.explicitNulls)

private def registerSym(addr: Addr, sym: Symbol) =
symAtAddr(addr) = sym

Expand Down Expand Up @@ -601,7 +611,8 @@ class TreeUnpickler(reader: TastyReader,
val rhsStart = currentAddr
val rhsIsEmpty = nothingButMods(end)
if (!rhsIsEmpty) skipTree()
val (givenFlags, annotFns, privateWithin) = readModifiers(end)
val (givenFlags0, annotFns, privateWithin) = readModifiers(end)
val givenFlags = if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty else givenFlags0
pickling.println(i"creating symbol $name at $start with flags ${givenFlags.flagsString}, isAbsType = $isAbsType, $ttag")
val flags = normalizeFlags(tag, givenFlags, name, isAbsType, rhsIsEmpty)
def adjustIfModule(completer: LazyType) =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ class ExtensionMethods extends MiniPhase with DenotTransformer with FullParamete

// Create extension methods, except if the class comes from Scala 2
// because it adds extension methods before pickling.
if (!(valueClass.is(Scala2x)))
if !valueClass.is(Scala2x, butNot = Scala2Tasty) then
for (decl <- valueClass.classInfo.decls)
if isMethodWithExtension(decl) then
enterInModuleClass(createExtensionMethod(decl, moduleClassSym.symbol))
Expand Down
6 changes: 6 additions & 0 deletions compiler/src/dotty/tools/dotc/transform/Pickler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,12 @@ class Pickler extends Phase {
pickler, treePkl.buf.addrOfTree, treePkl.docString, tree,
scratch.commentBuffer)

val attributes = Attributes(
scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
explicitNulls = ctx.settings.YexplicitNulls.value,
)
AttributePickler.pickleAttributes(attributes, pickler, scratch.attributeBuffer)

val pickled = pickler.assembleParts()

def rawBytes = // not needed right now, but useful to print raw format.
Expand Down
7 changes: 7 additions & 0 deletions scala2-library-tasty-tests/src/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ object HelloWorld:
testScala2ObjectParents()
testScala2CaseClassUnderscoreMembers()
testScalaNumberUnderlying()
testArrayOps()
scala.collection.mutable.UnrolledBufferTest.test()
}

Expand Down Expand Up @@ -68,3 +69,9 @@ object HelloWorld:
val _: Object = MyNumber2(BigInt(1)).underlying
val _: Object = (MyNumber2(BigInt(1)): ScalaNumber).underlying
}

def testArrayOps() = {
new collection.ArrayOps[String](Array[String]("foo")).exists(x => true)
}

end HelloWorld
21 changes: 20 additions & 1 deletion tasty/src/dotty/tools/tasty/TastyFormat.scala
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,15 @@ All elements of a position section are serialized as Ints
Standard Section: "Comments" Comment*
```none
Comment = Length Bytes LongInt // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
Comment = UTF8 LongInt // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
```
Standard Section: "Attributes" Attribute*
```none
Attribute = SCALA2STANDARDLIBRARYattr
EXPLICITNULLSattr
```
**************************************************************************************/

object TastyFormat {
Expand Down Expand Up @@ -361,6 +368,7 @@ object TastyFormat {
final val ASTsSection = "ASTs"
final val PositionsSection = "Positions"
final val CommentsSection = "Comments"
final val AttributesSection = "Attributes"

/** Tags used to serialize names, should update [[TastyFormat$.nameTagToString]] if a new constant is added */
class NameTags {
Expand Down Expand Up @@ -597,6 +605,12 @@ object TastyFormat {
final val firstNatASTTreeTag = IDENT
final val firstLengthTreeTag = PACKAGE


// Attributes tags

final val SCALA2STANDARDLIBRARYattr = 1
final val EXPLICITNULLSattr = 2

/** Useful for debugging */
def isLegalTag(tag: Int): Boolean =
firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE ||
Expand Down Expand Up @@ -812,6 +826,11 @@ object TastyFormat {
case HOLE => "HOLE"
}

def attributeTagToString(tag: Int): String = tag match {
case SCALA2STANDARDLIBRARYattr => "SCALA2STANDARDLIBRARYattr"
case EXPLICITNULLSattr => "EXPLICITNULLSattr"
}

/** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.
* If negative, minus the number of leading non-reference trees.
*/
Expand Down

0 comments on commit 319e865

Please sign in to comment.