Support the -Ypickle-java flag.

- Add JAVAattr and OUTLINEattr TASTy attributes, ELIDED tree tag.
  ELIDED trees are pickled as rhs of java term definitions.
  ELIDED trees can only be unpickled if OUTLINEattr is present.
  For now OUTLINEattr implies JAVAattr. In the future we might
  expand OUTLINEattr to include outline Scala typing.
- Keep java mode compilation units up to ExtractAPI phase if pipeline.

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -31,6 +31,11 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Is this the compilation unit of a Java file */
   def isJava: Boolean = source.file.name.endsWith(".java")
 
+  /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
+  def typedAsJava = isJava || tastyAttributes.exists(_.isJava)
+
+  var tastyAttributes: Option[tasty.Attributes] = None
+
   /** The source version for this unit, as determined by a language import */
   var sourceVersion: Option[SourceVersion] = None
 

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -17,6 +17,7 @@ import reporting.*
 import annotation.constructorOnly
 import printing.Formatting.hl
 import config.Printers
+import parsing.Parsers
 
 import scala.annotation.internal.sharable
 import scala.annotation.threadUnsafe
@@ -143,8 +144,13 @@ object desugar {
 
   /** A value definition copied from `vdef` with a tpt typetree derived from it */
   def derivedTermParam(vdef: ValDef)(using Context): ValDef =
+    derivedTermParam(vdef, vdef.unforcedRhs)
+
+  def derivedTermParam(vdef: ValDef, rhs: LazyTree)(using Context): ValDef =
     cpy.ValDef(vdef)(
-      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef))
+      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef),
+      rhs = rhs
+    )
 
 // ----- Desugar methods -------------------------------------------------
 
@@ -544,8 +550,11 @@ object desugar {
       constrTparams.zipWithConserve(impliedTparams)((tparam, impliedParam) =>
         derivedTypeParam(tparam).withAnnotations(impliedParam.mods.annotations))
     val derivedVparamss =
-      constrVparamss.nestedMap(vparam =>
-        derivedTermParam(vparam).withAnnotations(Nil))
+      constrVparamss.nestedMap: vparam =>
+        val derived =
+          if ctx.compilationUnit.isJava then derivedTermParam(vparam, Parsers.unimplementedExpr)
+          else derivedTermParam(vparam)
+        derived.withAnnotations(Nil)
 
     val constr = cpy.DefDef(constr1)(paramss = joinParams(constrTparams, constrVparamss))
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -436,6 +436,6 @@ private sealed trait YSettings:
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
 
   // Pipeline compilation options
-  val YpickleJava: Setting[Boolean] = BooleanSetting("-Ypickle-java", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
+  val YpickleJava: Setting[Boolean] = BooleanSetting("-Ypickle-java", "Pickler phase should compute pickles for .java defined symbols for use by build tools", aliases = List("-Yjava-tasty"))
   val YpickleWrite: Setting[AbstractFile] = OutputSetting("-Ypickle-write", "directory|jar", "destination for generated .sig files containing type signatures.", NoAbstractFile, aliases = List("-Yearly-tasty"))
 end YSettings

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1059,6 +1059,7 @@ class Definitions {
   @tu lazy val RetainsByNameAnnot: ClassSymbol = requiredClass("scala.annotation.retainsByName")
 
   @tu lazy val JavaRepeatableAnnot: ClassSymbol = requiredClass("java.lang.annotation.Repeatable")
+  @tu lazy val JavaAnnotationAnnot: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Initialization annotations
   @tu lazy val InitModule: Symbol = requiredModule("scala.annotation.init")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -211,6 +211,7 @@ object Phases {
     private var myPostTyperPhase: Phase = uninitialized
     private var mySbtExtractDependenciesPhase: Phase = uninitialized
     private var myPicklerPhase: Phase = uninitialized
+    private var mySbtExtractApiPhase: Phase = uninitialized
     private var myInliningPhase: Phase = uninitialized
     private var myStagingPhase: Phase = uninitialized
     private var mySplicingPhase: Phase = uninitialized
@@ -236,6 +237,7 @@ object Phases {
     final def postTyperPhase: Phase = myPostTyperPhase
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
+    final def sbtExtractApiPhase: Phase = mySbtExtractApiPhase
     final def inliningPhase: Phase = myInliningPhase
     final def stagingPhase: Phase = myStagingPhase
     final def splicingPhase: Phase = mySplicingPhase
@@ -264,6 +266,7 @@ object Phases {
       myPostTyperPhase = phaseOfClass(classOf[PostTyper])
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
+      mySbtExtractApiPhase = phaseOfClass(classOf[sbt.ExtractAPI])
       myInliningPhase = phaseOfClass(classOf[Inlining])
       myStagingPhase = phaseOfClass(classOf[Staging])
       mySplicingPhase = phaseOfClass(classOf[Splicing])
@@ -333,16 +336,25 @@ object Phases {
     def subPhases: List[Run.SubPhase] = Nil
     final def traversals: Int = if subPhases.isEmpty then 1 else subPhases.length
 
+    /** skip the phase for a Java compilation unit, may depend on -Ypickle-java */
+    def skipIfJava(using Context): Boolean = true
+
     /** @pre `isRunnable` returns true */
     def run(using Context): Unit
 
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
+      val skipJavaUnits =
+        ctx.settings.YpickleJava.value && ctx.phase <= sbtExtractApiPhase && skipIfJava
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
-          try run
+          try
+            if skipJavaUnits && unit.typedAsJava then
+              ()
+            else
+              run
           catch case ex: Throwable if !ctx.run.enrichedErrorMessage =>
             println(ctx.run.enrichErrorMessage(s"unhandled exception while running $phaseName on $unit"))
             throw ex
@@ -495,6 +507,7 @@ object Phases {
   def postTyperPhase(using Context): Phase              = ctx.base.postTyperPhase
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
+  def sbtExtractApiPhase(using Context): Phase          = ctx.base.sbtExtractDependenciesPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
   def stagingPhase(using Context): Phase               = ctx.base.stagingPhase
   def splicingPhase(using Context): Phase               = ctx.base.splicingPhase

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1696,7 +1696,7 @@ object SymDenotations {
             c.ensureCompleted()
       end completeChildrenIn
 
-      if is(Sealed) || isAllOf(JavaEnumTrait) then
+      if is(Sealed) || isAllOf(JavaEnumTrait) && isClass then
         if !is(ChildrenQueried) then
           // Make sure all visible children are completed, so that
           // they show up in Child annotations. A possible child is visible if it

compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

@@ -14,11 +14,13 @@ object AttributePickler:
     pickler: TastyPickler,
     buf: TastyBuffer
   ): Unit =
-    if attributes.scala2StandardLibrary || attributes.explicitNulls then // or any other attribute is set
+    if attributes.nonEmpty then
       pickler.newSection(AttributesSection, buf)
       // Pickle attributes
       if attributes.scala2StandardLibrary then buf.writeNat(TastyFormat.SCALA2STANDARDLIBRARYattr)
       if attributes.explicitNulls then buf.writeNat(TastyFormat.EXPLICITNULLSattr)
+      if attributes.isJava then buf.writeNat(TastyFormat.JAVAattr)
+      if attributes.isOutline then buf.writeNat(TastyFormat.OUTLINEattr)
     end if
 
   end pickleAttributes

compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

@@ -18,15 +18,21 @@ class AttributeUnpickler(reader: TastyReader):
   lazy val attributes: Attributes = {
     var scala2StandardLibrary = false
     var explicitNulls = false
+    var isJava = false
+    var isOutline = false
     for attributeTag <- attributeTags do
       attributeTag match
         case TastyFormat.SCALA2STANDARDLIBRARYattr => scala2StandardLibrary = true
         case TastyFormat.EXPLICITNULLSattr => explicitNulls = true
+        case TastyFormat.JAVAattr => isJava = true
+        case TastyFormat.OUTLINEattr => isOutline = true
         case attribute =>
           assert(false, "Unexpected attribute value: " + attribute)
     Attributes(
       scala2StandardLibrary,
       explicitNulls,
+      isJava,
+      isOutline,
     )
   }
 

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -3,4 +3,12 @@ package dotty.tools.dotc.core.tasty
 class Attributes(
   val scala2StandardLibrary: Boolean,
   val explicitNulls: Boolean,
-)
+  val isJava: Boolean,
+  val isOutline: Boolean,
+) {
+  def nonEmpty: Boolean = scala2StandardLibrary || explicitNulls || isJava || isOutline
+}
+
+object Attributes {
+  lazy val empty = new Attributes(false, false, false, false)
+}

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -20,8 +20,11 @@ object DottyUnpickler {
   /** Exception thrown if classfile is corrupted */
   class BadSignature(msg: String) extends RuntimeException(msg)
 
-  class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], attributeUnpickler: Option[AttributeUnpickler])
-  extends SectionUnpickler[TreeUnpickler](ASTsSection) {
+  class TreeSectionUnpickler(
+      posUnpickler: Option[PositionUnpickler],
+      commentUnpickler: Option[CommentUnpickler],
+      attributeUnpickler: Option[AttributeUnpickler]
+  ) extends SectionUnpickler[TreeUnpickler](ASTsSection) {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler =
       new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, attributeUnpickler)
   }
@@ -35,6 +38,7 @@ object DottyUnpickler {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
       new CommentUnpickler(reader)
   }
+
   class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
       new AttributeUnpickler(reader)
@@ -61,6 +65,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
   def enter(roots: Set[SymDenotation])(using Context): Unit =
     treeUnpickler.enter(roots)
 
+  def attributes: Option[Attributes] = attributeUnpicklerOpt.map(_.attributes)
+
   protected def treeSectionUnpickler(
     posUnpicklerOpt: Option[PositionUnpickler],
     commentUnpicklerOpt: Option[CommentUnpickler],

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -24,7 +24,7 @@ import quoted.QuotePatterns
 object TreePickler:
   class StackSizeExceeded(val mdef: tpd.MemberDef) extends Exception
 
-class TreePickler(pickler: TastyPickler) {
+class TreePickler(pickler: TastyPickler, attributes: Attributes) {
   val buf: TreeBuffer = new TreeBuffer
   pickler.newSection(ASTsSection, buf)
   import buf.*
@@ -323,6 +323,11 @@ class TreePickler(pickler: TastyPickler) {
     if (!tree.isEmpty) pickleTree(tree)
   }
 
+  def pickleElidedUnlessEmpty(tree: Tree, tp: Type)(using Context): Unit =
+    if !tree.isEmpty then
+      writeByte(ELIDED)
+      pickleType(tp)
+
   def pickleDef(tag: Int, mdef: MemberDef, tpt: Tree, rhs: Tree = EmptyTree, pickleParams: => Unit = ())(using Context): Unit = {
     val sym = mdef.symbol
 
@@ -338,7 +343,12 @@ class TreePickler(pickler: TastyPickler) {
           case _: Template | _: Hole => pickleTree(tpt)
           case _ if tpt.isType => pickleTpt(tpt)
         }
-        pickleTreeUnlessEmpty(rhs)
+        if attributes.isOutline && sym.isTerm && attributes.isJava then
+          // TODO: if we introduce outline typing for Scala definitions
+          // then we will need to update the check here
+          pickleElidedUnlessEmpty(rhs, tpt.tpe)
+        else
+          pickleTreeUnlessEmpty(rhs)
         pickleModifiers(sym, mdef)
       }
     catch

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -47,6 +47,8 @@ import dotty.tools.tasty.TastyFormat.*
 import scala.annotation.constructorOnly
 import scala.annotation.internal.sharable
 import scala.compiletime.uninitialized
+import dotty.tools.tasty.UnpickleException
+import dotty.tools.tasty.TastyFormat
 
 /** Unpickler for typed trees
  *  @param reader              the reader from which to unpickle
@@ -107,6 +109,11 @@ class TreeUnpickler(reader: TastyReader,
   private val explicitNulls =
     attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
 
+  private val unpicklingJava =
+    attributeUnpicklerOpt.exists(_.attributes.isJava)
+
+  private val isOutline = attributeUnpicklerOpt.exists(_.attributes.isOutline)
+
   private def registerSym(addr: Addr, sym: Symbol) =
     symAtAddr(addr) = sym
 
@@ -115,6 +122,13 @@ class TreeUnpickler(reader: TastyReader,
    */
   def enter(roots: Set[SymDenotation])(using Context): Unit = {
     this.roots = roots
+    if isOutline && !unpicklingJava then
+      // TODO: overly cautious here, eventually we could enable this with 2-pass compilation.
+      throw UnpickleException("Outline TASTy is not supported for Scala sources.")
+    if unpicklingJava && !ctx.settings.YpickleJava.value then
+      throw UnpickleException(
+        "Top level class comes from Java sources. " +
+        "TASTy signatures are not supported for Java without the -Ypickle-java flag.")
     val rdr = new TreeReader(reader).fork
     ownerTree = new OwnerTree(NoAddr, 0, rdr.fork, reader.endAddr)
     if (rdr.isTopLevel)
@@ -612,7 +626,10 @@ class TreeUnpickler(reader: TastyReader,
       val rhsIsEmpty = nothingButMods(end)
       if (!rhsIsEmpty) skipTree()
       val (givenFlags0, annotFns, privateWithin) = readModifiers(end)
-      val givenFlags = if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty else givenFlags0
+      val givenFlags =
+        if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty
+        else if unpicklingJava then givenFlags0 | JavaDefined
+        else givenFlags0
       pickling.println(i"creating symbol $name at $start with flags ${givenFlags.flagsString}, isAbsType = $isAbsType, $ttag")
       val flags = normalizeFlags(tag, givenFlags, name, isAbsType, rhsIsEmpty)
       def adjustIfModule(completer: LazyType) =
@@ -867,6 +884,7 @@ class TreeUnpickler(reader: TastyReader,
       def readRhs(using Context): LazyTree =
         if (nothingButMods(end))
           EmptyTree
+
         else if sym.isInlineMethod && !sym.is(Deferred) then
           // The body of an inline method is stored in an annotation, so no need to unpickle it again
           new Trees.Lazy[Tree] {
@@ -1045,6 +1063,8 @@ class TreeUnpickler(reader: TastyReader,
       val parentReader = fork
       val parents = readParents(withArgs = false)(using parentCtx)
       val parentTypes = parents.map(_.tpe.dealias)
+      if cls.is(JavaDefined) && parentTypes.exists(_.derivesFrom(defn.JavaAnnotationAnnot)) then
+        cls.setFlag(JavaAnnotation)
       val self =
         if (nextByte == SELFDEF) {
           readByte()
@@ -1205,7 +1225,12 @@ class TreeUnpickler(reader: TastyReader,
 
       def completeSelect(name: Name, sig: Signature, target: Name): Select =
         val qual = readTree()
-        val denot = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot0 = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot =
+          if unpicklingJava && name == tpnme.Object && denot0.symbol == defn.ObjectClass then
+            defn.FromJavaObjectType.denot
+          else
+            denot0
         makeSelect(qual, name, denot)
 
       def readQualId(): (untpd.Ident, TypeRef) =
@@ -1224,6 +1249,11 @@ class TreeUnpickler(reader: TastyReader,
           forkAt(readAddr()).readTree()
         case IDENT =>
           untpd.Ident(readName()).withType(readType())
+        case ELIDED =>
+          if !isOutline then
+            report.error(
+              s"Illegal elided tree in unpickler without ${attributeTagToString(OUTLINEattr)}, ${ctx.source}")
+          untpd.Ident(nme.WILDCARD).withType(readType())
         case IDENTtpt =>
           untpd.Ident(readName().toTypeName).withType(readType())
         case SELECT =>

compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

@@ -46,9 +46,14 @@ class ReadTasty extends Phase {
             case unpickler: tasty.DottyUnpickler =>
               if (cls.rootTree.isEmpty) None
               else {
-                val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
-                unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
-                Some(unit)
+                if unpickler.attributes.exists(_.isJava) && !ctx.settings.YpickleJava.value then
+                  // filter out Java compilation units if -Ypickle-java is not set
+                  None
+                else
+                  val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
+                  unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
+                  unit.tastyAttributes = unpickler.attributes
+                  Some(unit)
               }
             case tree: Tree[?] =>
               // TODO handle correctly this case correctly to get the tree or avoid it completely.

compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala

@@ -136,7 +136,7 @@ object JavaParsers {
       ValDef(name, tpt, EmptyTree).withMods(Modifiers(Flags.JavaDefined | Flags.Param))
 
     def makeConstructor(formals: List[Tree], tparams: List[TypeDef], flags: FlagSet = Flags.JavaDefined): DefDef = {
-      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p) }
+      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p).withMods(Modifiers(flags)) }
       DefDef(nme.CONSTRUCTOR, joinParams(tparams, List(vparams)), TypeTree(), EmptyTree).withMods(Modifiers(flags))
     }
 

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -81,6 +81,9 @@ object Parsers {
   private val InCond: Region => Region = Scanners.InParens(LPAREN, _)
   private val InFor : Region => Region = Scanners.InBraces(_)
 
+  def unimplementedExpr(using Context): Select =
+    Select(scalaDot(nme.Predef), nme.???)
+
   abstract class ParserCommon(val source: SourceFile)(using Context) {
 
     val in: ScannerCommon
@@ -148,9 +151,6 @@ object Parsers {
      */
     def syntaxError(msg: Message, span: Span): Unit =
       report.error(msg, source.atSpan(span))
-
-    def unimplementedExpr(using Context): Select =
-      Select(scalaDot(nme.Predef), nme.???)
   }
 
   trait OutlineParserCommon extends ParserCommon {