Export diagnostics (including unused warnings) to SemanticDB (scala#1…

…7835)

fix scala#17535

## What this PR does
This PR splits `ExtractSemanticDB` to `PostTyper` and `PostInlining` (as
`ChechUnused` does).

- The `PostTyper` phase
- It extracts SemanticDB information such as symbol definitions, symbol
occurrences, type information, and synthetics.
- ~**This phase does not write the information to a `.semanticdb`
file**; instead, it attaches the SemanticDB information to the top-level
tree.~
  - And write `.semanticdb` file.
-  The `PostInlining` phase
- It extracts diagnostics from `ctx.reporter` and attaches them to the
SemanticDB information extracted in the `PostTyper` phase.
- Afterwards, it updates the SemanticDB to a `.semanticdb` file (if
there's warning in the file).
- **We need to run this phase after the `CheckUnused.PostInlining` phase
so that we can extract the warnings generated by "-Wunused".**

Also,

- Reporter now stores `warnings` in addition to `errors`
- Tweaked SemanticDBTest to show the generated diagnostics to
`metac.expect`

### Concerns

- Since we attach the SemanticDB information to the top-level tree, it
lives in-memory during the compilation which may leads to more memory
usage for compilation
- Now, Reporter stores all warnings in addition to errors (to convey the
warnings across phases), which also may cause some more memory
consumption.

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -39,7 +39,7 @@ class Compiler {
     List(new CheckShadowing) :: // Check shadowing elements
     List(new YCheckPositions) ::    // YCheck positions
     List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
-    List(new semanticdb.ExtractSemanticDB) :: // Extract info into .semanticdb files
+    List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
     List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
@@ -72,6 +72,7 @@ class Compiler {
          new ExpandSAMs,             // Expand single abstract method closures to anonymous classes
          new ElimRepeated,           // Rewrite vararg parameters and arguments
          new RefChecks) ::           // Various checks mostly related to abstract members and overriding
+    List(new semanticdb.ExtractSemanticDB.AppendDiagnostics) :: // Attach warnings to extracted SemanticDB and write to .semanticdb file
     List(new init.Checker) ::        // Check initialization of objects
     List(new ProtectedAccessors,     // Add accessors for protected members
          new ExtensionMethods,       // Expand methods of value classes with extension methods

compiler/src/dotty/tools/dotc/reporting/Reporter.scala

@@ -109,9 +109,14 @@ abstract class Reporter extends interfaces.ReporterResult {
 
   private var errors: List[Error] = Nil
 
+  private var warnings: List[Warning] = Nil
+
   /** All errors reported by this reporter (ignoring outer reporters) */
   def allErrors: List[Error] = errors
 
+  /** All warnings reported by this reporter (ignoring outer reporters) */
+  def allWarnings: List[Warning] = warnings
+
   /** Were sticky errors reported? Overridden in StoreReporter. */
   def hasStickyErrors: Boolean = false
 
@@ -153,7 +158,9 @@ abstract class Reporter extends interfaces.ReporterResult {
         markReported(dia)
         withMode(Mode.Printing)(doReport(dia))
         dia match {
-          case _: Warning => _warningCount += 1
+          case w: Warning =>
+            warnings = w :: warnings
+            _warningCount += 1
           case e: Error   =>
             errors = e :: errors
             _errorCount += 1

compiler/src/dotty/tools/dotc/semanticdb/DiagnosticOps.scala

@@ -0,0 +1,23 @@
+package dotty.tools.dotc.semanticdb
+
+import dotty.tools.dotc.reporting.Diagnostic
+import dotty.tools.dotc.{semanticdb => s}
+import dotty.tools.dotc.interfaces.Diagnostic.{ERROR, INFO, WARNING}
+import dotty.tools.dotc.core.Contexts.Context
+import scala.annotation.internal.sharable
+
+object DiagnosticOps:
+  @sharable private val asciiColorCodes = "\u001B\\[[;\\d]*m".r
+  extension (d: Diagnostic)
+    def toSemanticDiagnostic: s.Diagnostic =
+      val severity = d.level match
+        case ERROR => s.Diagnostic.Severity.ERROR
+        case WARNING => s.Diagnostic.Severity.WARNING
+        case INFO => s.Diagnostic.Severity.INFORMATION
+        case _ => s.Diagnostic.Severity.INFORMATION
+      val msg = asciiColorCodes.replaceAllIn(d.msg.message, m => "")
+      s.Diagnostic(
+        range = Scala3.range(d.pos.span, d.pos.source),
+        severity = severity,
+        message = msg
+      )

compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala

@@ -21,20 +21,34 @@ import transform.SymUtils._
 
 import scala.collection.mutable
 import scala.annotation.{ threadUnsafe => tu, tailrec }
+import scala.jdk.CollectionConverters._
 import scala.PartialFunction.condOpt
+import typer.ImportInfo.withRootImports
 
 import dotty.tools.dotc.{semanticdb => s}
 import dotty.tools.io.{AbstractFile, JarArchive}
+import dotty.tools.dotc.semanticdb.DiagnosticOps.*
+import scala.util.{Using, Failure, Success}
+
 
 /** Extract symbol references and uses to semanticdb files.
  *  See https://scalameta.org/docs/semanticdb/specification.html#symbol-1
  *  for a description of the format.
- *  TODO: Also extract type information
+ *
+ *  Here, we define two phases for "ExtractSemanticDB", "PostTyper" and "PostInlining".
+ *
+ *  The "PostTyper" phase extracts SemanticDB information such as symbol
+ *  definitions, symbol occurrences, type information, and synthetics
+ *  and write .semanticdb file.
+ *
+ *  The "PostInlining" phase extracts diagnostics from "ctx.reporter" and
+ *  attaches them to the SemanticDB information extracted in the "PostTyper" phase.
+ *  We need to run this phase after the "CheckUnused.PostInlining" phase
+ *  so that we can extract the warnings generated by "-Wunused".
  */
-class ExtractSemanticDB extends Phase:
-  import Scala3.{_, given}
+class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends Phase:
 
-  override val phaseName: String = ExtractSemanticDB.name
+  override val phaseName: String = ExtractSemanticDB.phaseNamePrefix + phaseMode.toString()
 
   override val description: String = ExtractSemanticDB.description
 
@@ -46,14 +60,145 @@ class ExtractSemanticDB extends Phase:
   // Check not needed since it does not transform trees
   override def isCheckable: Boolean = false
 
-  override def run(using Context): Unit =
-    val unit = ctx.compilationUnit
-    val extractor = Extractor()
-    extractor.extract(unit.tpdTree)
-    ExtractSemanticDB.write(unit.source, extractor.occurrences.toList, extractor.symbolInfos.toList, extractor.synthetics.toList)
+  override def runOn(units: List[CompilationUnit])(using ctx: Context): List[CompilationUnit] = {
+    val sourceRoot = ctx.settings.sourceroot.value
+    val appendDiagnostics = phaseMode == ExtractSemanticDB.PhaseMode.AppendDiagnostics
+    if (appendDiagnostics)
+      val warnings = ctx.reporter.allWarnings.groupBy(w => w.pos.source)
+      units.flatMap { unit =>
+        warnings.get(unit.source).map { ws =>
+          val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports
+          val outputDir =
+            ExtractSemanticDB.semanticdbPath(
+              unit.source,
+              ExtractSemanticDB.semanticdbOutDir(using unitCtx),
+              sourceRoot
+            )
+          (outputDir, ws.map(_.toSemanticDiagnostic))
+        }
+      }.asJava.parallelStream().forEach { case (out, warnings) =>
+        ExtractSemanticDB.appendDiagnostics(warnings, out)
+      }
+    else
+      val writeSemanticdbText = ctx.settings.semanticdbText.value
+      units.foreach { unit =>
+        val unitCtx = ctx.fresh.setCompilationUnit(unit).withRootImports
+        val outputDir =
+          ExtractSemanticDB.semanticdbPath(
+            unit.source,
+            ExtractSemanticDB.semanticdbOutDir(using unitCtx),
+            sourceRoot
+          )
+        val extractor = ExtractSemanticDB.Extractor()
+        extractor.extract(unit.tpdTree)(using unitCtx)
+        ExtractSemanticDB.write(
+          unit.source,
+          extractor.occurrences.toList,
+          extractor.symbolInfos.toList,
+          extractor.synthetics.toList,
+          outputDir,
+          sourceRoot,
+          writeSemanticdbText
+        )
+      }
+    units
+  }
+
+  def run(using Context): Unit = unsupported("run")
+end ExtractSemanticDB
+
+object ExtractSemanticDB:
+  import java.nio.file.Path
+  import java.nio.file.Files
+  import java.nio.file.Paths
+
+  val phaseNamePrefix: String = "extractSemanticDB"
+  val description: String = "extract info into .semanticdb files"
+
+  enum PhaseMode:
+    case ExtractSemanticInfo
+    case AppendDiagnostics
+
+  class ExtractSemanticInfo extends ExtractSemanticDB(PhaseMode.ExtractSemanticInfo)
+
+  class AppendDiagnostics extends ExtractSemanticDB(PhaseMode.AppendDiagnostics)
+
+  private def semanticdbTarget(using Context): Option[Path] =
+    Option(ctx.settings.semanticdbTarget.value)
+      .filterNot(_.isEmpty)
+      .map(Paths.get(_))
+
+  /** Destination for generated classfiles */
+  private def outputDirectory(using Context): AbstractFile =
+    ctx.settings.outputDir.value
+
+  /** Output directory for SemanticDB files */
+  private def semanticdbOutDir(using Context): Path =
+    semanticdbTarget.getOrElse(outputDirectory.jpath)
+
+  private def absolutePath(path: Path): Path = path.toAbsolutePath.normalize
+
+  private def write(
+    source: SourceFile,
+    occurrences: List[SymbolOccurrence],
+    symbolInfos: List[SymbolInformation],
+    synthetics: List[Synthetic],
+    outpath: Path,
+    sourceRoot: String,
+    semanticdbText: Boolean
+  ): Unit =
+    Files.createDirectories(outpath.getParent())
+    val doc: TextDocument = TextDocument(
+      schema = Schema.SEMANTICDB4,
+      language = Language.SCALA,
+      uri = Tools.mkURIstring(Paths.get(relPath(source, sourceRoot))),
+      text = if semanticdbText then String(source.content) else "",
+      md5 = internal.MD5.compute(String(source.content)),
+      symbols = symbolInfos,
+      occurrences = occurrences,
+      synthetics = synthetics,
+    )
+    val docs = TextDocuments(List(doc))
+    val out = Files.newOutputStream(outpath)
+    try
+      val stream = internal.SemanticdbOutputStream.newInstance(out)
+      docs.writeTo(stream)
+      stream.flush()
+    finally
+      out.close()
+  end write
+
+  private def appendDiagnostics(
+    diagnostics: Seq[Diagnostic],
+    outpath: Path
+  ): Unit =
+    Using.Manager { use =>
+      val in = use(Files.newInputStream(outpath))
+      val sin = internal.SemanticdbInputStream.newInstance(in)
+      val docs = TextDocuments.parseFrom(sin)
+
+      val out = use(Files.newOutputStream(outpath))
+      val sout = internal.SemanticdbOutputStream.newInstance(out)
+      TextDocuments(docs.documents.map(_.withDiagnostics(diagnostics))).writeTo(sout)
+      sout.flush()
+    } match
+      case Failure(ex) => // failed somehow, should we say something?
+      case Success(_) => // success to update semanticdb, say nothing
+  end appendDiagnostics
+
+  private def relPath(source: SourceFile, sourceRoot: String) =
+    SourceFile.relativePath(source, sourceRoot)
+
+  private def semanticdbPath(source: SourceFile, base: Path, sourceRoot: String): Path =
+    absolutePath(base)
+      .resolve("META-INF")
+      .resolve("semanticdb")
+      .resolve(relPath(source, sourceRoot))
+      .resolveSibling(source.name + ".semanticdb")
 
   /** Extractor of symbol occurrences from trees */
   class Extractor extends TreeTraverser:
+    import Scala3.{_, given}
     given s.SemanticSymbolBuilder = s.SemanticSymbolBuilder()
     val synth = SyntheticsExtractor()
     given converter: s.TypeOps = s.TypeOps()
@@ -465,55 +610,5 @@ class ExtractSemanticDB extends Phase:
           registerSymbol(vparam.symbol, symkinds)
         traverse(vparam.tpt)
       tparams.foreach(tp => traverse(tp.rhs))
-
-
-object ExtractSemanticDB:
-  import java.nio.file.Path
-  import java.nio.file.Files
-  import java.nio.file.Paths
-
-  val name: String = "extractSemanticDB"
-  val description: String = "extract info into .semanticdb files"
-
-  private def semanticdbTarget(using Context): Option[Path] =
-    Option(ctx.settings.semanticdbTarget.value)
-      .filterNot(_.isEmpty)
-      .map(Paths.get(_))
-
-  private def semanticdbText(using Context): Boolean =
-    ctx.settings.semanticdbText.value
-
-  private def outputDirectory(using Context): AbstractFile = ctx.settings.outputDir.value
-
-  def write(
-    source: SourceFile,
-    occurrences: List[SymbolOccurrence],
-    symbolInfos: List[SymbolInformation],
-    synthetics: List[Synthetic],
-  )(using Context): Unit =
-    def absolutePath(path: Path): Path = path.toAbsolutePath.normalize
-    val relPath = SourceFile.relativePath(source, ctx.settings.sourceroot.value)
-    val outpath = absolutePath(semanticdbTarget.getOrElse(outputDirectory.jpath))
-      .resolve("META-INF")
-      .resolve("semanticdb")
-      .resolve(relPath)
-      .resolveSibling(source.name + ".semanticdb")
-    Files.createDirectories(outpath.getParent())
-    val doc: TextDocument = TextDocument(
-      schema = Schema.SEMANTICDB4,
-      language = Language.SCALA,
-      uri = Tools.mkURIstring(Paths.get(relPath)),
-      text = if semanticdbText then String(source.content) else "",
-      md5 = internal.MD5.compute(String(source.content)),
-      symbols = symbolInfos,
-      occurrences = occurrences,
-      synthetics = synthetics,
-    )
-    val docs = TextDocuments(List(doc))
-    val out = Files.newOutputStream(outpath)
-    try
-      val stream = internal.SemanticdbOutputStream.newInstance(out)
-      docs.writeTo(stream)
-      stream.flush()
-    finally
-      out.close()
+  end Extractor
+end ExtractSemanticDB

compiler/src/dotty/tools/dotc/semanticdb/Scala3.scala

@@ -29,7 +29,7 @@ object Scala3:
 
   private val WILDCARDTypeName = nme.WILDCARD.toTypeName
 
-  def range(span: Span, treeSource: SourceFile)(using Context): Option[Range] =
+  def range(span: Span, treeSource: SourceFile): Option[Range] =
     def lineCol(offset: Int) = (treeSource.offsetToLine(offset), treeSource.column(offset))
     val (startLine, startCol) = lineCol(span.start)
     val (endLine, endCol) = lineCol(span.end)
@@ -486,6 +486,8 @@ object Scala3:
 
   given Ordering[SymbolInformation] = Ordering.by[SymbolInformation, String](_.symbol)(IdentifierOrdering())
 
+  given Ordering[Diagnostic] = (x, y) => compareRange(x.range, y.range)
+
   given Ordering[Synthetic] = (x, y) => compareRange(x.range, y.range)
 
   /**

compiler/src/dotty/tools/dotc/semanticdb/Tools.scala

@@ -69,6 +69,8 @@ object Tools:
     sb.append("Language => ").append(languageString(doc.language)).nl
     sb.append("Symbols => ").append(doc.symbols.length).append(" entries").nl
     sb.append("Occurrences => ").append(doc.occurrences.length).append(" entries").nl
+    if doc.diagnostics.nonEmpty then
+      sb.append("Diagnostics => ").append(doc.diagnostics.length).append(" entries").nl
     if doc.synthetics.nonEmpty then
       sb.append("Synthetics => ").append(doc.synthetics.length).append(" entries").nl
     sb.nl
@@ -78,6 +80,10 @@ object Tools:
     sb.append("Occurrences:").nl
     doc.occurrences.sorted.foreach(processOccurrence)
     sb.nl
+    if doc.diagnostics.nonEmpty then
+      sb.append("Diagnostics:").nl
+      doc.diagnostics.sorted.foreach(d => processDiag(d))
+      sb.nl
     if doc.synthetics.nonEmpty then
       sb.append("Synthetics:").nl
       doc.synthetics.sorted.foreach(s => processSynth(s, synthPrinter))
@@ -108,6 +114,20 @@ object Tools:
   private def processSynth(synth: Synthetic, printer: SyntheticPrinter)(using sb: StringBuilder): Unit =
     sb.append(printer.pprint(synth)).nl
 
+  private def processDiag(d: Diagnostic)(using sb: StringBuilder): Unit =
+    d.range match
+      case Some(range) => processRange(sb, range)
+      case _ => sb.append("[):")
+    sb.append(" ")
+    d.severity match
+      case Diagnostic.Severity.ERROR => sb.append("[error]")
+      case Diagnostic.Severity.WARNING => sb.append("[warning]")
+      case Diagnostic.Severity.INFORMATION => sb.append("[info]")
+      case _ => sb.append("[unknown]")
+    sb.append(" ")
+    sb.append(d.message)
+    sb.nl
+
   private def processOccurrence(occ: SymbolOccurrence)(using sb: StringBuilder, sourceFile: SourceFile): Unit =
     occ.range match
     case Some(range) =>

compiler/test/dotty/tools/dotc/semanticdb/SemanticdbTests.scala

@@ -142,7 +142,8 @@ class SemanticdbTests:
       "-sourceroot", expectSrc.toString,
       "-classpath", target.toString,
       "-Xignore-scala2-macros",
-      "-usejavacp"
+      "-usejavacp",
+      "-Wunused:all"
     ) ++ inputFiles().map(_.toString)
     val exit = Main.process(args)
     assertFalse(s"dotc errors: ${exit.errorCount}", exit.hasErrors)

presentation-compiler/src/main/dotty/tools/pc/SemanticdbTextDocumentProvider.scala

@@ -32,8 +32,7 @@ class SemanticdbTextDocumentProvider(
       SourceFile.virtual(filePath.toString, validCode)
     )
     val tree = driver.currentCtx.run.units.head.tpdTree
-    val extract = ExtractSemanticDB()
-    val extractor = extract.Extractor()
+    val extractor = ExtractSemanticDB.Extractor()
     extractor.traverse(tree)(using driver.currentCtx)
     val path = workspace
       .flatMap { workspacePath =>

tests/semanticdb/expect/Deprecated.expect.scala

@@ -0,0 +1,4 @@
+object Deprecated/*<-_empty_::Deprecated.*/ {
+  @deprecated/*->scala::deprecated#*/ def deprecatedMethod/*<-_empty_::Deprecated.deprecatedMethod().*/ = ???/*->scala::Predef.`???`().*/
+  def main/*<-_empty_::Deprecated.main().*/ = deprecatedMethod/*->_empty_::Deprecated.deprecatedMethod().*/
+}