Merge pull request #52 from mpollmeier/scala3/remove-tasty-files

Scala3: remove tasty files

Author: eed3si9n

README.md

@@ -146,6 +146,10 @@ proguardMergeStrategies in Proguard ++= Seq(
 Completely custom merge strategies can also be created. See the plugin source
 code for how this could be done.
 
+Scala 3
+---------------
+ProGuard doesn't handle Scala 3's TASTy files, which contain much more information than Java's class files. Therefore, we need to post-process the ProGuard output JAR and remove all TASTy files for classes that have been obfuscated. To determine which classes have been obfuscated, you must configure the `-mappingsfile` option, e.g., via `Proguard / proguardOptions += ProguardOptions.mappingsFile("mappings.txt")`. See the [Scala 3 test project](src/sbt-test/proguard/scala3), which is included in the scripted tests.
+
 
 Sample projects
 ---------------

src/main/scala/com/typesafe/sbt/ProguardKeys.scala

@@ -63,6 +63,9 @@ trait ProguardKeys {
         |    public static $returnType $methodName ($inputType);
         |}""".stripMargin
     }
+
+    def mappingsFile(fileName: String): String =
+      s"-printmapping $fileName"
   }
 
   object ProguardMerge {

src/main/scala/com/typesafe/sbt/SbtProguard.scala

@@ -1,9 +1,11 @@
 package com.lightbend.sbt
 
 import com.lightbend.sbt.proguard.Merge
-import sbt.Keys._
+import java.nio.file.{Files, FileSystems}
 import sbt._
-import java.nio.file.FileSystems
+import sbt.Keys._
+import sbt.internal.util.ManagedLogger
+import scala.collection.JavaConverters._
 import scala.sys.process.Process
 
 object SbtProguard extends AutoPlugin {
@@ -102,11 +104,29 @@ object SbtProguard extends AutoPlugin {
     val managedClasspathValue = managedClasspath.value
     val streamsValue = streams.value
     val outputsValue = proguardOutputs.value
+    val proguardOutputJar = (Proguard/proguardOutputs).value.head
     val cachedProguard = FileFunction.cached(streams.value.cacheDirectory / "proguard", FilesInfo.hash) { _ =>
       outputsValue foreach IO.delete
       streamsValue.log.debug("Proguard configuration:")
       proguardOptions.value foreach (streamsValue.log.debug(_))
       runProguard(proguardConfigurationValue, javaOptionsInProguardValue, managedClasspathValue.files, streamsValue.log)
+
+      if (scalaBinaryVersion.value == "3") {
+        streamsValue.log.info("This is a Scala 3 build - will now remove the TASTy files from the ProGuard outputs")
+        val mappingsFile = findMappingsFileConfig(
+          options = (Proguard/proguardOptions).value,
+          baseDir = (Proguard/proguardDirectory).value
+        ).getOrElse(throw new AssertionError(
+          """mappings file not found in proguardOptions. Please configure it using e.g. `-printmapping mapings.txt`
+            | - it must be configured for a Scala 3 build so we can remove the TASTy files for obfuscated classes""".stripMargin
+        ))
+        removeTastyFilesForObfuscatedClasses(
+          mappingsFile,
+          proguardOutputJar = proguardOutputJar,
+          logger = streamsValue.log
+        )
+      }
+
       outputsValue.toSet
     }
     val inputs = (proguardConfiguration.value +: inputFiles(proguardFilteredInputs.value)).toSet
@@ -128,4 +148,72 @@ object SbtProguard extends AutoPlugin {
     val exitCode = Process("java", options) ! log
     if (exitCode != 0) sys.error("Proguard failed with exit code [%s]" format exitCode)
   }
+
+  def removeTastyFilesForObfuscatedClasses(mappingsFile: File, proguardOutputJar: File, logger: ManagedLogger): Unit = {
+    val obfuscatedClasses = findObfuscatedClasses(mappingsFile)
+
+    if (obfuscatedClasses.nonEmpty) {
+      logger.info(s"found ${obfuscatedClasses.size} classes that have been obfuscated; will now remove their TASTy files (bar some that are still required), since those contain even more information than the class files")
+      // note: we must not delete the TASTy files for unobfuscated classes since that would break the REPL
+      val tastyEntriesForObfuscatedClasses = obfuscatedClasses.map { className =>
+        val zipEntry = "/" + className.replaceAll("\\.", "/") // `/` instead of `.`
+        val tastyFileConvention = zipEntry.replaceFirst("\\$.*", "")
+        s"$tastyFileConvention.tasty"
+      }
+
+      val deletedEntries = deleteFromJar(proguardOutputJar, tastyEntriesForObfuscatedClasses)
+      logger.info(s"deleted ${deletedEntries.size} TASTy files from $proguardOutputJar")
+      deletedEntries.foreach(println)
+    }
+  }
+
+  def findMappingsFileConfig(options: Seq[String], baseDir: File): Option[File] = {
+    options.find(_.startsWith("-printmapping")).flatMap { keyValue =>
+      keyValue.split(" ") match {
+        case Array(key, value) => Some(value)
+        case _ =>
+          None
+      }
+    }.map { value =>
+      val mappingsFile = file(value)
+      if (mappingsFile.isAbsolute) mappingsFile
+      else baseDir / value
+    }
+  }
+
+  def findObfuscatedClasses(mappingsFile: File): Set[String] = {
+    // a typical mapping file entry looks like this:
+    // `io.joern.x2cpg.passes.linking.filecompat.FileNameCompat -> io.joern.x2cpg.passes.a.a.a:`
+    val mapping = "(.*) -> (.*):".r
+
+    val classesThatHaveBeenObfuscated = for {
+      line <- Files.lines(mappingsFile.toPath).iterator().asScala
+      // the lines ending with `:` list the classname mappings:
+      if line.endsWith(":")
+      // extract the original and obfuscated name via regex matching:
+      mapping(original, obfuscated) = line
+      // if both sides are identical, this class didn't get obfuscated:
+      if original != obfuscated
+    } yield original
+
+    classesThatHaveBeenObfuscated.toSet
+  }
+
+  /** Deletes all entries from a jar that is in the given set of entry paths.
+    * Returns the Paths of the deleted entries. */
+  def deleteFromJar(jar: File, toDelete: Set[String]): Seq[String] = {
+    val zipFs = FileSystems.newFileSystem(jar.toPath, null: ClassLoader)
+
+    val deletedEntries = for {
+      zipRootDir <- zipFs.getRootDirectories.asScala
+      entry <- Files.walk(zipRootDir).iterator.asScala
+      if (toDelete.contains(entry.toString))
+    } yield {
+      Files.delete(entry)
+      entry.toString
+    }
+
+    zipFs.close()
+    deletedEntries.toSeq
+  }
 }

src/sbt-test/proguard/scala3/build.sbt

@@ -0,0 +1,14 @@
+import java.nio.file.FileSystems
+
+enablePlugins(SbtProguard)
+
+scalaVersion := "3.6.3"
+name := "scala3"
+
+Proguard / proguardOptions ++= Seq("-dontoptimize", "-dontnote", "-dontwarn", "-ignorewarnings")
+Proguard / proguardOptions += ProguardOptions.keepMain("Test")
+Proguard / proguardOptions += ProguardOptions.mappingsFile("mappings.txt")
+
+Proguard / proguardInputs := (Compile / dependencyClasspath).value.files
+
+Proguard / proguardFilteredInputs ++= ProguardOptions.noFilter((Compile / packageBin).value)

src/sbt-test/proguard/scala3/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.github.sbt" % "sbt-proguard" % sys.props("project.version"))

src/sbt-test/proguard/scala3/src/main/scala/Test.scala

@@ -0,0 +1,10 @@
+object Test {
+  def main(args: Array[String]) = {
+    println(ObfuscateMe.foo)
+  }
+}
+
+
+object ObfuscateMe {
+  def foo: String = "test"
+}

src/sbt-test/proguard/scala3/test

@@ -0,0 +1 @@
+> check

src/sbt-test/proguard/scala3/test.sbt

@@ -0,0 +1,30 @@
+import java.nio.file.{Files, FileSystems}
+import scala.jdk.CollectionConverters.*
+import scala.sys.process.Process
+
+// for sbt scripted test:
+TaskKey[Unit]("check") := {
+  val expected = "test\n"
+  val proguardResultJar = (Proguard / proguard).value.head
+  val output = Process("java", Seq("-jar", proguardResultJar.absString)).!!
+    .replaceAllLiterally("\r\n", "\n")
+  if (output != expected) sys.error("Unexpected output:\n" + output)
+
+  // older java releases (e.g. java 11) requires a classloader parameter, which may be null...
+  // for later java release this isn't required any more
+  val zipFs = FileSystems.newFileSystem(proguardResultJar.toPath, null: ClassLoader)
+  val jarEntries = zipFs.getRootDirectories.asScala
+    .flatMap(Files.walk(_).iterator.asScala)
+    .toSeq
+  zipFs.close()
+
+  val obfuscateMeEntries = jarEntries.filter(_.toString.contains("ObfuscateMe"))
+  assert(
+    obfuscateMeEntries.isEmpty,
+    s"""class `ObfuscateMe` should be obfuscated and not appear in the proguard output jar,
+       |neither the class file nor the tasty file. However, we found the following: ${obfuscateMeEntries.mkString(",")}
+       |""".stripMargin
+
+  )
+}
+