Simplify import typings

cli/src/main/scala/org/scalablytyped/converter/cli/Main.scala

@@ -2,6 +2,7 @@ package org.scalablytyped.converter.cli
 
 import com.olvind.logging.{stdout, storing, LogLevel, Logger}
 import fansi.{Attr, Color, Str}
+import org.scalablytyped.converter.internal.ImportTypingsUtil
 import org.scalablytyped.converter.internal.importer._
 import org.scalablytyped.converter.internal.importer.build.{BloopCompiler, PublishedSbtProject, SbtProject}
 import org.scalablytyped.converter.internal.importer.documentation.Npmjs
@@ -15,7 +16,7 @@ import org.scalablytyped.converter.internal.{constants, files, sets, BuildInfo,
 import org.scalablytyped.converter.{Flavour, Selection}
 import scopt.{OParser, OParserBuilder, Read}
 
-import scala.collection.immutable.SortedSet
+import scala.collection.immutable.{SortedMap, SortedSet}
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.Duration
@@ -66,8 +67,8 @@ object Main {
     includeProject = false,
   )
 
-  val parseCachePath = Some(files.existing(constants.defaultCacheFolder / 'parse).toNIO)
-  val t0             = System.currentTimeMillis
+  val parseCacheDirOpt = Some(files.existing(constants.defaultCacheFolder / 'parse).toNIO)
+  val t0               = System.currentTimeMillis
 
   val logger: Logger[(Array[Logger.Stored], Unit)] =
     storing().zipWith(stdout.filter(LogLevel.warn))
@@ -207,9 +208,6 @@ object Main {
 
         val packageJson = Json.force[PackageJson](packageJsonPath)
 
-        val projectSource: Option[LibTsSource.FromFolder] =
-          if (includeProject) Some(LibTsSource.FromFolder(InFolder(inDir), TsIdentLibrary(inDir.last))) else None
-
         val wantedLibs: SortedSet[TsIdentLibrary] =
           libsFromCmdLine match {
             case sets.EmptySet() =>
@@ -221,21 +219,54 @@ object Main {
             case otherwise => otherwise
           }
 
-        val bootstrapped = Bootstrap.fromNodeModules(InFolder(nodeModulesPath), conversion, wantedLibs)
+        /* ImportTypings only uses the library names, not the versions */
+        val wantedLibsEmptyVersion: SortedMap[TsIdentLibrary, String] =
+          SortedMap.empty[TsIdentLibrary, String] ++ (wantedLibs.map(k => (k, "")))
 
-        val sources: Vector[LibTsSource] = {
-          bootstrapped.initialLibs match {
-            case Left(unresolved) => sys.error(unresolved.msg)
-            case Right(initial)   => projectSource.foldLeft(initial)(_ :+ _)
-          }
-        }
+        val publishLocalFolder = constants.defaultLocalPublishFolder
+        val fromFolder         = InFolder(nodeModulesPath)
+        val targetFolder       = c.paths.out
+
+        val input = ImportTypingsUtil.Input(
+          converterVersion = BuildInfo.version,
+          conversion       = conversion,
+          wantedLibs       = wantedLibsEmptyVersion,
+        )
+
+        val logger: Logger[(Array[Logger.Stored], Unit)] =
+          storing().zipWith(stdout.filter(LogLevel.warn))
+
+        val parseCacheDirOpt = Some(
+          files.existing(constants.defaultCacheFolder / "parse").toNIO,
+        )
+
+        val compiler = Await.result(
+          BloopCompiler(logger.filter(LogLevel.debug).void, conversion.versions, failureCacheFolderOpt = None),
+          Duration.Inf,
+        )
+
+        val ensureSourceFilesWritten = true
+
+        val (results, successes, failures) = ImportTypingsUtil.get(
+          input,
+          logger.void,
+          parseCacheDirOpt,
+          publishLocalFolder,
+          fromFolder,
+          targetFolder,
+          compiler,
+          includeProject,
+          ensureSourceFilesWritten,
+        )
 
         println(
           table(fansi.Color.LightBlue)(
             "directory" -> inDir.toString,
             "includeDev" -> includeDev.toString,
             "includeProject" -> includeProject.toString,
-            "wantedLibs" -> sources.map(s => s.libName.value).mkString(", "),
+            // TODO how to resolve this? where to put this table printing?
+            // Bootstrapped is inside ImportTypingsUtil.get and includeDev, for example, is not.
+//            "wantedLibs" -> sources.map(s => s.libName.value).mkString(", "),
             "useScalaJsDomTypes" -> conversion.useScalaJsDomTypes.toString,
             "flavour" -> conversion.flavour.toString,
             "outputPackage" -> conversion.outputPackage.unescaped,
@@ -250,67 +281,9 @@ object Main {
           ),
         )
 
-        val compiler = Await.result(
-          BloopCompiler(logger.filter(LogLevel.debug).void, conversion.versions, failureCacheFolderOpt = None),
-          Duration.Inf,
-        )
-
-        val Pipeline: RecPhase[LibTsSource, PublishedSbtProject] =
-          RecPhase[LibTsSource]
-            .next(
-              new Phase1ReadTypescript(
-                calculateLibraryVersion = PackageJsonOnly,
-                resolve                 = bootstrapped.libraryResolver,
-                ignored                 = conversion.ignoredLibs,
-                ignoredModulePrefixes   = conversion.ignoredModulePrefixes,
-                pedantic                = false,
-                parser                  = PersistingParser(parseCachePath, bootstrapped.inputFolders, logger.void),
-                expandTypeMappings      = conversion.expandTypeMappings,
-              ),
-              "typescript",
-            )
-            .next(
-              new Phase2ToScalaJs(
-                pedantic                 = false,
-                scalaVersion             = conversion.versions.scala,
-                enableScalaJsDefined     = conversion.enableScalaJsDefined,
-                outputPkg                = conversion.outputPackage,
-                flavour                  = conversion.flavourImpl,
-                useDeprecatedModuleNames = conversion.useDeprecatedModuleNames,
-              ),
-              "scala.js",
-            )
-            .next(
-              new PhaseFlavour(conversion.flavourImpl, maybePrivateWithin = conversion.privateWithin),
-              conversion.flavourImpl.toString,
-            )
-            .next(
-              new Phase3Compile(
-                versions                   = conversion.versions,
-                compiler                   = compiler,
-                targetFolder               = c.paths.out,
-                organization               = conversion.organization,
-                publishLocalFolder         = constants.defaultLocalPublishFolder,
-                metadataFetcher            = Npmjs.No,
-                softWrites                 = true,
-                flavour                    = conversion.flavourImpl,
-                generateScalaJsBundlerFile = false,
-                ensureSourceFilesWritten   = true,
-              ),
-              "build",
-            )
-
-        val results: Map[LibTsSource, PhaseRes[LibTsSource, PublishedSbtProject]] =
-          sources
-            .map(source => source -> PhaseRunner(Pipeline, (_: LibTsSource) => logger.void, NoListener)(source))
-            .toMap
-
         val td = System.currentTimeMillis - t0
         logger.warn(td)
 
-        val failures: Map[LibTsSource, Either[Throwable, String]] =
-          results.collect { case (_, PhaseRes.Failure(errors)) => errors }.reduceOption(_ ++ _).getOrElse(Map.empty)
-
         if (failures.nonEmpty) {
           println(
             Color.Red(s"Failure: You might try --ignoredLibs ${failures.keys.map(_.libName.value).mkString(", ")}"),
@@ -326,13 +299,6 @@ object Main {
 
           1
         } else {
-          val allSuccesses: Map[LibTsSource, PublishedSbtProject] = {
-            def go(source: LibTsSource, p: PublishedSbtProject): Map[LibTsSource, PublishedSbtProject] =
-              Map(source -> p) ++ p.project.deps.flatMap { case (k, v) => go(k, v) }
-
-            results.collect { case (s, PhaseRes.Ok(res)) => go(s, res) }.reduceOption(_ ++ _).getOrElse(Map.empty)
-          }
-
           val short: Seq[SbtProject] =
             results
               .collect { case (_, PhaseRes.Ok(res)) => res.project }
@@ -341,7 +307,7 @@ object Main {
               .sortBy(_.name)
 
           println()
-          println(s"Successfully converted ${allSuccesses.keys.map(x => Color.Green(x.libName.value)).mkString(", ")}")
+          println(s"Successfully converted ${successes.keys.map(x => Color.Green(x.libName.value)).mkString(", ")}")
           println("To use in sbt: ")
           println(Color.DarkGray(s"""libraryDependencies ++= Seq(
                |  ${short.map(p => p.reference.asSbt).mkString("", ",\n  ", "")}

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ImportTypingsUtil.scala

@@ -0,0 +1,137 @@
+package org.scalablytyped.converter
+package internal
+
+import java.nio.file.Path
+
+import com.olvind.logging.Logger
+import io.circe013.{Decoder, Encoder}
+import org.scalablytyped.converter.internal.importer._
+import org.scalablytyped.converter.internal.importer.build.{Compiler, IvyLayout, PublishedSbtProject}
+import org.scalablytyped.converter.internal.importer.documentation.Npmjs
+import org.scalablytyped.converter.internal.maps._
+import org.scalablytyped.converter.internal.phases.{PhaseListener, PhaseRes, PhaseRunner, RecPhase}
+import org.scalablytyped.converter.internal.scalajs.Dep
+import org.scalablytyped.converter.internal.ts._
+import os.RelPath
+
+import scala.collection.immutable.SortedMap
+
+object ImportTypingsUtil {
+  case class Input(
+      converterVersion: String,
+      conversion:       ConversionOptions,
+      wantedLibs:       SortedMap[TsIdentLibrary, String],
+  ) {
+    lazy val packageJsonHash: String =
+      Digest.of(IArray.fromTraversable(wantedLibs.map { case (name, v) => s"${name.value} $v" })).hexString
+  }
+
+  object Input {
+    implicit val encodes: Encoder[Input] = io.circe013.generic.semiauto.deriveEncoder
+    implicit val decodes: Decoder[Input] = io.circe013.generic.semiauto.deriveDecoder
+  }
+
+  type Results   = Map[LibTsSource, PhaseRes[LibTsSource, PublishedSbtProject]]
+  type Successes = Map[LibTsSource, PublishedSbtProject]
+  type Failures  = Map[LibTsSource, Either[Throwable, String]]
+
+  def get(
+      input:                    Input,
+      logger:                   Logger[Unit],
+      parseCacheDirOpt:         Option[Path],
+      publishLocalFolder:       os.Path,
+      fromFolder:               InFolder,
+      targetFolder:             os.Path,
+      compiler:                 Compiler,
+      includeProject:           Boolean = false,
+      ensureSourceFilesWritten: Boolean = false,
+  ): (Results, Successes, Failures) = {
+    if (input.conversion.expandTypeMappings =/= EnabledTypeMappingExpansion.DefaultSelection) {
+      logger.warn("Changing stInternalExpandTypeMappings not encouraged. It might blow up")
+    }
+
+    if (input.conversion.enableLongApplyMethod) {
+      logger.warn("enableLongApplyMethod is deprecated and untested. You should migrate to the builder pattern.")
+    }
+
+    val bootstrapped = Bootstrap.fromNodeModules(fromFolder, input.conversion, input.wantedLibs.keySet)
+
+    val inDir = fromFolder.path / os.up
+
+    val projectSource: Option[LibTsSource.FromFolder] =
+      if (includeProject) Some(LibTsSource.FromFolder(InFolder(inDir), TsIdentLibrary(inDir.last))) else None
+
+    val sources: Vector[LibTsSource] = {
+      bootstrapped.initialLibs match {
+        case Left(unresolved) => sys.error(unresolved.msg)
+        case Right(initial)   => projectSource.foldLeft(initial)(_ :+ _)
+      }
+    }
+
+    logger.info(s"Importing ${sources.map(_.libName.value).mkString(", ")}")
+
+    val cachedParser = PersistingParser(parseCacheDirOpt, bootstrapped.inputFolders, logger)
+
+    val Phases: RecPhase[LibTsSource, PublishedSbtProject] = RecPhase[LibTsSource]
+      .next(
+        new Phase1ReadTypescript(
+          resolve                 = bootstrapped.libraryResolver,
+          calculateLibraryVersion = CalculateLibraryVersion.PackageJsonOnly,
+          ignored                 = input.conversion.ignoredLibs,
+          ignoredModulePrefixes   = input.conversion.ignoredModulePrefixes,
+          pedantic                = false,
+          parser                  = cachedParser,
+          expandTypeMappings      = input.conversion.expandTypeMappings,
+        ),
+        "typescript",
+      )
+      .next(
+        new Phase2ToScalaJs(
+          pedantic                 = false,
+          scalaVersion             = input.conversion.versions.scala,
+          enableScalaJsDefined     = input.conversion.enableScalaJsDefined,
+          outputPkg                = input.conversion.outputPackage,
+          flavour                  = input.conversion.flavourImpl,
+          useDeprecatedModuleNames = input.conversion.useDeprecatedModuleNames,
+        ),
+        "scala.js",
+      )
+      .next(
+        new PhaseFlavour(input.conversion.flavourImpl, maybePrivateWithin = input.conversion.privateWithin),
+        input.conversion.flavourImpl.toString,
+      )
+      .next(
+        new Phase3Compile(
+          versions                   = input.conversion.versions,
+          compiler                   = compiler,
+          targetFolder               = targetFolder,
+          organization               = input.conversion.organization,
+          publishLocalFolder         = publishLocalFolder,
+          metadataFetcher            = Npmjs.No,
+          softWrites                 = true,
+          flavour                    = input.conversion.flavourImpl,
+          generateScalaJsBundlerFile = false,
+          ensureSourceFilesWritten   = false,
+        ),
+        "build",
+      )
+
+    val results: Results =
+      sources
+        .map(s => (s: LibTsSource) -> PhaseRunner(Phases, (_: LibTsSource) => logger, PhaseListener.NoListener)(s))
+        .toMap
+        .toSorted
+
+    val successes: Successes = {
+      def go(source: LibTsSource, p: PublishedSbtProject): Successes =
+        Map(source -> p) ++ p.project.deps.flatMap { case (k, v) => go(k, v) }
+
+      results.collect { case (s, PhaseRes.Ok(res)) => go(s, res) }.reduceOption(_ ++ _).getOrElse(Map.empty)
+    }
+
+    val failures: Failures =
+      results.collect { case (_, PhaseRes.Failure(errors)) => errors }.reduceOption(_ ++ _).getOrElse(Map.empty)
+
+    (results, successes, failures)
+  }
+}