consistency: work on strings longer, convert to bytes only on write.

this allows us to pick the whitespace-insensitive digesting. Hopefully this will negate platform newline differences

core/src/main/scala/org/scalablytyped/converter/internal/files.scala

@@ -54,7 +54,7 @@ object files {
   }
 
   def syncAbs(
-      absolutePathFiles: IArray[(os.Path, Array[Byte])],
+      absolutePathFiles: IArray[(os.Path, String)],
       folder:            os.Path,
       deleteUnknowns:    Boolean,
       soft:              Boolean,
@@ -74,11 +74,13 @@ object files {
 
     absolutePathFiles.foreach {
       case (file, content) =>
-        if (soft) softWriteBytes(file.toNIO, content) else writeBytes(file.toNIO, content)
+        val bytes = content.getBytes(constants.Utf8)
+        if (soft) softWriteBytes(file.toNIO, bytes) else writeBytes(file.toNIO, bytes)
+        ()
     }
   }
 
-  def sync(fs: IArray[(os.RelPath, Array[Byte])], folder: os.Path, deleteUnknowns: Boolean, soft: Boolean): Unit =
+  def sync(fs: IArray[(os.RelPath, String)], folder: os.Path, deleteUnknowns: Boolean, soft: Boolean): Unit =
     syncAbs(fs.map { case (relPath, content) => folder / relPath -> content }, folder, deleteUnknowns, soft)
 
   def softWrite[T](path: os.Path)(f: PrintWriter => T): Synced =

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Phase3Compile.scala

@@ -1,8 +1,6 @@
 package org.scalablytyped.converter.internal
 package importer
 
-import java.time.ZonedDateTime
-
 import com.olvind.logging.{Formatter, Logger}
 import org.scalablytyped.converter.internal.importer.build._
 import org.scalablytyped.converter.internal.importer.documentation.Npmjs
@@ -11,6 +9,7 @@ import org.scalablytyped.converter.internal.phases.{GetDeps, IsCircular, Phase,
 import org.scalablytyped.converter.internal.scalajs._
 import org.scalablytyped.converter.internal.scalajs.flavours.FlavourImpl
 
+import java.time.ZonedDateTime
 import scala.collection.immutable.SortedSet
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -32,7 +31,7 @@ class Phase3Compile(
     ensureSourceFilesWritten:   Boolean,
 ) extends Phase[LibTsSource, LibScalaJs, PublishedSbtProject] {
 
-  val ScalaFiles: PartialFunction[(os.RelPath, Array[Byte]), Array[Byte]] = {
+  val ScalaFiles: PartialFunction[(os.RelPath, String), String] = {
     case (path, value)
         if path.ext === "scala" ||
           path.ext === "sbt" ||
@@ -66,23 +65,24 @@ class Phase3Compile(
         val resourcesDir  = os.RelPath("src") / 'main / 'resources
         val metadataOpt   = Try(Await.result(metadataFetcher(lib.source, logger), 2.seconds)).toOption.flatten
         val compilerPaths = CompilerPaths.of(versions, targetFolder, lib.libName)
-        val resources: IArray[(os.RelPath, Array[Byte])] =
+        val resources: IArray[(os.RelPath, String)] =
           if (generateScalaJsBundlerFile) ScalaJsBundlerDepFile(lib.source.libName, lib.libVersion)
           else Empty
 
-        val sbtLayout = ContentSbtProject(
-          versions        = versions,
-          comments        = lib.packageTree.comments,
-          organization    = organization,
-          name            = lib.libName,
-          version         = VersionHack.TemplateValue,
-          localDeps       = deps.toIArrayValues,
-          deps            = flavour.dependencies,
-          scalaFiles      = scalaFiles.map { case (relPath, content) => sourcesDir / relPath -> content },
-          resources       = resources.map { case (relPath, content) => resourcesDir / relPath -> content },
-          metadataOpt     = metadataOpt,
-          declaredVersion = Some(lib.libVersion),
-        )
+        val sbtLayout: SbtProjectLayout[os.RelPath, String] =
+          ContentSbtProject(
+            versions        = versions,
+            comments        = lib.packageTree.comments,
+            organization    = organization,
+            name            = lib.libName,
+            version         = VersionHack.TemplateValue,
+            localDeps       = deps.toIArrayValues,
+            deps            = flavour.dependencies,
+            scalaFiles      = scalaFiles.map { case (relPath, content) => sourcesDir / relPath -> content },
+            resources       = resources.map { case (relPath, content) => resourcesDir / relPath -> content },
+            metadataOpt     = metadataOpt,
+            declaredVersion = Some(lib.libVersion),
+          )
 
         val digest                = Digest.of(sbtLayout.all.collect(ScalaFiles))
         val finalVersion          = lib.libVersion.version(digest)

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ScalaJsBundlerDepFile.scala

@@ -24,7 +24,7 @@ object ScalaJsBundlerDepFile {
     implicit val Decoder: Decoder[NpmDependencies] = io.circe013.generic.semiauto.deriveDecoder
   }
 
-  def apply(libName: TsIdentLibrary, v: LibraryVersion): IArray[(os.RelPath, Array[Byte])] =
+  def apply(libName: TsIdentLibrary, v: LibraryVersion): IArray[(os.RelPath, String)] =
     (v.libraryVersion, v.inGit) match {
       case (Some(version), None) if libName =/= ts.TsIdent.std =>
         val deps = List(Map(libName.value -> version))
@@ -35,7 +35,7 @@ object ScalaJsBundlerDepFile {
               `test-dependencies`       = deps,
               `compile-devDependencies` = Nil,
               `test-devDependencies`    = Nil,
-            ).asJson.spaces2.getBytes(constants.Utf8),
+            ).asJson.spaces2,
         )
       case _ => Empty
     }

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/build/ContentSbtProject.scala

@@ -15,11 +15,11 @@ object ContentSbtProject {
       version:         String,
       localDeps:       IArray[PublishedSbtProject],
       deps:            Set[Dep],
-      scalaFiles:      IArray[(os.RelPath, Array[Byte])],
-      resources:       IArray[(os.RelPath, Array[Byte])],
+      scalaFiles:      IArray[(os.RelPath, String)],
+      resources:       IArray[(os.RelPath, String)],
       metadataOpt:     Option[Npmjs.Data],
       declaredVersion: Option[LibraryVersion],
-  ): SbtProjectLayout[os.RelPath, Array[Byte]] = {
+  ): SbtProjectLayout[os.RelPath, String] = {
 
     val buildSbt = {
       val allDeps: IArray[Dep] = IArray.fromTraversable(deps) ++ IArray(versions.runtime) ++ localDeps.map(d =>
@@ -43,13 +43,13 @@ object ContentSbtProject {
       .map(dep => s"addSbtPlugin(${dep.asSbt})")
       .mkString("", "\n", "\n")
 
-    val readme: (os.RelPath, Array[Byte]) =
-      os.RelPath("readme.md") -> ProjectReadme(name, declaredVersion, metadataOpt, comments).getBytes(constants.Utf8)
+    val readme: (os.RelPath, String) =
+      os.RelPath("readme.md") -> ProjectReadme(name, declaredVersion, metadataOpt, comments)
 
     SbtProjectLayout(
-      os.RelPath("build.sbt") -> buildSbt.getBytes(constants.Utf8),
-      os.RelPath("project") / "build.properties" -> s"sbt.version=${Versions.sbtVersion}".getBytes(constants.Utf8),
-      os.RelPath("project") / "plugins.sbt" -> pluginsSbt.getBytes(constants.Utf8),
+      os.RelPath("build.sbt") -> buildSbt,
+      os.RelPath("project") / "build.properties" -> s"sbt.version=${Versions.sbtVersion}",
+      os.RelPath("project") / "plugins.sbt" -> pluginsSbt,
       readme,
       scalaFiles,
       resources,

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/build/VersionHack.scala

@@ -8,9 +8,9 @@ package build
 object VersionHack {
   val TemplateValue = "$$$___VERSION___$$$"
 
-  def templateVersion[K](out: SbtProjectLayout[K, Array[Byte]], version: String): SbtProjectLayout[K, Array[Byte]] = {
+  def templateVersion[K](out: SbtProjectLayout[K, String], version: String): SbtProjectLayout[K, String] = {
     val (key, content) = out.buildSbt
-    val patched        = new String(content, constants.Utf8).replace(TemplateValue, version)
-    out.copy(buildSbt = key -> patched.getBytes(constants.Utf8))
+    val patched        = content.replace(TemplateValue, version)
+    out.copy(buildSbt = key -> patched)
   }
 }

importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala

@@ -366,7 +366,6 @@ target/
 
       GenerateSbtPlugin(
         isDeprecated  = true,
-        versions      = config.conversion.versions,
         organization  = config.conversion.organization,
         projectName   = config.projectName,
         projectDir    = sbtProjectDir,

importer/src/main/scala/org/scalablytyped/converter/internal/importer/build/GenerateSbtPlugin.scala

@@ -9,7 +9,6 @@ import org.scalablytyped.converter.internal.stringUtils.quote
 object GenerateSbtPlugin {
   def apply(
       isDeprecated:  Boolean,
-      versions:      Versions,
       organization:  String,
       projectName:   ProjectName,
       projectDir:    os.Path,
@@ -39,7 +38,7 @@ object GenerateSbtPlugin {
       projectName:   ProjectName,
       projects:      Set[PublishedSbtProject],
       pluginVersion: String,
-  ): IArray[(os.RelPath, Array[Byte])] = {
+  ): IArray[(os.RelPath, String)] = {
 
     val buildSbt = s"""name := "sbt-${projectName.value}"
       |organization := ${quote(organization)}
@@ -104,8 +103,6 @@ object GenerateSbtPlugin {
       os.RelPath("build.sbt") -> buildSbt,
       os.RelPath("project") / "build.properties" -> s"sbt.version=${Versions.sbtVersion}",
       pluginSourcePath -> pluginSource,
-    ).map {
-      case (relPath, str) => relPath -> str.getBytes(constants.Utf8)
-    }
+    )
   }
 }

sbt-converter/src/main/scala/org/scalablytyped/converter/internal/ImportTypingsGenSources.scala

@@ -114,7 +114,7 @@ object ImportTypingsGenSources {
           Minimization.findReferences(globalScope, input.minimizeKeep, packagesWithShouldMinimize)
         }
 
-        val outFiles: IArray[(os.Path, Array[Byte])] = {
+        val outFiles: IArray[(os.Path, String)] = {
           IArray.fromTraversable {
             libs.par.flatMap {
               case (source, lib) =>

scalajs/src/main/scala/org/scalablytyped/converter/internal/scalajs/Printer.scala

@@ -1,12 +1,10 @@
 package org.scalablytyped.converter.internal
 package scalajs
 
-import java.io._
-
 import org.scalablytyped.converter.internal.scalajs.transforms.ShortenNames
 import org.scalablytyped.converter.internal.stringUtils.quote
 
-import scala.collection.mutable
+import java.io._
 
 object debugPrinter extends Printer.Impl(Name(""), Versions.Scala3)
 
@@ -15,21 +13,21 @@ object Printer {
 
   private[Printer] class Registry() {
 
-    private val files = IArray.Builder.empty[(os.RelPath, Array[Byte])]
+    private val files = IArray.Builder.empty[(os.RelPath, String)]
 
     def write(file: os.RelPath)(f: PrintWriter => Unit): Unit = {
       val w  = new StringWriter()
       val pw = new PrintWriter(w)
 
       try {
         f(pw)
-        files += ((file, w.toString.getBytes(constants.Utf8)))
+        files += ((file, w.toString))
       } finally {
         pw.close()
       }
     }
 
-    def result: IArray[(os.RelPath, Array[Byte])] =
+    def result: IArray[(os.RelPath, String)] =
       files.result()
   }
 
@@ -39,7 +37,7 @@ object Printer {
       tree:            ContainerTree,
       outputPackage:   Name,
       scalaVersion:    Versions.Scala,
-  ): IArray[(os.RelPath, Array[Byte])] = {
+  ): IArray[(os.RelPath, String)] = {
     val reg = new Registry()
 
     new Impl(outputPackage, scalaVersion).apply(

tests/antd/check-3/a/antd/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "antd"
-version := "4.3.1-67bb52"
+version := "4.3.1-c39d19"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/antd/check-3/r/rc-field-form/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "rc-field-form"
-version := "1.4.4-b244b9"
+version := "1.4.4-31770c"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/augment-module/check-3/l/lodash/build.sbt

@@ -1,11 +1,11 @@
 organization := "org.scalablytyped"
 name := "lodash"
-version := "4.14-198769"
+version := "4.14-1362a8"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(
   "com.olvind" %%% "scalablytyped-runtime" % "2.4.2",
-  "org.scalablytyped" %%% "std" % "0.0-unknown-96e918")
+  "org.scalablytyped" %%% "std" % "0.0-unknown-7b3e8b")
 publishArtifact in packageDoc := false
 scalacOptions ++= List("-encoding", "utf-8", "-feature", "-language:implicitConversions", "-language:higherKinds", "-language:existentials", "-no-indent", "-source:future")
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))

tests/augment-module/check-3/l/lodash_dot_add/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "lodash_dot_add"
-version := "3.7-9ecd5b"
+version := "3.7-f5f013"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/augment-module/check-3/s/std/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "std"
-version := "0.0-unknown-96e918"
+version := "0.0-unknown-7b3e8b"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/aws-sdk/check-3/a/aws-sdk/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "aws-sdk"
-version := "2.247.1-3c4d63"
+version := "2.247.1-04039c"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/babylon/check-3/b/babylon/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "babylon"
-version := "0.0-unknown-cfc667"
+version := "0.0-unknown-14539c"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/babylon/check-3/n/node/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "node"
-version := "0.0-unknown-e17b3d"
+version := "0.0-unknown-4cf870"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/bigint/check-3/b/bigint/build.sbt

@@ -1,11 +1,11 @@
 organization := "org.scalablytyped"
 name := "bigint"
-version := "v5.5.3-91e2a1"
+version := "v5.5.3-4e71ee"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(
   "com.olvind" %%% "scalablytyped-runtime" % "2.4.2",
-  "org.scalablytyped" %%% "std" % "0.0-unknown-20c883")
+  "org.scalablytyped" %%% "std" % "0.0-unknown-2d9f14")
 publishArtifact in packageDoc := false
 scalacOptions ++= List("-encoding", "utf-8", "-feature", "-language:implicitConversions", "-language:higherKinds", "-language:existentials", "-no-indent", "-source:future")
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))

tests/bigint/check-3/s/std/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "std"
-version := "0.0-unknown-20c883"
+version := "0.0-unknown-2d9f14"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/chart.js/check-3/c/chart_dot_js/build.sbt

@@ -1,11 +1,11 @@
 organization := "org.scalablytyped"
 name := "chart_dot_js"
-version := "0.0-unknown-51ecc6"
+version := "0.0-unknown-0151aa"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(
   "com.olvind" %%% "scalablytyped-runtime" % "2.4.2",
-  "org.scalablytyped" %%% "std" % "0.0-unknown-ff20c0")
+  "org.scalablytyped" %%% "std" % "0.0-unknown-c56b98")
 publishArtifact in packageDoc := false
 scalacOptions ++= List("-encoding", "utf-8", "-feature", "-language:implicitConversions", "-language:higherKinds", "-language:existentials", "-no-indent", "-source:future")
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))

tests/chart.js/check-3/s/std/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "std"
-version := "0.0-unknown-ff20c0"
+version := "0.0-unknown-c56b98"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/cldrjs/check-3/c/cldrjs/build.sbt

@@ -1,11 +1,11 @@
 organization := "org.scalablytyped"
 name := "cldrjs"
-version := "0.4.4-ada9d5"
+version := "0.4.4-93ce29"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(
   "com.olvind" %%% "scalablytyped-runtime" % "2.4.2",
-  "org.scalablytyped" %%% "std" % "0.0-unknown-207f5d")
+  "org.scalablytyped" %%% "std" % "0.0-unknown-43cef5")
 publishArtifact in packageDoc := false
 scalacOptions ++= List("-encoding", "utf-8", "-feature", "-language:implicitConversions", "-language:higherKinds", "-language:existentials", "-no-indent", "-source:future")
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))

tests/cldrjs/check-3/s/std/build.sbt

@@ -1,6 +1,6 @@
 organization := "org.scalablytyped"
 name := "std"
-version := "0.0-unknown-207f5d"
+version := "0.0-unknown-43cef5"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(

tests/commander/check-3/c/commander/build.sbt

@@ -1,12 +1,12 @@
 organization := "org.scalablytyped"
 name := "commander"
-version := "2.15.1-598b7c"
+version := "2.15.1-b8209e"
 scalaVersion := "3.2.0"
 enablePlugins(ScalaJSPlugin)
 libraryDependencies ++= Seq(
   "com.olvind" %%% "scalablytyped-runtime" % "2.4.2",
-  "org.scalablytyped" %%% "node" % "0.0-unknown-441446",
-  "org.scalablytyped" %%% "std" % "0.0-unknown-b67ad2")
+  "org.scalablytyped" %%% "node" % "0.0-unknown-654197",
+  "org.scalablytyped" %%% "std" % "0.0-unknown-065f9f")
 publishArtifact in packageDoc := false
 scalacOptions ++= List("-encoding", "utf-8", "-feature", "-language:implicitConversions", "-language:higherKinds", "-language:existentials", "-no-indent", "-source:future")
 licenses += ("MIT", url("http://opensource.org/licenses/MIT"))