parser: remove carriage returns

ScalablyTyped · Nov 15, 2022 · 0e70c00 · 0e70c00
1 parent 3005dd1
commit 0e70c00
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 26 deletions.
diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala
@@ -1,20 +1,17 @@
 package org.scalablytyped.converter.internal
 package ts
 
-import java.io.StringWriter
-
 import com.olvind.logging
 import com.olvind.logging.Logger
 import org.scalablytyped.converter.Selection
 import org.scalablytyped.converter.internal.importer.Phase1ReadTypescript
 import org.scalablytyped.converter.internal.ts.parser.TsParser
 import org.scalablytyped.converter.internal.ts.transforms.SetCodePath
-
+import org.scalablytyped.converter.internal.ts.parser.ParserHarness._
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
 trait TypeExpansionHarness {
-  val parser  = new TsParser(None)
   val libName = TsIdentLibrarySimple("testing")
 
   def Transformations(logger: Logger[Unit]): List[TsParsedFile => TsParsedFile] =
@@ -26,7 +23,7 @@ trait TypeExpansionHarness {
     )
 
   def run(input: String): TsParsedFile = {
-    val parsed       = parser(input).get
+    val parsed       = TsParser.parsedTsFile(input).get
     val logger       = logging.stringWriter()
     val withCodePath = SetCodePath.visitTsParsedFile(CodePath.HasPath(libName, TsQIdent.empty))(parsed)
 

diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala
@@ -22,7 +22,7 @@ final class CommentTests extends AnyFunSuite with Matchers {
   test("line comment") {
     val content = " //asdasdasd\n"
     shouldParseAs(content, TsParser.lexical.comment)(
-      TsParser.lexical.CommentLineToken(content),
+      TsParser.lexical.CommentLineToken("//asdasdasd\n"),
     )
   }
 
@@ -71,7 +71,8 @@ final class CommentTests extends AnyFunSuite with Matchers {
       )
 
     value.comments.cs.zip(expecteds.cs).foreach {
-      case (actual, expected) => actual should equal(expected)
+      case (actual, expected) =>
+        actual should equal(expected)
     }
   }
 
@@ -401,7 +402,7 @@ declare const NaN: number;
 declare const Infinity: number;
 """
 
-    val forced: TsParsedFile = TsParser(content).force
+    val forced: TsParsedFile = TsParser.parsedTsFile(content).force
     assert(forced.directives.length == 1)
   }
 

diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala
@@ -5,6 +5,7 @@ import org.scalatest.Assertion
 import org.scalatest.matchers.should.Matchers._
 
 import scala.util.parsing.combinator.Parsers
+import scala.util.parsing.input.CharSequenceReader
 
 object ParserHarness {
   implicit final class Forcer[T](res: Parsers#ParseResult[T]) {
@@ -18,6 +19,19 @@ object ParserHarness {
       )
   }
 
+  implicit def UseLexerParser[T](p: TsLexer.Parser[T]): String => Parsers#ParseResult[T] =
+    (str: String) => {
+      val str1 = cleanedString(str)
+      p(new CharSequenceReader(str1))
+    }
+
+  /** enable direct use of parsers with strings * */
+  implicit def UseParserParser[T](p: TsParser.Parser[T]): String => Parsers#ParseResult[T] =
+    (str: String) => {
+      val str1 = cleanedString(str)
+      TsParser.phrase(p)(new TsParser.lexical.Scanner(str1))
+    }
+
   def withTsFile[T](resourceName: String)(f: String => T): T =
     f(files.content(InFile(os.Path(getClass.getResource(s"/$resourceName").getFile))))
 

diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala
@@ -31,8 +31,6 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo
         }
         .mkString("")
   }
-  implicit def FromString[T](p: Parser[T]): String => ParseResult[T] =
-    (str: String) => p.apply(new CharSequenceReader(str))
 
   // format: off
 
@@ -161,12 +159,12 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo
   }
 
   private val newLine: Parser[Char] =
-    ('\r'.? ~> '\n') | ('\r') ^^ (_ => '\n')
+    '\n'
 
   /** A character-parser that matches a white-space character (and returns it).
     * We dont ignore newlines */
   override val whitespaceChar: Parser[Char] = {
-    elem("space char", ch => ch <= ' ' && ch != EofCh && ch != '\n' && ch != '\r')
+    elem("space char", ch => ch <= ' ' && ch != EofCh && ch != '\n')
   }
 
   override val whitespace: Parser[Any] =
@@ -190,7 +188,7 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo
 
   val comment: Parser[CommentToken] = {
     val oneLine: Parser[CommentLineToken] =
-      (whitespaceChar.* ~ '/' ~ '/' ~ rep(chrExcept(EofCh, '\n', '\r'))) ^^ {
+      (whitespaceChar.* ~ '/' ~ '/' ~ rep(chrExcept(EofCh, '\n'))) ^^ {
         case cs1 ~ c1 ~ c2 ~ cs2 =>
           CommentLineToken(s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}\n")
       }
@@ -199,8 +197,7 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo
       (whitespaceChar.* ~ '/' ~ '*' ~ rep(not('*' ~ '/') ~> chrExcept(EofCh)) ~ '*' ~ '/' ~ whitespaceChar.* ~ newLine.*) ^^ {
         case cs1 ~ c1 ~ c2 ~ cs2 ~ c3 ~ c4 ~ cs3 ~ cs4 =>
           CommentBlockToken(
-            s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}$c3$c4${chars2string(cs3)}${chars2string(cs4)}"
-              .replaceAll("\r", ""),
+            s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}$c3$c4${chars2string(cs3)}${chars2string(cs4)}",
           )
       }
 

diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala
@@ -21,10 +21,6 @@ class TsParser(path: Option[(os.Path, Int)]) extends StdTokenParsers with Parser
   protected def perhapsParens[T](p: Parser[T]): Parser[T] =
     p | ("(" ~> p <~ ")")
 
-  /** enable direct use of parsers with strings **/
-  implicit def FromString[T](p: Parser[T]): String => ParseResult[T] =
-    (str: String) => phrase(p)(new lexical.Scanner(str.trim))
-
   def memo[P](p: Parser[P]): Parser[P] =
     path match {
       case Some((_, length)) =>
@@ -58,8 +54,8 @@ class TsParser(path: Option[(os.Path, Int)]) extends StdTokenParsers with Parser
       }
     }
 
-  def apply(content: String): ParseResult[TsParsedFile] =
-    parsedTsFile(content)
+//  def apply(content: String): ParseResult[TsParsedFile] =
+//    parsedTsFile(content)
 
   /** handle stray comments **/
   override def Parser[T](f: Input => ParseResult[T]): Parser[T] =

diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala
@@ -2,15 +2,21 @@ package org.scalablytyped.converter.internal
 package ts
 
 package object parser {
+  val BOM = "\uFEFF"
+
+  def cleanedString(s1: String): String = {
+    val s2 = if (s1.startsWith(BOM)) s1.replace(BOM, "") else s1
+    val s3 = s2.replace("\r\n", "\n").trim
+    s3
+  }
+
   def parseFile(inFile: InFile): Either[String, TsParsedFile] = parseFileContent(inFile, os.read.bytes(inFile.path))
 
   def parseFileContent(inFile: InFile, bytes: Array[Byte]): Either[String, TsParsedFile] = {
-    val BOM = "\uFEFF"
-    val s1  = new String(bytes, constants.Utf8)
-    val s2  = if (s1.startsWith(BOM)) s1.replace(BOM, "") else s1
-    val p   = new TsParser(Some((inFile.path, s2.length)))
+    val str = cleanedString(new String(bytes, constants.Utf8))
+    val p   = new TsParser(Some((inFile.path, str.length)))
 
-    p(s2) match {
+    p.phrase(p.parsedTsFile)(new TsParser.lexical.Scanner(str)) match {
       case p.Success(t, _) =>
         Right(t)