From 0e70c00220baf94c789d49edeffc392df958224f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Raddum=20Berg?= Date: Tue, 15 Nov 2022 02:36:30 +0100 Subject: [PATCH] parser: remove carriage returns --- .../internal/ts/TypeExpansionHarness.scala | 7 ++----- .../internal/ts/parser/CommentTests.scala | 7 ++++--- .../internal/ts/parser/ParserHarness.scala | 14 ++++++++++++++ .../converter/internal/ts/parser/TsLexer.scala | 11 ++++------- .../converter/internal/ts/parser/TsParser.scala | 8 ++------ .../converter/internal/ts/parser/package.scala | 16 +++++++++++----- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala index 4f312292ee..32d1a283ba 100644 --- a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala +++ b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/TypeExpansionHarness.scala @@ -1,20 +1,17 @@ package org.scalablytyped.converter.internal package ts -import java.io.StringWriter - import com.olvind.logging import com.olvind.logging.Logger import org.scalablytyped.converter.Selection import org.scalablytyped.converter.internal.importer.Phase1ReadTypescript import org.scalablytyped.converter.internal.ts.parser.TsParser import org.scalablytyped.converter.internal.ts.transforms.SetCodePath - +import org.scalablytyped.converter.internal.ts.parser.ParserHarness._ import scala.reflect.ClassTag import scala.util.control.NonFatal trait TypeExpansionHarness { - val parser = new TsParser(None) val libName = TsIdentLibrarySimple("testing") def Transformations(logger: Logger[Unit]): List[TsParsedFile => TsParsedFile] = @@ -26,7 +23,7 @@ trait TypeExpansionHarness { ) def run(input: String): TsParsedFile = { - val parsed = parser(input).get + val parsed = TsParser.parsedTsFile(input).get val logger = logging.stringWriter() val withCodePath = SetCodePath.visitTsParsedFile(CodePath.HasPath(libName, TsQIdent.empty))(parsed) diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala index e7d42fd2ce..da0add3d6d 100644 --- a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala +++ b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/CommentTests.scala @@ -22,7 +22,7 @@ final class CommentTests extends AnyFunSuite with Matchers { test("line comment") { val content = " //asdasdasd\n" shouldParseAs(content, TsParser.lexical.comment)( - TsParser.lexical.CommentLineToken(content), + TsParser.lexical.CommentLineToken("//asdasdasd\n"), ) } @@ -71,7 +71,8 @@ final class CommentTests extends AnyFunSuite with Matchers { ) value.comments.cs.zip(expecteds.cs).foreach { - case (actual, expected) => actual should equal(expected) + case (actual, expected) => + actual should equal(expected) } } @@ -401,7 +402,7 @@ declare const NaN: number; declare const Infinity: number; """ - val forced: TsParsedFile = TsParser(content).force + val forced: TsParsedFile = TsParser.parsedTsFile(content).force assert(forced.directives.length == 1) } diff --git a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala index a94840633e..b1b86efd48 100644 --- a/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala +++ b/importer/src/test/scala/org/scalablytyped/converter/internal/ts/parser/ParserHarness.scala @@ -5,6 +5,7 @@ import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers._ import scala.util.parsing.combinator.Parsers +import scala.util.parsing.input.CharSequenceReader object ParserHarness { implicit final class Forcer[T](res: Parsers#ParseResult[T]) { @@ -18,6 +19,19 @@ object ParserHarness { ) } + implicit def UseLexerParser[T](p: TsLexer.Parser[T]): String => Parsers#ParseResult[T] = + (str: String) => { + val str1 = cleanedString(str) + p(new CharSequenceReader(str1)) + } + + /** enable direct use of parsers with strings * */ + implicit def UseParserParser[T](p: TsParser.Parser[T]): String => Parsers#ParseResult[T] = + (str: String) => { + val str1 = cleanedString(str) + TsParser.phrase(p)(new TsParser.lexical.Scanner(str1)) + } + def withTsFile[T](resourceName: String)(f: String => T): T = f(files.content(InFile(os.Path(getClass.getResource(s"/$resourceName").getFile)))) diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala index 05a5f4a435..60ff014e71 100644 --- a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala +++ b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsLexer.scala @@ -31,8 +31,6 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo } .mkString("") } - implicit def FromString[T](p: Parser[T]): String => ParseResult[T] = - (str: String) => p.apply(new CharSequenceReader(str)) // format: off @@ -161,12 +159,12 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo } private val newLine: Parser[Char] = - ('\r'.? ~> '\n') | ('\r') ^^ (_ => '\n') + '\n' /** A character-parser that matches a white-space character (and returns it). * We dont ignore newlines */ override val whitespaceChar: Parser[Char] = { - elem("space char", ch => ch <= ' ' && ch != EofCh && ch != '\n' && ch != '\r') + elem("space char", ch => ch <= ' ' && ch != EofCh && ch != '\n') } override val whitespace: Parser[Any] = @@ -190,7 +188,7 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo val comment: Parser[CommentToken] = { val oneLine: Parser[CommentLineToken] = - (whitespaceChar.* ~ '/' ~ '/' ~ rep(chrExcept(EofCh, '\n', '\r'))) ^^ { + (whitespaceChar.* ~ '/' ~ '/' ~ rep(chrExcept(EofCh, '\n'))) ^^ { case cs1 ~ c1 ~ c2 ~ cs2 => CommentLineToken(s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}\n") } @@ -199,8 +197,7 @@ object TsLexer extends Lexical with StdTokens with ParserHelpers with ImplicitCo (whitespaceChar.* ~ '/' ~ '*' ~ rep(not('*' ~ '/') ~> chrExcept(EofCh)) ~ '*' ~ '/' ~ whitespaceChar.* ~ newLine.*) ^^ { case cs1 ~ c1 ~ c2 ~ cs2 ~ c3 ~ c4 ~ cs3 ~ cs4 => CommentBlockToken( - s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}$c3$c4${chars2string(cs3)}${chars2string(cs4)}" - .replaceAll("\r", ""), + s"${chars2string(cs1)}$c1$c2${chars2string(cs2)}$c3$c4${chars2string(cs3)}${chars2string(cs4)}", ) } diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala index 800fb42a36..9d69e3e1c5 100644 --- a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala +++ b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/TsParser.scala @@ -21,10 +21,6 @@ class TsParser(path: Option[(os.Path, Int)]) extends StdTokenParsers with Parser protected def perhapsParens[T](p: Parser[T]): Parser[T] = p | ("(" ~> p <~ ")") - /** enable direct use of parsers with strings **/ - implicit def FromString[T](p: Parser[T]): String => ParseResult[T] = - (str: String) => phrase(p)(new lexical.Scanner(str.trim)) - def memo[P](p: Parser[P]): Parser[P] = path match { case Some((_, length)) => @@ -58,8 +54,8 @@ class TsParser(path: Option[(os.Path, Int)]) extends StdTokenParsers with Parser } } - def apply(content: String): ParseResult[TsParsedFile] = - parsedTsFile(content) +// def apply(content: String): ParseResult[TsParsedFile] = +// parsedTsFile(content) /** handle stray comments **/ override def Parser[T](f: Input => ParseResult[T]): Parser[T] = diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala index d37b0cfeca..867736f05d 100644 --- a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala +++ b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/parser/package.scala @@ -2,15 +2,21 @@ package org.scalablytyped.converter.internal package ts package object parser { + val BOM = "\uFEFF" + + def cleanedString(s1: String): String = { + val s2 = if (s1.startsWith(BOM)) s1.replace(BOM, "") else s1 + val s3 = s2.replace("\r\n", "\n").trim + s3 + } + def parseFile(inFile: InFile): Either[String, TsParsedFile] = parseFileContent(inFile, os.read.bytes(inFile.path)) def parseFileContent(inFile: InFile, bytes: Array[Byte]): Either[String, TsParsedFile] = { - val BOM = "\uFEFF" - val s1 = new String(bytes, constants.Utf8) - val s2 = if (s1.startsWith(BOM)) s1.replace(BOM, "") else s1 - val p = new TsParser(Some((inFile.path, s2.length))) + val str = cleanedString(new String(bytes, constants.Utf8)) + val p = new TsParser(Some((inFile.path, str.length))) - p(s2) match { + p.phrase(p.parsedTsFile)(new TsParser.lexical.Scanner(str)) match { case p.Success(t, _) => Right(t)