make flatMap consume whitespace, introduce flatMapX that doesn't

Author: lihaoyi

fastparse/src/fastparse/internal/MacroImpls.scala

@@ -163,7 +163,7 @@ object MacroImpls {
   }
 
 
-  def flatMapMacro[T: c.WeakTypeTag, V: c.WeakTypeTag]
+  def flatMapXMacro[T: c.WeakTypeTag, V: c.WeakTypeTag]
                   (c: Context)
                   (f: c.Expr[T => ParsingRun[V]]): c.Expr[ParsingRun[V]] = {
     import c.universe._
@@ -176,6 +176,30 @@ object MacroImpls {
     }
   }
 
+  def flatMapMacro[T: c.WeakTypeTag, V: c.WeakTypeTag]
+                  (c: Context)
+                  (f: c.Expr[T => ParsingRun[V]])
+                  (whitespace: c.Expr[ParsingRun[Any] => ParsingRun[Unit]]): c.Expr[ParsingRun[V]] = {
+    import c.universe._
+
+    val lhs0 = c.prefix.asInstanceOf[c.Expr[EagerOps[T]]]
+    reify {
+      val lhs = lhs0.splice.parse0
+      whitespace.splice match{ case ws =>
+        if (!lhs.isSuccess) lhs.asInstanceOf[ParsingRun[V]]
+        else {
+          val oldCapturing = lhs.noDropBuffer
+          val successValue = lhs.successValue
+          lhs.noDropBuffer = true
+          ws(lhs)
+          lhs.noDropBuffer = oldCapturing
+          if (!lhs.isSuccess && lhs.cut) lhs.asInstanceOf[ParsingRun[V]]
+          else f.splice(successValue.asInstanceOf[T])
+        }
+      }
+    }
+  }
+
   def eitherMacro[T: c.WeakTypeTag, V >: T: c.WeakTypeTag]
                   (c: Context)
                   (other: c.Expr[ParsingRun[V]])

fastparse/src/fastparse/package.scala

@@ -177,12 +177,21 @@ package object fastparse {
               (implicit ctx: P[Any]): P[T] = macro MacroImpls.filterMacro[T]
     /**
       * Transforms the result of this parser using the given function into a
-      * new parser which is applied. Useful for doing dependent parsing, e.g.
-      * when parsing JSON you may first parse a character to see if it's a `[`,
-      * `{`, or `"`, and then deciding whether you next want to parse an array,
-      * dictionary or string.
+      * new parser which is applied (after whitespace). Useful for doing
+      * dependent parsing, e.g. when parsing JSON you may first parse a
+      * character to see if it's a `[`, `{`, or `"`, and then deciding whether
+      * you next want to parse an array, dictionary or string.
       */
-    def flatMap[V](f: T => P[V]): P[V] = macro MacroImpls.flatMapMacro[T, V]
+    def flatMap[V](f: T => P[V])
+                  (implicit whitespace: P[Any] => P[Unit]): P[V] = macro MacroImpls.flatMapMacro[T, V]
+    /**
+      * Transforms the result of this parser using the given function into a
+      * new parser which is applied (without consuming whitespace). Useful for
+      * doing dependent parsing, e.g. when parsing JSON you may first parse a
+      * character to see if it's a `[`, `{`, or `"`, and then deciding whether
+      * you next want to parse an array, dictionary or string.
+      */
+    def flatMapX[V](f: T => P[V]): P[V] = macro MacroImpls.flatMapXMacro[T, V]
 
     /**
       * Either-or operator: tries to parse the left-hand-side, and if that
@@ -195,7 +204,7 @@ package object fastparse {
     /**
       * Capture operator; makes the parser return the span of input it parsed
       * as a [[String]], which can then be processed further using [[~]],
-      * [[map]] or [[flatMap]]
+      * [[map]] or [[flatMapX]]
       */
     def !(implicit ctx: P[Any]): P[String] = macro MacroImpls.captureMacro
 
@@ -590,9 +599,9 @@ package object fastparse {
 
   /**
     * Like [[AnyChar]], but returns the single character it parses. Useful
-    * together with [[EagerOps.flatMap]] to provide one-character-lookahead
+    * together with [[EagerOps.flatMapX]] to provide one-character-lookahead
     * style parsing: [[SingleChar]] consumes the single character, and then
-    * [[EagerOps.flatMap]] can `match` on that single character and decide
+    * [[EagerOps.flatMapX]] can `match` on that single character and decide
     * which downstream parser you wish to invoke
     */
   def SingleChar(implicit ctx: P[_]): P[Char] = {

fastparse/test/src/fastparse/IndentationTests.scala

@@ -23,7 +23,7 @@ object IndentationTests extends TestSuite{
     def number[_: P]: P[Int] = P( CharIn("0-9").rep(1).!.map(_.toInt) )
 
     def deeper[_: P]: P[Int] = P( " ".rep(indent + 1).!.map(_.length) )
-    def blockBody[_: P]: P[Seq[Int]] = "\n" ~ deeper.flatMap(i =>
+    def blockBody[_: P]: P[Seq[Int]] = "\n" ~ deeper.flatMapX(i =>
       new Parser(indent = i).factor.rep(1, sep = ("\n" + " " * i)./)
     )
     def block[_: P]: P[Int] = P( CharIn("+\\-*/").! ~/ blockBody).map(eval)

fastparse/test/src/fastparse/ParsingTests.scala

@@ -1,7 +1,7 @@
 package test.fastparse
 import fastparse._
 import utest._
-import NoWhitespace._
+
 object ParsingTests extends TestSuite{
 
 
@@ -33,7 +33,7 @@ object ParsingTests extends TestSuite{
 
   }
   val tests = Tests {
-
+    import NoWhitespace._
 
     'literal - {
       checkFail(implicit c => "Hello WOrld!", ("Hello", 0), 0)
@@ -185,7 +185,8 @@ object ParsingTests extends TestSuite{
         checkFail(implicit c => ("Hello" ~/ "Bye").?, ("HelloBoo", 0), 5)
       }
       'flatMap - {
-        checkFlatmap()
+        checkFail(implicit c => ("Hello" ~/ "Boo").flatMapX(_ => Fail).?, ("HelloBoo", 0), 8)
+        checkFail(implicit c => (("Hello" ~/ "Boo").flatMapX(_ => Pass) ~ Fail).?, ("HelloBoo", 0), 8)
       }
       'filter - {
         checkFail(implicit c => ("Hello" ~/ "Boo").filter(_ => false) | "", ("HelloBoo", 0), 8)
@@ -212,11 +213,17 @@ object ParsingTests extends TestSuite{
       val msg = f.trace().msg
       msg ==> """Expected "hello" | "world":1:1, found "cow" """.trim
     }
+    'whitespaceFlatMap{
+      checkWhitespaceFlatMap()
+    }
   }
-  // Broken out of the TestSuite block to avoid problems in our 2.10.x
-  // build due to https://issues.scala-lang.org/browse/SI-7987
-  def checkFlatmap() = {
-    checkFail(implicit c => ("Hello" ~/ "Boo").flatMap(_ => Fail).?, ("HelloBoo", 0), 8)
-    checkFail(implicit c => (("Hello" ~/ "Boo").flatMap(_ => Pass) ~ Fail).?, ("HelloBoo", 0), 8)
+
+  def checkWhitespaceFlatMap() = {
+    import fastparse._, SingleLineWhitespace._
+    def parser[_: P] = P( CharsWhileIn("a").!.flatMap{n => "b" * n.length} ~ End )
+    val Parsed.Success(_, _) = parse("aaa bbb", parser(_))
+    val Parsed.Success(_, _) = parse("aa    bb", parser(_))
+    val Parsed.Failure(_, _, _) = parse("aaa bb", parser(_))
+    val Parsed.Failure(_, _, _) = parse("aaa b", parser(_))
   }
 }

pythonparse/src/pythonparse/Statements.scala

@@ -189,7 +189,7 @@ class Statements(indent: Int){
         _.collectFirst{ case (s, None) => s}
       }.filter(_.isDefined).map(_.get)
     }
-    def indented = P( deeper.flatMap{ nextIndent =>
+    def indented = P( deeper.flatMapX{ nextIndent =>
       new Statements(nextIndent).stmt.repX(1, spaces.repX(1) ~~ (" " * nextIndent | "\t" * nextIndent)).map(_.flatten)
     } )
     P( indented | " ".rep ~ simple_stmt )

readme/Changelog.scalatex

@@ -7,7 +7,7 @@
                 user-facing interface; binary and source incompatible with
                 @sect.ref{1.0.0}
             @li
-                3-5x performance improvements on most benchmarks
+                3-4x performance improvements on most benchmarks
             @li
                 Parsers are no longer immutable objects, but just methods taking
                 and returning @code{fastparse.ParsingRun} instances
@@ -25,6 +25,9 @@
             @li
                 Dropped support for Byte Parsers (due to low uptake), Scala 2.10,
                 Scala-Native (until 0.4.0 is out)
+            @li
+                @code{.flatMap} now consumes whitespace between the first and
+                second parsers; use @code{.flatMapX} if you want to avoid this.
     @sect{1.0.0}
         @ul
            @li

readme/WritingParsers.scalatex

@@ -196,6 +196,11 @@
             @p
                 Which is equivalent and behaves exactly the same.
 
+            @p
+                Note that @code{.flatMap} consumes whitespace between the first
+                and second parsers; in cases where you do not want to do this,
+                use @code{.flatMapX}
+
         @sect{Filter}
             @hl.ref(tests/"ExampleTests.scala", Seq("'filter", ""))