Fix a bug that a large XML can't be parsed (#154)

GitHub: fix GH-150

If a parsed XML is later than `2 ** 31 - 1`, we can't parse it. Because
`StringScanner`s position is stored as `int`. We can avoid the
restriction by dropping large parsed content.

Co-authored-by: Sutou Kouhei <[email protected]>

lib/rexml/parsers/baseparser.rb

@@ -204,6 +204,8 @@ def peek depth=0
 
       # Returns the next event.  This is a +PullEvent+ object.
       def pull
+        @source.drop_parsed_content
+
         pull_event.tap do |event|
           @listeners.each do |listener|
             listener.receive event

lib/rexml/source.rb

@@ -55,6 +55,7 @@ class Source
     attr_reader :encoding
 
     module Private
+      SCANNER_RESET_SIZE = 100000
       PRE_DEFINED_TERM_PATTERNS = {}
       pre_defined_terms = ["'", '"', "<"]
       pre_defined_terms.each do |term|
@@ -84,6 +85,12 @@ def buffer
       @scanner.rest
     end
 
+    def drop_parsed_content
+      if @scanner.pos > Private::SCANNER_RESET_SIZE
+        @scanner.string = @scanner.rest
+      end
+    end
+
     def buffer_encoding=(encoding)
       @scanner.string.force_encoding(encoding)
     end

test/parser/test_base_parser.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: false
+
+require 'rexml/parsers/baseparser'
+
+module REXMLTests
+  class BaseParserTester < Test::Unit::TestCase
+    def test_large_xml
+      large_text = "a" * 100_000
+      xml = <<-XML
+        <?xml version="1.0"?>
+        <root>
+          <child>#{large_text}</child>
+          <child>#{large_text}</child>
+        </root>
+      XML
+
+      parser = REXML::Parsers::BaseParser.new(xml)
+      while parser.has_next?
+        parser.pull
+      end
+
+      assert do
+        parser.position < xml.bytesize
+      end
+    end
+  end
+end