Require CRLF line endings in request line and headers

Disallow bare CR, LF, NUL in header and request lines. Tighten
parsing of request lines to only allow single spaces, as specified
in the RFCs.

Forcing this RFC-compliant behavior breaks a lot of tests, so
fix the tests to correctly use CRLF instead of LF for requests
(other than the specific checks for handling of bad requests).

Fixes #137

lib/webrick/httprequest.rb

@@ -458,7 +458,7 @@ def read_request_line(socket)
       end
 
       @request_time = Time.now
-      if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line
+      if /^(\S+) (\S++)(?: HTTP\/(\d+\.\d+))?\r\n/mo =~ @request_line
         @request_method = $1
         @unparsed_uri   = $2
         @http_version   = HTTPVersion.new($3 ? $3 : "0.9")
@@ -471,7 +471,7 @@ def read_request_line(socket)
     def read_header(socket)
       if socket
         while line = read_line(socket)
-          break if /\A(#{CRLF}|#{LF})\z/om =~ line
+          break if /\A#{CRLF}\z/om =~ line
           if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH
             raise HTTPStatus::RequestEntityTooLarge, 'headers too large'
           end

lib/webrick/httputils.rb

@@ -173,16 +173,18 @@ def parse_header(raw)
       field = nil
       raw.each_line{|line|
         case line
-        when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):(.*?)\z/om
-          field, value = $1, $2.strip
+        when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):([^\r\n\0]*?)\r\n\z/om
+          field, value = $1, $2
           field.downcase!
           header[field] = HEADER_CLASSES[field].new unless header.has_key?(field)
           header[field] << value
-        when /^\s+(.*?)/om
-          value = line.strip
+        when /^\s+([^\r\n\0]*?)\r\n/om
           unless field
             raise HTTPStatus::BadRequest, "bad header '#{line}'."
           end
+          value = line
+          value.lstrip!
+          value.slice!(-2..-1)
           header[field][-1] << " " << value
         else
           raise HTTPStatus::BadRequest, "bad header '#{line}'."

test/webrick/test_filehandler.rb

@@ -33,7 +33,7 @@ def make_range_request(range_spec)
       Range: #{range_spec}
 
     END_OF_REQUEST
-    return StringIO.new(msg.gsub(/^ {6}/, ""))
+    return StringIO.new(msg.gsub(/^ {6}/, "").gsub("\n", "\r\n"))
   end
 
   def make_range_response(file, range_spec)