Improved performance of http connections with tcp_nodelay; version bu…

…mp 0.2.1

Gemfile

@@ -3,8 +3,8 @@ source :rubygems
 gemspec
 
 group :development, :test do
-  gem 'ruby-debug',   :platforms => :mri_18
-  gem 'ruby-debug19', :platforms => :mri_19
+  gem 'pry'
+  gem 'pry-nav'
 
   gem 'eventmachine', '1.0.0.beta.4'
   gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request.git'

Gemfile.lock

@@ -1,48 +1,54 @@
 GIT
   remote: git://github.com/igrigorik/em-http-request.git
-  revision: 1a4123d36a298e8043482ad7b20cb18dfbc2616b
+  revision: 6159c1643fc38e9a947c58834221856ed6fdf392
   specs:
-    em-http-request (1.0.0)
+    em-http-request (1.0.2)
       addressable (>= 2.2.3)
       cookiejar
       em-socksify
-      eventmachine (>= 1.0.0.beta.3)
+      eventmachine (>= 1.0.0.beta.4)
       http_parser.rb (>= 0.5.3)
 
 GIT
   remote: git://github.com/igrigorik/em-synchrony.git
-  revision: f07b2a139ba05a1eaf3559541f3584c73afe05b2
+  revision: 572482b3d5753813d6643530714887301095e124
   specs:
-    em-synchrony (1.0.0)
+    em-synchrony (1.0.1)
       eventmachine (>= 1.0.0.beta.1)
 
 PATH
   remote: .
   specs:
-    uber-s3 (0.1.5)
+    uber-s3 (0.2.0)
       mime-types (~> 1.17)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.2.6)
-    archive-tar-minitar (0.5.2)
-    columnize (0.3.4)
+    addressable (2.2.7)
+    coderay (1.0.6)
     cookiejar (0.3.0)
     diff-lcs (1.1.3)
-    em-socksify (0.1.0)
-      eventmachine
+    em-socksify (0.2.0)
+      eventmachine (>= 1.0.0.beta.4)
     eventmachine (1.0.0.beta.4)
     eventmachine (1.0.0.beta.4-java)
     http_parser.rb (0.5.3)
     http_parser.rb (0.5.3-java)
-    linecache (0.46)
-      rbx-require-relative (> 0.0.4)
-    linecache19 (0.5.12)
-      ruby_core_source (>= 0.1.4)
-    mime-types (1.17.2)
+    method_source (0.7.1)
+    mime-types (1.18)
+    pry (0.9.8.4)
+      coderay (~> 1.0.5)
+      method_source (~> 0.7.1)
+      slop (>= 2.4.4, < 3)
+    pry (0.9.8.4-java)
+      coderay (~> 1.0.5)
+      method_source (~> 0.7.1)
+      slop (>= 2.4.4, < 3)
+      spoon (~> 0.0)
+    pry-nav (0.2.0)
+      pry (~> 0.9.8.1)
     rake (0.9.2.2)
-    rbx-require-relative (0.0.5)
     rspec (2.7.0)
       rspec-core (~> 2.7.0)
       rspec-expectations (~> 2.7.0)
@@ -51,21 +57,8 @@ GEM
     rspec-expectations (2.7.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.7.0)
-    ruby-debug (0.10.4)
-      columnize (>= 0.1)
-      ruby-debug-base (~> 0.10.4.0)
-    ruby-debug-base (0.10.4)
-      linecache (>= 0.3)
-    ruby-debug-base19 (0.11.25)
-      columnize (>= 0.3.1)
-      linecache19 (>= 0.5.11)
-      ruby_core_source (>= 0.1.4)
-    ruby-debug19 (0.11.6)
-      columnize (>= 0.3.1)
-      linecache19 (>= 0.5.11)
-      ruby-debug-base19 (>= 0.11.19)
-    ruby_core_source (0.1.5)
-      archive-tar-minitar (>= 0.5.2)
+    slop (2.4.4)
+    spoon (0.0.1)
 
 PLATFORMS
   java
@@ -75,8 +68,8 @@ DEPENDENCIES
   em-http-request!
   em-synchrony!
   eventmachine (= 1.0.0.beta.4)
+  pry
+  pry-nav
   rake
   rspec (~> 2.7.0)
-  ruby-debug
-  ruby-debug19
   uber-s3!

README.md

@@ -1,7 +1,7 @@
 # Uber-S3
 
-A simple, but very fast, S3 client written in Ruby supporting
-synchronous and asynchronous HTTP communication.
+A simple, but very fast, S3 client for Ruby supporting
+synchronous (net-http) and asynchronous (em+fibers) io.
 
 
 ## Examples
@@ -102,13 +102,13 @@ s3.objects('/path').each {|obj| puts obj }
 
 ## Ruby version notes
 
-* Tested on MRI 1.9.2 (net_http / em_http_fibered adapters) -- recommended
-* Ruby 1.8.7 works for net/http clients, em_http_fibered adapter requires fibers (duh)
-* JRuby in 1.9 mode + em_http_fibered adapter should work, but held back because of issues with EM
+* Tested on MRI 1.9.2, MRI 1.9.3 (net_http / em_http_fibered adapters)
+* Tested on JRuby 1.7-dev in 1.9 mode (net_http)
+* Ruby 1.8.7 works for net/http clients, em_http_fibered adapter requires fibers
 
 ## Other notes
 
-* If Nokogiri is available, it will use it instead of REXML
+* If Nokogiri is available, it will be automatically used instead of REXML
 
 ## TODO
 
@@ -150,4 +150,4 @@ Yea... async adapter dominates. The 100x1KB files were 29x faster to upload, and
 
 ## License
 
-MIT License - Copyright (c) 2011 Nulayer Inc.
+MIT License - Copyright (c) 2012 Nulayer Inc.

benchmark/bm.rb

@@ -22,7 +22,7 @@
 require 'benchmark'
 
 NUM_FILES  = 100
-DATA_SIZE  = 1024 # in bytes
+DATA_SIZE  = 1048 # in bytes
 
 
 ## Prepare files and data -----------------------------------------------------
@@ -43,7 +43,7 @@
 # Saving objects
 save_object_bm = Proc.new do |client|
   files.each do |filename|
-    # $stderr.puts filename
+    $stderr.puts filename
     ret = client.store(filename, data)
     $stderr.puts "Error storing file #{filename}" if !ret
   end

lib/uber-s3/connection/net_http.rb

@@ -11,19 +11,29 @@ def request(verb, url, headers={}, body=nil)
 
       self.uri = URI.parse(url)
 
-      # Keep connection open
+      # Init and open a HTTP connection
       self.http ||= Net::HTTP.new(uri.host, uri.port)
-
+      if !http.started? || !http.active?
+        http.start
+
+        if Socket.const_defined?(:TCP_NODELAY)
+          socket = http.instance_variable_get(:@socket)
+          socket.io.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+        end
+      end
+
       req_klass = instance_eval("Net::HTTP::"+verb.to_s.capitalize)
       req = req_klass.new(uri.to_s, headers)
 
       req.body = body if !body.nil? && !body.empty?
 
-      # Make HTTP request      
+      # Make HTTP request
       r = http.request(req)
-
+
+      # $stderr.puts "active? " + http.active?.to_s
+
       # Auto-decode any gzipped objects
-      if verb == :get && r.header['content-encoding'] == 'gzip'
+      if verb == :get && r.header['Content-Encoding'] == 'gzip'
         gz = Zlib::GzipReader.new(StringIO.new(r.body))
         response_body = gz.read
       else

lib/uber-s3/object.rb

@@ -131,9 +131,10 @@ def value=(value)
       def parse_response_header!
         # Meta..
         self.meta ||= {}
-        response.header.keys.sort.select {|k| k =~ /^x-amz-meta-/ }.each do |amz_key|
+        response.header.keys.sort.select {|k| k =~ /^x.amz.meta./i }.each do |amz_key|
+
           # TODO.. value is an array.. meaning a meta attribute can have multiple values
-          meta[amz_key.gsub(/^x-amz-meta-/, '')] = response.header[amz_key].first
+          meta[amz_key.gsub(/^x.amz.meta./i, '')] = [response.header[amz_key]].flatten.first
 
           # TODO.. em-http adapters return headers that look like X_AMZ_META_ .. very annoying.
         end

lib/uber-s3/version.rb

@@ -1,3 +1,3 @@
 class UberS3
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

spec/spec_helper.rb

@@ -14,18 +14,17 @@
 SPEC_ROOT = UBERS3_ROOT + '/spec'
 
 begin
-  require 'ruby-debug'
+  require 'pry'
+  require 'pry-nav'
 rescue LoadError
 end
 
 require 'uber-s3'
 require 'rspec'
-# require 'fabrication'
 
 # Requires supporting files with custom matchers and macros, etc,
 # in ./support/ and its subdirectories.
 # Dir["#{SPEC_ROOT}/support/**/*.rb"].each {|f| require f}
-# Dir["#{SPEC_ROOT}/fabricators/**/*.rb"].each {|f| require f}
 
 SETTINGS = YAML.load(File.read("#{SPEC_ROOT}/config/settings.yml"))['test']
 

spec/uber-s3/object_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 
 describe UberS3::Object do
-  [:net_http, :em_http_fibered].each do |connection_adapter|
-  # [:net_http].each do |connection_adapter|
+  # [:net_http, :em_http_fibered].each do |connection_adapter|
+  [:net_http].each do |connection_adapter|
 
     context "#{connection_adapter}: Object" do
       let(:s3) do

uber-s3.gemspec

@@ -22,5 +22,4 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency('rake')
   s.add_development_dependency('rspec', ['~> 2.7.0'])
-  # s.add_development_dependency('fabrication', ['~> 0.9.5'])
 end