First pass

.gitignore

@@ -0,0 +1,2 @@
+spec/config/settings.yml
+

Gemfile

@@ -0,0 +1,10 @@
+source :rubygems
+
+gemspec
+
+group :development, :test do
+  gem 'ruby-debug19'
+
+  gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request.git'
+  gem 'em-synchrony', :git => 'git://github.com/igrigorik/em-synchrony.git'
+end

Gemfile.lock

@@ -0,0 +1,65 @@
+GIT
+  remote: git://github.com/igrigorik/em-http-request.git
+  revision: 2712fbed9d4d8058e77573882e59023364bb1406
+  specs:
+    em-http-request (1.0.0.beta.3)
+      addressable (>= 2.2.3)
+      em-socksify
+      eventmachine (>= 1.0.0.beta.3)
+      http_parser.rb (>= 0.5.1)
+
+GIT
+  remote: git://github.com/igrigorik/em-synchrony.git
+  revision: 574d71fc7a93ded2857171d3f55f944b661df1ca
+  specs:
+    em-synchrony (0.3.0.beta.1)
+      eventmachine (>= 1.0.0.beta.1)
+
+PATH
+  remote: .
+  specs:
+    uber-s3 (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.6)
+    archive-tar-minitar (0.5.2)
+    columnize (0.3.2)
+    diff-lcs (1.1.2)
+    em-socksify (0.1.0)
+      eventmachine
+    eventmachine (1.0.0.beta.3)
+    http_parser.rb (0.5.1)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    rake (0.8.7)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.0)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  em-http-request!
+  em-synchrony!
+  rake
+  rspec (~> 2.6.0)
+  ruby-debug19
+  uber-s3!

README.md

@@ -0,0 +1,10 @@
+# Uber-S3
+
+A simple, but very fast, S3 client written in Ruby supporting
+synchronous and asynchronous HTTP communication.
+
+
+## S3 API Docs
+
+- S3 REST API: http://docs.amazonwebservices.com/AmazonS3/latest/API/
+- S3 Request Authorization: http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html

Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => :spec

benchmarks/bm.rb

@@ -0,0 +1,95 @@
+begin
+  require 'bundler'
+  Bundler.setup(:default, :test)
+rescue LoadError => e
+  # Fall back on doing an unlocked resolve at runtime.
+  $stderr.puts e.message
+  $stderr.puts "Try running `bundle install`"
+  exit!
+end
+
+$:.unshift(File.dirname(__FILE__))
+$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+UBERS3_ROOT = File.expand_path('../../', __FILE__)
+SETTINGS = YAML.load(File.read("#{UBERS3_ROOT}/spec/config/settings.yml"))['test']
+
+# begin
+#   require 'ruby-debug'
+# rescue LoadError
+# end
+
+require 'uber-s3'
+require 'benchmark'
+
+NUM_FILES  = 50
+DATA_SIZE  = 1024 # in bytes
+
+
+## Prepare files and data -----------------------------------------------------
+
+require 'digest/md5'
+
+# Prepare files and data
+files = []
+NUM_FILES.times do
+  files << "/benchmark/"+Digest::MD5.hexdigest(rand.to_s)
+end
+
+data = (1..DATA_SIZE).map{|i| ('a'..'z').to_a[rand(26)]}.join
+
+
+## Bench cases ----------------------------------------------------------------
+
+# Saving objects
+save_object_bm = Proc.new do |client|
+  files.each do |filename|
+    # $stderr.puts filename
+    ret = client.store(filename, data)
+    $stderr.puts "Error storing file #{filename}" if !ret
+  end
+end
+
+
+## Clients --------------------------------------------------------------------
+
+s3 = {}.tap do |clients|
+  [:net_http, :em_http_sync].each do |mode|
+    clients[mode] = UberS3.new({
+      :access_key         => SETTINGS['access_key'],
+      :secret_access_key  => SETTINGS['secret_access_key'],
+      :bucket             => SETTINGS['bucket'],
+      :persistent         => true,
+      :adapter            => mode
+    })
+  end
+end
+
+
+## Let's run this thing -------------------------------------------------------
+
+Benchmark.bm do |bm|
+  bm.report("saving #{NUM_FILES}x#{DATA_SIZE} byte objects (net-http)") do
+    save_object_bm.call(s3[:net_http])
+  end
+
+  bm.report("saving #{NUM_FILES}x#{DATA_SIZE} byte objects (em-http-sync)") do
+    EM.run do
+      Fiber.new {
+        # EM.add_periodic_timer(1) { $stderr.puts "hi" }
+        save_object_bm.call(s3[:em_http_sync])
+
+        EM.stop
+      }.resume
+    end
+  end
+end
+
+__END__
+
+Running this shows that em-http-request is double the speed for small files.
+Unfortunately this example still doesn't show the benefits of async concurrency.
+Still more work to do ... but the results should be amazing.
+
+      user     system      total        real
+saving 50x1024 byte objects (net-http)  0.080000   0.040000   0.120000 ( 20.096848)
+saving 50x1024 byte objects (em-http-sync)  0.080000   0.030000   0.110000 ( 10.222595)

benchmarks/broken.rb

@@ -0,0 +1,92 @@
+begin
+  require 'bundler'
+  Bundler.setup(:default, :test)
+rescue LoadError => e
+  # Fall back on doing an unlocked resolve at runtime.
+  $stderr.puts e.message
+  $stderr.puts "Try running `bundle install`"
+  exit!
+end
+
+$:.unshift(File.dirname(__FILE__))
+$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+UBERS3_ROOT = File.expand_path('../../', __FILE__)
+SETTINGS = YAML.load(File.read("#{UBERS3_ROOT}/spec/config/settings.yml"))['test']
+
+# begin
+#   require 'ruby-debug'
+# rescue LoadError
+# end
+
+require 'uber-s3'
+require 'benchmark'
+
+NUM_FILES  = 100
+DATA_SIZE  = 1024 # in bytes
+
+
+NUM_FIBERS = 2
+require 'fiber_pool'
+@fiber_pool = FiberPool.new(NUM_FIBERS)
+
+
+## Prepare files and data -----------------------------------------------------
+
+require 'digest/md5'
+
+# Prepare files and data
+files = []
+NUM_FILES.times do
+  files << "/benchmark/"+Digest::MD5.hexdigest(rand.to_s)
+end
+
+data = (1..DATA_SIZE).map{|i| ('a'..'z').to_a[rand(26)]}.join
+
+
+## Bench cases ----------------------------------------------------------------
+
+# Saving objects
+save_object_bm = Proc.new do |client|
+  files.each do |filename|
+    work = Proc.new do
+      $stderr.puts filename
+      ret = client.store(filename, data)
+      $stderr.puts "Error storing file #{filename}" if !ret
+    end
+
+    @fiber_pool.spawn(&work)
+  end
+end
+
+
+## Clients --------------------------------------------------------------------
+
+s3 = {}.tap do |clients|
+  [:net_http, :em_http_sync].each do |mode|
+    clients[mode] = UberS3.new({
+      :access_key         => SETTINGS['access_key'],
+      :secret_access_key  => SETTINGS['secret_access_key'],
+      :bucket             => SETTINGS['bucket'],
+      :persistent         => true,
+      :adapter            => mode
+    })
+  end
+end
+
+
+## Let's run this thing -------------------------------------------------------
+
+Benchmark.bmbm do |bm|
+  # bm.report("saving #{NUM_FILES} objects (net-http)") do
+  #   save_object_bm.call(s3[:net_http], false)
+  # end
+
+  bm.report("saving #{NUM_FILES} objects (em-http-sync)") do
+    EM.run do
+      # EM.add_periodic_timer(1) { $stderr.puts "hi" }
+      save_object_bm.call(s3[:em_http_sync])
+
+      EM.stop
+    end
+  end
+end

benchmarks/fiber_pool.rb

@@ -0,0 +1,82 @@
+# Author::    Mohammad A. Ali  (mailto:[email protected])
+# Copyright:: Copyright (c) 2008 eSpace, Inc.
+# License::   Distributes under the same terms as Ruby
+
+require 'fiber'
+
+class Fiber
+
+  #Attribute Reference--Returns the value of a fiber-local variable, using
+  #either a symbol or a string name. If the specified variable does not exist,
+  #returns nil.
+  def [](key)
+    local_fiber_variables[key]
+  end
+
+  #Attribute Assignment--Sets or creates the value of a fiber-local variable,
+  #using either a symbol or a string. See also Fiber#[].
+  def []=(key,value)
+    local_fiber_variables[key] = value
+  end
+
+  private
+
+  def local_fiber_variables
+    @local_fiber_variables ||= {}
+  end
+end
+
+class FiberPool
+
+  # gives access to the currently free fibers
+  attr_reader :fibers
+  attr_reader :busy_fibers
+
+  # Code can register a proc with this FiberPool to be called
+  # every time a Fiber is finished.  Good for releasing resources
+  # like ActiveRecord database connections.
+  attr_accessor :generic_callbacks
+
+  # Prepare a list of fibers that are able to run different blocks of code
+  # every time. Once a fiber is done with its block, it attempts to fetch
+  # another one from the queue
+  def initialize(count = 100)
+    @fibers,@busy_fibers,@queue,@generic_callbacks = [],{},[],[]
+    count.times do |i|
+      fiber = Fiber.new do |block|
+        loop do
+          block.call
+          # callbacks are called in a reverse order, much like c++ destructor
+          Fiber.current[:callbacks].pop.call while Fiber.current[:callbacks].length > 0
+          generic_callbacks.each do |cb|
+            cb.call
+          end
+          unless @queue.empty?
+            block = @queue.shift
+          else
+            @busy_fibers.delete(Fiber.current.object_id)
+            @fibers.unshift Fiber.current
+            block = Fiber.yield
+          end
+        end
+      end
+      fiber[:callbacks] = []
+      fiber[:em_keys] = []
+      @fibers << fiber
+    end
+  end
+
+  # If there is an available fiber use it, otherwise, leave it to linger
+  # in a queue
+  def spawn(&block)
+    if fiber = @fibers.shift
+      fiber[:callbacks] = []
+      @busy_fibers[fiber.object_id] = fiber
+      fiber.resume(block)
+    else
+      @queue << block
+    end
+    self # we are keen on hiding our queue
+  end
+
+end