Add new http cache support, and some fixes

Gemfile

@@ -6,7 +6,7 @@ group :development, :test do
   gem 'ruby-debug',   :platforms => :mri_18
   gem 'ruby-debug19', :platforms => :mri_19
 
-  gem 'eventmachine', '1.0.0.beta.3'  
+  gem 'eventmachine', '1.0.0.beta.4'
   gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request.git'
   gem 'em-synchrony', :git => 'git://github.com/igrigorik/em-synchrony.git'
 

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: git://github.com/igrigorik/em-http-request.git
-  revision: a1623d48a4edb57c58e76369123c7b9bd517a29a
+  revision: 1a4123d36a298e8043482ad7b20cb18dfbc2616b
   specs:
     em-http-request (1.0.0)
       addressable (>= 2.2.3)
@@ -11,16 +11,16 @@ GIT
 
 GIT
   remote: git://github.com/igrigorik/em-synchrony.git
-  revision: 39954edecdca6078cb18cea890558339122bd2e1
+  revision: f07b2a139ba05a1eaf3559541f3584c73afe05b2
   specs:
     em-synchrony (1.0.0)
       eventmachine (>= 1.0.0.beta.1)
 
 PATH
   remote: .
   specs:
-    uber-s3 (0.1.3)
-      mime-types (~> 1.16)
+    uber-s3 (0.1.5)
+      mime-types (~> 1.17)
 
 GEM
   remote: http://rubygems.org/
@@ -32,16 +32,16 @@ GEM
     diff-lcs (1.1.3)
     em-socksify (0.1.0)
       eventmachine
-    eventmachine (1.0.0.beta.3)
-    eventmachine (1.0.0.beta.3-java)
+    eventmachine (1.0.0.beta.4)
+    eventmachine (1.0.0.beta.4-java)
     http_parser.rb (0.5.3)
     http_parser.rb (0.5.3-java)
     linecache (0.46)
       rbx-require-relative (> 0.0.4)
     linecache19 (0.5.12)
       ruby_core_source (>= 0.1.4)
-    mime-types (1.16)
-    rake (0.9.2)
+    mime-types (1.17.2)
+    rake (0.9.2.2)
     rbx-require-relative (0.0.5)
     rspec (2.7.0)
       rspec-core (~> 2.7.0)
@@ -74,7 +74,7 @@ PLATFORMS
 DEPENDENCIES
   em-http-request!
   em-synchrony!
-  eventmachine (= 1.0.0.beta.3)
+  eventmachine (= 1.0.0.beta.4)
   rake
   rspec (~> 2.7.0)
   ruby-debug

examples/basic.rb

@@ -7,8 +7,8 @@
 require 'uber-s3'
 
 s3 = UberS3.new({
-  :access_key         => 'AKIAIIG7K3KMY5BPW7TA',
-  :secret_access_key  => 'BnWNYdKSyEkkrhlkFgPvQ+bVW8J3XEDj3+6ocWCm',
+  :access_key         => 'x',
+  :secret_access_key  => 'y',
   :bucket             => 'nutestbucket',
   :adapter            => :net_http
 })

examples/basic_fibered.rb

@@ -11,8 +11,8 @@
 require 'uber-s3'
 
 s3 = UberS3.new({
-  :access_key         => 'AKIAIIG7K3KMY5BPW7TA',
-  :secret_access_key  => 'BnWNYdKSyEkkrhlkFgPvQ+bVW8J3XEDj3+6ocWCm',
+  :access_key         => 'x',
+  :secret_access_key  => 'y',
   :bucket             => 'nutestbucket',
   :adapter            => :em_http_fibered
 })

examples/concurrent.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+$:<< '../lib' << 'lib'
+###############################################################################
+#
+###############################################################################
+
+# require 'uber-s3'
+# 
+# s3 = UberS3.new({
+#   :access_key         => 'x',
+#   :secret_access_key  => 'y',
+#   :bucket             => 'nutestbucket',
+#   :adapter            => :em_http_fibered,
+#   :concurrency        => 100
+# })
+
+require 'eventmachine'
+require 'em-http'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+require 'ruby-debug'
+
+CONCURRENCY = 10
+
+EM.run do
+  pool = EM::Synchrony::ConnectionPool.new(:size => CONCURRENCY) do
+    EM::HttpRequest.new('http://www.google.ca/')
+  end
+
+  p = { :keepalive => true }
+  num = 40
+  counter = num
+
+  num.times do
+    Fiber.new {
+      x = pool.get(p)
+      puts "GOT IT #{x.response.length}"
+      counter -= 1
+
+      EM.stop if counter <= 0
+    }.resume
+  end
+
+end
+
+
+
+__END__
+###############################################################################
+# This example demonstrates the combined power of eventmachine+fibers.
+# Notice how the example code is exactly the same as basic.rb
+# except instead it's using a non-blocking http client.. boom.
+#
+# Btw, make sure to bundle up first...
+###############################################################################
+
+require 'uber-s3'
+
+s3 = UberS3.new({
+  :access_key         => 'x',
+  :secret_access_key  => 'y',
+  :bucket             => 'nutestbucket',
+  :adapter            => :em_http_fibered
+})
+
+x = Proc.new do
+  # Traverse all objects in the bucket -- beware :)
+  # This will only load the keys and basic info of the object, not the data
+  s3.objects('/').each do |obj|
+    puts obj
+    # puts obj.value # this will actually fetch the data on demand
+  end
+end
+
+EM.run do
+  Fiber.new {
+    x.call
+    EM.stop
+  }.resume
+end

lib/uber-s3/object.rb

@@ -1,12 +1,11 @@
 class UberS3
   class Object
     include Operation::Object::AccessPolicy
-    include Operation::Object::CacheControl
     include Operation::Object::ContentDisposition
     include Operation::Object::ContentEncoding
     include Operation::Object::ContentMd5
     include Operation::Object::ContentType
-    include Operation::Object::Expires
+    include Operation::Object::HttpCache
     include Operation::Object::Meta
     include Operation::Object::StorageClass
 
@@ -44,12 +43,13 @@ def save
       gzip_content!
 
       # Standard pass through values
-      headers['Cache-Control']        = cache_control
       headers['Content-Disposition']  = content_disposition
       headers['Content-Encoding']     = content_encoding
       headers['Content-Length']       = size.to_s
       headers['Content-Type']         = content_type
+      headers['Cache-Control']        = cache_control
       headers['Expires']              = expires
+      headers['Pragma']               = pragma
 
       headers.each {|k,v| headers.delete(k) if v.nil? || v.empty? }
 

lib/uber-s3/operation/object/http_cache.rb

@@ -0,0 +1,37 @@
+module UberS3::Operation::Object
+  module HttpCache
+
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+
+      base.instance_eval do
+        attr_accessor :cache_control, :expires, :pragma, :ttl
+      end
+    end
+
+    module ClassMethods
+    end
+
+    module InstanceMethods
+
+      # Helper method that will set the max-age for cache-control
+      def ttl=(seconds)
+        @ttl = seconds
+        self.cache_control = "public,max-age=#{seconds}"
+      end
+
+      # Expires can take a time or string
+      def expires=(val)
+        if val.is_a?(String)
+          self.expires = val
+        elsif val.is_a?(Time)
+          # RFC 1123 format 
+          self.expires = val.strftime("%a, %d %b %Y %H:%I:%S %Z")
+        end
+      end
+
+    end
+
+  end
+end

lib/uber-s3/response.rb

@@ -13,14 +13,14 @@ def initialize(options={})
       self.body   = options[:body]
       self.raw    = options[:raw]
 
-      check_for_errors!
+      success?
     end
 
     # TODO: can/should we normalize the keys..? downcase.. etc.?
     # def header=(header)
     # end
 
-    def check_for_errors!
+    def success?
       return if status < 400 || body.to_s.empty?
 
       # Errors are XML

lib/uber-s3/version.rb

@@ -1,3 +1,3 @@
 class UberS3
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

research/fiber-http.rb

@@ -0,0 +1,54 @@
+require 'bundler'
+Bundler.setup
+
+require 'ruby-debug'
+require 'eventmachine'
+require 'em-http-request'
+require 'fiber'
+
+
+def http_get(url)
+  f = Fiber.current
+
+  req = EM::HttpRequest.new(url).get
+  req.callback { f.resume(req) }
+  req.errback  { f.resume(req) }
+
+  Fiber.yield
+end
+
+EM.run do
+  urls = [
+    ['http://nulayer.com/', 'http://twitter.com/#!/nupeter/'],
+    ['http://google.ca/', 'http://twitter.com/#!/jeffbrenner/'],
+    ['http://facebook.com/', 'http://twitter.com/#!/nulayer/'],
+    ['http://data.crowdreel.com.s3.amazonaws.com/2011/05/17-02/1f55685cc1e9c91d80079a49485c718b1f53d97f.jpg', 'http://twitter.com/#!/jack/'],
+    ['http://data.crowdreel.com.s3.amazonaws.com/2011/05/17-02/54acc4f94b81268982aab94e0d273693962741ab.jpg', 'http://twitter.com/#!/ev/'],
+    ['http://data.crowdreel.com.s3.amazonaws.com/2011/05/17-02/66ca6cb906f1b517cc576477b3d69e155dc21284.jpg', 'http://twitter.com/#!/presslyapp/'],
+    ['http://data.crowdreel.com.s3.amazonaws.com/2011/05/17-02/cfeb066a4f0ba75a2081898a6c750d371aafd7e9.jpg', 'http://twitter.com/#!/crpwdreel/']
+  ]
+
+  puts "init"
+
+  Fiber.new {
+    # url = 'http://nulayer.com/'
+    # ret = http_get(url)
+    # puts ret.response_header.status.to_s+" from #{url}"
+
+    urls.each do |url|
+      Fiber.new {
+        puts "A start: #{url[0]}"
+        ret = http_get(url[0])
+        puts "A done:  #{url[0]} -- #{ret.response_header.status.to_s}"
+
+        Fiber.new {
+          puts "B start: #{url[1]}"
+          ret = http_get(url[0])
+          puts "B done:  #{url[1]} -- #{ret.response_header.status.to_s}"
+        }.resume
+      }.resume
+    end
+  }.resume
+
+  puts "eof"
+end

research/fiber-test.rb

@@ -0,0 +1,20 @@
+require 'fiber'
+
+def woot
+  x = nil
+
+  f = Fiber.new { |k|
+    puts k
+    y = Fiber.yield 1
+    puts y
+    x = "hello"
+    2
+  }
+
+  puts f.resume(5)
+  puts f.resume(6)
+  puts x
+
+end
+
+woot

research/iterate/Gemfile

@@ -1,6 +1,8 @@
 source :rubygems
 
-gem 'eventmachine', '1.0.0.beta.3'  
+gem 'mime-types'
+
+gem 'eventmachine', '1.0.0.beta.4'
 gem 'em-http-request', :git => 'git://github.com/igrigorik/em-http-request.git'
 gem 'em-synchrony', :git => 'git://github.com/igrigorik/em-synchrony.git'