Merge branch 'master' of https://github.com/ChokePointProject/cpp-backend

Conflicts:
	Makefile
	db/seeds.rb

Author: Simon Funke

Makefile

@@ -2,5 +2,6 @@ default:
 	rake db:drop
 	rake db:create
 	rake db:migrate
-	rake load_mlab_data:all
 	rake db:seed
+	rake load_mlab_data:all
+	rake load_transparencyreport_data:all

app/models/content_removal_request.rb

@@ -1,8 +1,7 @@
 class ContentRemovalRequest < ActiveRecord::Base
   belongs_to :content_removal_request_period
+  belongs_to :product
+  belongs_to :reason
 
-  has_many :products
-  has_many :reasons
-
-  attr_accessible :courtOrders, :executive, :items
+  attr_accessible :court_orders, :executive, :items
 end

app/models/content_removal_request_period.rb

@@ -1,6 +1,7 @@
 class ContentRemovalRequestPeriod < ActiveRecord::Base
+  belongs_to :country
+
   has_many :content_removal_requests
-  has_one :country
 
   attr_accessible :period_start, :period_end, :percentage_complied
 end

app/models/country.rb

@@ -1,6 +1,8 @@
 class Country < ActiveRecord::Base
   belongs_to :continent
-  belongs_to :content_removal_request_period
+
+  has_many :content_removal_request_periods
+  has_many :content_removal_requests, :through => :content_removal_request_periods
 
   validates_presence_of :code, :name
   validates_uniqueness_of :code

app/models/product.rb

@@ -1,5 +1,5 @@
 class Product < ActiveRecord::Base
-  belongs_to :content_removal_request
+  has_many :content_removal_requests
 
   attr_accessible :name
 end

app/models/reason.rb

@@ -1,5 +1,5 @@
 class Reason < ActiveRecord::Base
-  belongs_to :content_removal_request
+  has_many :content_removal_requests
 
   attr_accessible :name
 end

db/migrate/20111109012514_create_content_removal_requests.rb

@@ -4,7 +4,7 @@ def self.up
       t.integer :content_removal_request_period_id
       t.integer :product_id
       t.integer :reason_id
-      t.integer :courtOrders
+      t.integer :court_orders
       t.integer :executive
       t.integer :items
       t.timestamps

db/schema.rb

@@ -51,7 +51,7 @@
     t.integer  "content_removal_request_period_id"
     t.integer  "product_id"
     t.integer  "reason_id"
-    t.integer  "courtOrders"
+    t.integer  "court_orders"
     t.integer  "executive"
     t.integer  "items"
     t.datetime "created_at"

db/seeds.rb

@@ -11,15 +11,8 @@
 
 h = Hash.from_xml(open('https://raw.github.com/ChokePointProject/cpp-web/master/tmp/countryInfo.xml'))['geonames']['country']
 
-# 1st sweep: continents
 h.each do |c|
-  Continent.find_or_create_by_code_and_name(c['continent'], c['continentName'])
-end
-
-# 2nd sweep: countries
-h.each do |c|
-  Country.create(
-    :continent_id => Continent.find_by_code(c['continent']),
+  n = Country.new(
     :code         => c['countryCode'],
     :name         => c['countryName'],
     :isoNumeric   => c['isoNumeric'],
@@ -36,6 +29,7 @@
     :bBoxEast     => c['bBoxEast'],
     :bBoxSouth    => c['bBoxSouth']
   )
-
+  n.continent = Continent.find_or_create_by_code_and_name(c['continent'], c['continentName'])
+  n.save!
 end
 

lib/tasks/load_transparencyreport_data.rake

@@ -2,9 +2,9 @@ namespace :load_transparencyreport_data do
   require 'csv' 
   require 'open-uri' 
 
-  desc "Loading transparencyreport data"
+  desc "Loading transparencyreport data: content removal request"
   task :load_content_removal_requests => :environment do
-    #csv_text = File.read('https://www.google.com/transparencyreport/governmentrequests/google-content-removal-requests-by-product-and-reason.csv')
+    # Read in XML data
     c = CSV.parse(open('https://www.google.com/transparencyreport/governmentrequests/google-content-removal-requests.csv'), :headers => true)
     c.each do |r|
       period_end = Date.strptime(r[0], '%m/%d/%Y')
@@ -13,14 +13,61 @@ namespace :load_transparencyreport_data do
         :period_end => period_end,
         :percentage_complied => r[4]
       )
-      #puts r[2], Country.find_by_code(r[2])
       e.country_id = Country.find_by_code(r[2]).id
       e.save!
     end
   end
 
+  desc "Loading transparencyreport data: content removal request by product"
+  task :load_content_removal_requests_by_product => :environment do
+    # Create reason 'undefined' (since this dataset doesn't contain reasons)
+    undefined = Reason.find_or_create_by_name('Undefined')
+    # Read in XML data
+    c = CSV.parse(open('https://www.google.com/transparencyreport/governmentrequests/google-content-removal-requests-by-product.csv'), :headers => true)
+    c.each do |r|
+      # Only add entries prior from 2010-12-31
+      if Date.strptime(r[0], '%m/%d/%Y') < Date.new(2010,12,31)
+        e = ContentRemovalRequest.new(
+          :court_orders => r[4],
+          :executive => r[5],
+          :items => r[6]
+        )
+        e.content_removal_request_period = ContentRemovalRequestPeriod.joins(:country).where('countries.code' => r[2]).find_by_period_end(Date.strptime(r[0], '%m/%d/%Y'))
+        e.product = Product.find_or_create_by_name(r[3])
+        e.reason = undefined
+        e.save!
+      end
+    end
+  end
+
+  desc "Loading transparencyreport data: content removal request by product and reason"
+  task :load_content_removal_requests_by_product_and_reason => :environment do
+    # Read in XML data
+    c = CSV.parse(open('https://www.google.com/transparencyreport/governmentrequests/google-content-removal-requests-by-product-and-reason.csv'), :headers => true)
+    c.each do |r|
+      period = ContentRemovalRequestPeriod.joins(:country).where('countries.code' => r[2]).find_by_period_end(Date.strptime(r[0], '%m/%d/%Y'))
+      # Delete superseded data
+      #puts period.content_removal_requests.joins(:product).where('products.name' => r[3]).count
+      #period.content_removal_requests.joins(:product).where('products.name' => r[3]).delete_all
+      # Create new data
+      e = ContentRemovalRequest.new(
+        :court_orders => r[5],
+        :executive => r[6],
+        :items => r[7]
+      )
+      e.content_removal_request_period = period
+      e.product = Product.find_or_create_by_name(r[3])
+      e.reason = Reason.find_or_create_by_name(r[4])
+      e.save!
+    end
+  end
+
   desc "Run all transparencyreport data tasks"
-  task :all => [:load_content_removal_requests]
+  task :all => [
+    :load_content_removal_requests,
+    :load_content_removal_requests_by_product,
+    :load_content_removal_requests_by_product_and_reason
+  ]
 
 end
 

test/fixtures/content_removal_requests.yml

@@ -2,14 +2,14 @@ one:
   content_removal_request_period_id: 1
   product_id: 1
   reason_id: 1
-  courtOrders: 1
+  court_orders: 1
   executive: 1
   items: 1
 
 two:
   content_removal_request_period_id: 1
   product_id: 1
   reason_id: 1
-  courtOrders: 1
+  court_orders: 1
   executive: 1
   items: 1