Use device detector to filter bots

.travis.yml

@@ -1,6 +1,6 @@
 language: ruby
 before_install:
-  - gem install bundler
+  - gem install bundler -v 1.17.3
 before_script:
   - cd tests/test_app
   - bundle exec rails g impressionist -f

Gemfile

@@ -10,7 +10,7 @@ platforms :jruby do
 end
 
 platforms :ruby, :mswin, :mingw do
-  gem 'sqlite3'
+  gem 'sqlite3', '~> 1.3.13'
 end
 
 group :test do

README.md

@@ -22,9 +22,11 @@ reporting yet.. this thingy just creates the data.
 
 What about bots?
 ----------------
-They are ignored.  1200 known bots have been added to the ignore list as of
-February 1, 2011.  Impressionist uses this list:
-http://www.user-agents.org/allagents.xml
+They are ignored by using [Device Detector's](https://rubygems.org/gems/device_detector/versions/0.2.0) user agent database.
+
+https://github.com/podigee/device_detector/blob/develop/regexes/bots.yml
+
+You can also get a list of bots identified, by running `rake impressionist:bots`
 
 Installation
 ------------

Rakefile

@@ -16,7 +16,7 @@ task :default => [:test, :test_app]
 namespace :impressionist do
   require File.dirname(__FILE__) + "/lib/impressionist/bots"
 
-  desc "output the list of bots from http://www.user-agents.org/"
+  desc "output the list of bots from https://github.com/podigee/device_detector"
   task :bots do
     p Impressionist::Bots.consume
   end

gemfiles/rails32.gemfile

@@ -10,7 +10,7 @@ platforms :jruby do
 end
 
 platforms :ruby, :mswin, :mingw do
-  gem 'sqlite3'
+  gem 'sqlite3', '~> 1.3.13'
 end
 
 group :test do

gemfiles/rails40.gemfile

@@ -10,7 +10,7 @@ platforms :jruby do
 end
 
 platforms :ruby, :mswin, :mingw do
-  gem 'sqlite3'
+  gem 'sqlite3', '~> 1.3.13'
 end
 
 group :test do

impressionist.gemspec

@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
   s.require_path  = 'lib'
   s.required_rubygems_version = Gem::Requirement.new('>= 1.3.6') if s.respond_to? :required_rubygems_version=
 
-  s.add_dependency 'nokogiri', RUBY_VERSION < '2.1.0' ? '~> 1.6.0' : '~> 1'
+  s.add_dependency 'device_detector', '~> 1.0'
   s.add_development_dependency 'bundler', '~> 1.0'
 end

lib/impressionist/bots.rb

@@ -1,21 +1,26 @@
 require 'timeout'
 require 'net/http'
-require 'nokogiri'
+require 'yaml'
 
 module Impressionist
   module Bots
-    LIST_URL = "http://www.user-agents.org/allagents.xml"
+    LIST_URL = "https://raw.githubusercontent.com/podigee/device_detector/develop/regexes/bots.yml"
     def self.consume
       Timeout.timeout(4) do
         response = Net::HTTP.get(URI.parse(LIST_URL))
-        doc = Nokogiri::XML(response)
-        list = []
-        doc.xpath('//user-agent').each do |agent|
-          type = agent.xpath("Type").text
-          list << agent.xpath("String").text.gsub("&lt;","<") if ["R","S"].include?(type) #gsub hack for badly formatted data
-        end
-        list
+        list = YAML.safe_load(response)
+        puts list.map { |entry| entry_details(entry) }
       end
     end
+
+    def self.entry_details(entry)
+      name = entry['name']
+      regex = entry['regex']
+      producer = entry['producer']
+
+      result = "\nAgent: #{name}\nRegex: #{regex}\n"
+      result += "Producer: #{producer['name']} - #{producer['url']}" if producer
+      result
+    end
   end
 end

tests/test_app/Gemfile

@@ -14,7 +14,7 @@ end
 
 platforms :ruby, :mswin, :mingw do
   gem 'pg'
-  gem 'sqlite3'
+  gem 'sqlite3', '~> 1.3.13'
   gem 'mysql2'
   gem 'mongoid', '~> 3.1'
   gem 'bson_ext'

tests/test_app/spec/models/bots_spec.rb

@@ -4,11 +4,12 @@
 
   describe "self.bot?" do
     it "is true if user_agent is matches wild card" do
-      Impressionist::Bots.bot?("google.com bot").should be_true
+      Impressionist::Bots.bot?("bot").should be_true
+      Impressionist::Bots.bot?("crawler").should be_true
     end
 
     it "is true if user_agent is on bot list" do
-      Impressionist::Bots.bot?("A-Online Search").should be_true
+      Impressionist::Bots.bot?("acoonbot").should be_true
     end
 
     it "is false if user_agent is blank" do
@@ -24,4 +25,4 @@
       Impressionist::Bots.bot?.should be_false
     end
   end
-end
+end