add licence matcher by url; tuned li filter for AWS pages;

Author: timgluz

.gitignore

@@ -51,4 +51,3 @@ pkg
 log/*.log
 scripts/set_vars_for_dev_reiz.sh
 tmp/*
-Gemfile.lock

Gemfile.lock

@@ -0,0 +1,290 @@
+GIT
+  remote: https://github.com/versioneye/versioneye-core.git
+  revision: 0d56a9b3262020916020d87d70b455bd794aa56f
+  tag: v9.5.24
+  specs:
+    versioneye-core (9.5.24)
+      actionmailer (~> 4.2.6)
+      aws-sdk (~> 2.4.0)
+      bundler (~> 1.12.5)
+      bunny (~> 2.4.0)
+      cocoapods-core (~> 0.39.0)
+      dalli (~> 2.7.0)
+      etcd (~> 0.3.0)
+      httparty (~> 0.13.1)
+      mongoid (~> 5.1.0)
+      naturalsorter (~> 3.0.14)
+      net-ldap (~> 0.15.0)
+      nokogiri (~> 1.6.8)
+      oauth (~> 0.5.0)
+      octokit (~> 4.3.0)
+      pdfkit (~> 0.8.0)
+      persistent_httparty (~> 0.1.0)
+      semverly (~> 1.0.0)
+      stripe (~> 1.48.0)
+      tire (~> 0.6.2)
+      will_paginate_mongoid (= 2.0.1)
+      wkhtmltopdf-binary (~> 0.9.9.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionmailer (4.2.7)
+      actionpack (= 4.2.7)
+      actionview (= 4.2.7)
+      activejob (= 4.2.7)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.7)
+      actionview (= 4.2.7)
+      activesupport (= 4.2.7)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.7)
+      activesupport (= 4.2.7)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activejob (4.2.7)
+      activesupport (= 4.2.7)
+      globalid (>= 0.3.0)
+    activemodel (4.2.7)
+      activesupport (= 4.2.7)
+      builder (~> 3.1)
+    activesupport (4.2.7)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.4.0)
+    amq-protocol (2.0.1)
+    ansi (1.5.0)
+    aws-sdk (2.4.3)
+      aws-sdk-resources (= 2.4.3)
+    aws-sdk-core (2.4.3)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.4.3)
+      aws-sdk-core (= 2.4.3)
+    bson (4.1.1)
+    builder (3.2.2)
+    bunny (2.4.0)
+      amq-protocol (>= 2.0.1)
+    capybara (2.7.1)
+      addressable
+      mime-types (>= 1.16)
+      nokogiri (>= 1.3.3)
+      rack (>= 1.0.0)
+      rack-test (>= 0.5.4)
+      xpath (~> 2.0)
+    cocoapods-core (0.39.0)
+      activesupport (>= 4.0.2)
+      fuzzy_match (~> 2.0.4)
+      nap (~> 1.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    dalli (2.7.6)
+    database_cleaner (1.5.3)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.20160615)
+      unf (>= 0.0.5, < 1.0.0)
+    erubis (2.7.0)
+    etcd (0.3.0)
+      mixlib-log
+    factory_girl (4.7.0)
+      activesupport (>= 3.0.0)
+    fakeweb (1.3.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    fuzzy_match (2.0.4)
+    gene_pool (1.4.1)
+      thread_safe
+    git (1.3.0)
+    github_api (0.14.4)
+      addressable (~> 2.4.0)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      oauth2 (~> 1.0.0)
+    globalid (0.3.7)
+      activesupport (>= 4.1.0)
+    hashdiff (0.3.0)
+    hashie (3.4.4)
+    hashr (0.0.22)
+    highline (1.7.8)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    httparty (0.13.7)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    i18n (0.7.0)
+    jeweler (2.1.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+      semver
+    jmespath (1.3.1)
+    json (1.8.3)
+    jwt (1.5.4)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.4)
+      mime-types (>= 1.16, < 4)
+    mime-types (2.99.2)
+    mini_portile2 (2.1.0)
+    minitest (5.9.0)
+    mixlib-log (1.6.0)
+    mongo (2.2.7)
+      bson (~> 4.0)
+    mongoid (5.1.3)
+      activemodel (~> 4.0)
+      mongo (~> 2.1)
+      origin (~> 2.2)
+      tzinfo (>= 0.3.37)
+    multi_json (1.12.1)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nap (1.1.0)
+    narray (0.6.1.2)
+    naturalsorter (3.0.14)
+    net-ldap (0.15.0)
+    netrc (0.11.0)
+    nokogiri (1.6.8)
+      mini_portile2 (~> 2.1.0)
+      pkg-config (~> 1.1.7)
+    oauth (0.5.1)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    octokit (4.3.0)
+      sawyer (~> 0.7.0, >= 0.5.3)
+    origin (2.2.0)
+    pdfkit (0.8.2)
+    persistent_http (1.0.6)
+      gene_pool (>= 1.3)
+    persistent_httparty (0.1.2)
+      httparty (~> 0.9)
+      persistent_http (< 2)
+    pkg-config (1.1.7)
+    rack (1.6.4)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.7)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6.0)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    rake (11.2.2)
+    rdoc (4.2.2)
+      json (~> 1.4)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.1)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    rspec_junit_formatter (0.2.3)
+      builder (< 4)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rufus-scheduler (3.2.1)
+    safe_yaml (1.0.4)
+    sawyer (0.7.0)
+      addressable (>= 2.3.5, < 2.5)
+      faraday (~> 0.8, < 0.10)
+    semver (1.0.1)
+    semverly (1.0.0)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.1)
+    shoulda-matchers (2.8.0)
+      activesupport (>= 3.0.0)
+    simplecov (0.12.0)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    stripe (1.48.0)
+      rest-client (>= 1.4, < 3.0)
+    tf-idf-similarity (0.1.5)
+      unicode_utils (~> 1.4)
+    thread_safe (0.3.5)
+    tire (0.6.2)
+      activemodel (>= 3.0)
+      activesupport
+      ansi
+      hashr (~> 0.0.19)
+      multi_json (~> 1.3)
+      rake
+      rest-client (~> 1.6)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.2)
+    unicode_utils (1.4.0)
+    vcr (3.0.3)
+    webmock (2.1.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    will_paginate (3.1.0)
+    will_paginate_mongoid (2.0.1)
+      mongoid
+      will_paginate (~> 3.0)
+    wkhtmltopdf-binary (0.9.9.3)
+    xpath (2.0.0)
+      nokogiri (~> 1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.12.0)
+  capybara (~> 2.7.1)
+  database_cleaner (~> 1.5.1)
+  factory_girl (~> 4.7.0)
+  fakeweb (~> 1.3.0)
+  jeweler (~> 2.1.1)
+  narray (= 0.6.1.2)
+  rdoc (~> 4.2.0)
+  rspec (~> 3.5.0)
+  rspec_junit_formatter (= 0.2.3)
+  rufus-scheduler (= 3.2.1)
+  shoulda
+  simplecov (~> 0.12.0)
+  tf-idf-similarity (= 0.1.5)
+  vcr (~> 3.0.1)
+  versioneye-core!
+  webmock (~> 2.1.0)
+
+BUNDLED WITH
+   1.12.5

data/licenses.json

@@ -40,8 +40,25 @@ def self.crawl_unidentified_urls(language)
     licenses = License.where(language: language, spdx_identifier: nil)
     licenses.to_a.each do |lic_db|
       n += 1
-      updated_lic = crawl_license_file(lic_matcher, lic_db)
-      failed += 1 if updated_lic.nil?
+			#first try to match by url without doing http request
+			spdx_id, score = lic_matcher.match_url(lic_db[:url])
+			if spdx_id.nil?
+				logger.info "crawl_unidentified_urls: going to fetch license text from #{lic_db[:url]}"
+	      spdx_id, score = crawl_license_file(lic_matcher, lic_db)
+			else
+				logger.info "crawl_unidentified_urls: found match by url - #{spdx_id}, #{lic_db[:url]}"
+			end
+
+			if spdx_id.nil?
+				logger.warn "crawl_unidentified_urls: detect no license for #{lic_db[:url]}"
+				failed += 1
+				next
+			end
+
+			lic_db.update(
+				spdx_identifier: spdx_id,
+				name: spdx_id
+			)
     end
 
     logger.info "crawl_unidentified_urls: done! crawled #{n} licenses, skipped: #{failed}"
@@ -53,14 +70,14 @@ def self.crawl_license_file(lic_matcher, lic_db)
     url = lic_db[:url].to_s.strip
     if url.empty?
       logger.warn "crawl_license_file: no url for #{prod_id}."
-      return
+      return []
     end
 
     res = fetch url
     if res.nil? or res.code != 200
       logger.error "crawl_license_file: failed to read #{prod_id} data from: #{url} - #{res.try(:code)}"
       lic_db.update(name: 'unknown')
-      return
+      return []
     end
 
     matches = case res.headers["content-type"]
@@ -76,21 +93,17 @@ def self.crawl_license_file(lic_matcher, lic_db)
     best_match = matches.to_a.first
     if best_match.nil? or best_match.empty?
       logger.warn "crawl_license_file: found no match for #{prod_id}, #{url}"
-      return
+      return []
     end
 
     if best_match.last < A_MIN_SCORE_CONFIDENCE
       logger.warn "crawl_license_file: #{prod_id} best match had too low score #{best_match}, #{url}"
-      return
+      return []
     end
 
     spdx_id, score = best_match
-    if url =~ /mit/i and spdx_id != 'MIT'
-      logger.error "crawl_license_file: didnt detect MIT for #{prod_id}, #{matches}, #{url}"
-      spdx_id, score = 'MIT', 1.0
-    end
 
-    #TODO: if these special cases gets bigger -> refactor own method or change solution
+    #TODO: if these special cases gets bigger -> refactor to License.match_url
     case url
     when 'http://jquery.org/license'
       spdx_id, score = 'MIT', 1.0 #Jquery license page doesnt include any license text
@@ -99,12 +112,7 @@ def self.crawl_license_file(lic_matcher, lic_db)
     end
 
     logger.debug "crawl_license_file: best match for #{prod_id} => #{spdx_id}:#{score} #{url}"
-    lic_db.update(
-      spdx_identifier: spdx_id,
-      name: spdx_id
-    )
-
-    return lic_db
+    [spdx_id, score]
   end