Merge branch 'master' into update_npm_crawler

Author: reiz

Gemfile

@@ -1,6 +1,6 @@
 source "http://rubygems.org"
 
-gem 'versioneye-core', :git => 'https://github.com/versioneye/versioneye-core.git', :tag => 'v12.8.0'
+gem 'versioneye-core', :git => 'https://github.com/versioneye/versioneye-core.git', :tag => 'v12.8.4'
 #gem 'versioneye-core'    , :path => "../versioneye-core"
 
 gem 'rufus-scheduler', '3.4.2'

Gemfile.lock

@@ -1,9 +1,9 @@
 GIT
   remote: https://github.com/versioneye/versioneye-core.git
-  revision: d46cd326d5eeab115f56bad5df5632228a87b5d3
-  tag: v12.8.0
+  revision: 0aedeaefa11a4cd159713f827af96f26673cfd5b
+  tag: v12.8.4
   specs:
-    versioneye-core (12.8.0)
+    versioneye-core (12.8.4)
       actionmailer (~> 5.1.0)
       aws-sdk (~> 2.10.7)
       bunny (~> 2.7.0)
@@ -64,13 +64,13 @@ GEM
       public_suffix (~> 2.0, >= 2.0.2)
     amq-protocol (2.2.0)
     ansi (1.5.0)
-    aws-sdk (2.10.16)
-      aws-sdk-resources (= 2.10.16)
-    aws-sdk-core (2.10.16)
+    aws-sdk (2.10.17)
+      aws-sdk-resources (= 2.10.17)
+    aws-sdk-core (2.10.17)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.10.16)
-      aws-sdk-core (= 2.10.16)
+    aws-sdk-resources (2.10.17)
+      aws-sdk-core (= 2.10.17)
     aws-sigv4 (1.0.1)
     bson (4.2.2)
     builder (3.2.3)

lib/versioneye/crawl.rb

@@ -106,14 +106,14 @@ def self.parse_url url_text
     end
 
     def self.fetch url
-     HTTParty.get url, { timeout: 5 }
+      HTTParty.get url, { timeout: 20 }
     rescue
       logger.error "failed to fetch data from #{url}"
       nil
     end
 
 
-    def self.fetch_json( url, ttl = 5, symbolize = true)
+    def self.fetch_json( url, ttl = 20, symbolize = true)
       res = Timeout::timeout(ttl) { HTTParty.get(url) }
       if res.code != 200
         self.logger.error "Failed to fetch JSON doc from: #{url} - #{res}"
@@ -127,7 +127,7 @@ def self.fetch_json( url, ttl = 5, symbolize = true)
       nil
     end
 
-    def self.post_json(url, options, ttl = 5)
+    def self.post_json(url, options, ttl = 20)
       res = Timeout::timeout(ttl) { HTTParty.post(url, options) }
       if res.code > 205
         logger.error "Failed to post data to the url: #{url}, #{res.code} - #{res.message}\n#{options}"

lib/versioneye/crawlers/crates_crawler.rb

@@ -34,6 +34,15 @@ def self.crawl( api_key = nil )
   end
 
 
+  def self.recrawl
+    api_key = fetch_api_key
+    return if api_key.to_s.empty?
+
+    Product.where(language: Product::A_LANGUAGE_RUST).to_a.each do |prod|
+      crawl_package(prod[:prod_key], api_key, false)
+    end
+  end
+
   def self.crawl_product_list(api_key, page_nr = 1, per_page = 100)
     logger.info "going to crawl product list from #{page_nr}, per_page: #{per_page}"
 
@@ -86,9 +95,22 @@ def self.crawl_package(product_id, api_key = nil, ignore_existing = true)
 
   def self.process_versions( product_db, product_doc, api_key, ignore_existing )
     owners = fetch_product_owners( api_key, product_db.prod_key )
-    product_license = product_doc[:crate][:license].to_s
+    if product_doc.nil?
+      logger.error "process_versions: product json response cant be empty"
+      return
+    end
+
+    if product_doc.has_key?(:versions) == false
+      logger.error "process_versions: product json has no :verisons field\n#{product_doc}"
+    end
+
     product_doc[:versions].each do |version_doc|
       version_num = version_doc[:num].to_s.strip
+      if version_num.empty?
+        log.error "process_versions: version_doc has no :num field\n#{version_doc}"
+        next
+      end
+
       if !product_db.version_by_number( version_num ).nil? && ignore_existing
         logger.info "process_versions: #{product_db.prod_key}:#{version_num} exist already"
         next
@@ -100,7 +122,7 @@ def self.process_versions( product_db, product_doc, api_key, ignore_existing )
         next
       end
 
-      upsert_version_licenses( product_db, version_db.version, product_license )
+      upsert_version_licenses( product_db, version_db.version, version_doc[:license] )
       upsert_version_links(    product_db, version_db.version, product_doc[:crate] )
       upsert_version_archive(  product_db, version_db.version, version_doc[:dl_path] )
       upset_version_devs(      product_db, version_db, owners )
@@ -123,7 +145,7 @@ def self.crawl_dependencies( product_db, version_db, api_key )
     end
 
     logger.info "crawl_dependencies: fetching version details for #{product_db.prod_key} - #{version}"
-    dep_docs = fetch_version_dependencies( api_key, product_db[:prod_key], version )
+    dep_docs = fetch_version_dependencies( api_key, product_db.prod_key, version )
     dep_docs.to_a.each do |dep_doc|
       upsert_product_dependency(product_db, version, dep_doc)
     end
@@ -133,6 +155,11 @@ def self.crawl_dependencies( product_db, version_db, api_key )
   #-- persistance helpers
 
   def self.upsert_product(product_doc)
+    if product_doc.nil?
+      logger.error "upsert_product: API response had no :crate subdocument"
+      return
+    end
+
     prod_key    = product_doc[:id].to_s.strip
     prod_key_dc = prod_key.downcase
     product_db = Product.where(
@@ -216,6 +243,11 @@ def self.upsert_product_owner(product_db, owner_doc, version_num)
 
 
   def self.upsert_version_licenses(product_db, version_label, license_label)
+    if license_label.nil? or license_label.empty?
+      logger.error "upsert_version_licenses: missing license of #{product_db}/#{version_label}"
+      return
+    end
+
     licenses = license_label.to_s.strip.split('/')
     licenses.to_a.each do |license|
       self.upsert_version_license(product_db, version_label, license)
@@ -228,12 +260,12 @@ def self.upsert_version_licenses(product_db, version_label, license_label)
   def self.upsert_version_license(product_db, version_label, license_name)
     license_name = license_name.to_s.strip
 
-    lic_db = License.where(
+    lic_db = License.find_or_create_by(
       language: product_db[:language],
       prod_key: product_db[:prod_key],
       version: version_label,
       name: license_name
-    ).first_or_create
+    )
 
     lic_db.update(source: 'crates')
     lic_db.save
@@ -243,14 +275,6 @@ def self.upsert_version_license(product_db, version_label, license_name)
 
 
   def self.upsert_product_dependency(product_db, version_id, dep_doc)
-    dep_db = Dependency.where(
-      prod_type: A_TYPE_CARGO,
-      language: product_db.language,
-      prod_key: product_db.prod_key,
-      prod_version: version_id,
-      dep_prod_key: dep_doc[:crate_id],
-    ).first_or_create
-
     scope = if dep_doc[:optional]
               Dependency::A_SCOPE_OPTIONAL
             elsif dep_doc[:target] == 'test'
@@ -259,12 +283,18 @@ def self.upsert_product_dependency(product_db, version_id, dep_doc)
               Dependency::A_SCOPE_COMPILE
             end
 
-    dep_db.update(
+    dep_db = Dependency.find_or_create_by(
+      prod_type: A_TYPE_CARGO,
+      language: product_db.language,
+      prod_key: product_db.prod_key,
+      prod_version: version_id,
+
+      dep_prod_key: dep_doc[:crate_id],
       version: dep_doc[:req],
-      name: dep_doc[:crate_id],
       scope: scope
     )
-
+    dep_db.name = dep_doc[:crate_id]
+    dep_db.save
     dep_db
   end
 
@@ -284,12 +314,12 @@ def self.upsert_version_links(product_db, version_id, product_doc)
 
 
   def self.upsert_version_link(product_db, version_id, name, url)
-    url_db = Versionlink.where(
+    url_db = Versionlink.find_or_create_by(
       language: product_db[:language],
       prod_key: product_db[:prod_key],
       version_id: version_id,
       link: url.to_s.strip
-    ).first_or_create
+    )
 
     url_db.update(name: name.to_s.strip)
     url_db.save
@@ -301,12 +331,12 @@ def self.upsert_version_archive(product_db, version_id, dl_path)
     pkg_name = "#{product_db[:prod_key]}-#{version_id}.crate"
     url = "#{API_HOST}/#{dl_path}"
 
-    url_db = Versionarchive.where(
+    url_db = Versionarchive.find_or_create_by(
       language: product_db[:language],
       prod_key: product_db[:prod_key],
       version_id: version_id,
       name: pkg_name
-    ).first_or_create
+    )
 
     url_db.update(link: url)
     url_db.save

lib/versioneye/crawlers/nuget_crawler.rb

@@ -8,6 +8,7 @@ class NugetCrawler < Versioneye::Crawl
   A_PROFILE_URL     = "https://www.nuget.org/profiles"
   A_LANGUAGE_CSHARP = Product::A_LANGUAGE_CSHARP
   A_TYPE_NUGET      = Project::A_TYPE_NUGET
+  A_TIMEOUT         = 30 # seconds
 
 
   def self.logger
@@ -46,7 +47,7 @@ def self.crawl_last_x_days( x_days = 10 )
   # otherwise will only crawl catalogs published on the date_txt
   # date_txt format: YYYY-mm-dd or any other format supported by DateTime.parse
   def self.crawl(date_txt = nil)
-    catalog = self.fetch_json("#{A_NUGET_URL}#{A_CATALOG_PATH}")
+    catalog = self.fetch_json("#{A_NUGET_URL}#{A_CATALOG_PATH}", A_TIMEOUT)
     if catalog.nil?
       self.logger.error "crawl: Failed to fetch the Nuget catalog."
       return nil
@@ -80,7 +81,7 @@ def self.crawl_catalog_page(the_page)
 
     self.logger.info "Crawling catalog page: #{the_page[:@id]} - items: #{the_page[:count]}"
 
-    page_items = fetch_json the_page[:@id]
+    page_items = fetch_json( the_page[:@id], A_TIMEOUT )
     if page_items.nil?
       logger.warn "crawl_catalog_page: failed to fetch items on the catalog page: #{the_page}"
       return
@@ -95,7 +96,7 @@ def self.crawl_package( the_package )
       return
     end
 
-    doc = fetch_json the_package[:@id]
+    doc = fetch_json( the_package[:@id], A_TIMEOUT)
     if doc.nil?
       logger.warn "crawl_package: failed to fetch package details from: #{the_package}"
       return
@@ -221,6 +222,15 @@ def self.create_new_version(product, product_doc)
       release_dt = parse_date_string( publish_date_label )
     end
 
+    #if package has no release or created date - then use commit Date
+    if release_dt.nil?
+      publish_date_label = product_doc[:'catalog:commitTimeStamp']
+      release_dt = parse_date_string(publish_date_label)
+
+      logger.warn "create_new_version: no `created` or `published` fields\n#{product}\n#{product_doc}"
+      logger.warn "\twill use commitTimeStamp instead - #{publish_date_label}"
+    end
+
     version_db = Version.new({
       version: product_doc[:version],
       released_at: release_dt,
@@ -362,15 +372,33 @@ def self.upsert_artefact(product, version)
 
 
   def self.upsert_artefact_sha(product, version, sha, sha_method)
-    artefact = Artefact.find_or_create_by(
-                  :language   => product.language,
-                  :prod_key   => product.prod_key,
-                  :version    => version,
-                  :prod_type  => product.prod_type,
-                  :packaging  => 'nupkg',
-                  :sha_value  => sha,
-                  :sha_method => sha_method )
+    artefact = Artefact.where(sha_value: sha).first
+    if artefact and artefact[:prod_key] != product[:prod_key]
+      logger.error "#-- CRITICAL ERROR ---------------------------------------------------"
+      logger.error "upsert_artefact_sha: sha `#{sha}` belongs to other product #{artefact}"
+      logger.error "\tcurrent_product: #{product} => #{version}, #{sha_method}"
+      return false
+    end
+
+    artefact ||= Artefact.new(
+      :language   => product.language,
+      :prod_key   => product.prod_key,
+      :prod_type  => product.prod_type
+    )
+
+    artefact.update(
+      version: version.to_s.strip,
+      packaging: 'nupkg',
+      sha_value: sha,
+      sha_method: sha_method
+    )
+
+
     artefact.save
+  rescue => e
+    logger.error "upsert_artefact_sha: failed to save #{product} sha #{sha_method}:#{sha}"
+    logger.error "\treason: #{e.message}"
+    false
   end