fix security vulnerabilities

Author: tom statter

datashift.gemspec

@@ -22,11 +22,11 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   s.add_runtime_dependency 'rails', '>= 4.2', '~> 5.2'
-  s.add_runtime_dependency 'thor', '<= 0.20', '>= 0.18'
+  s.add_runtime_dependency 'thor', '>= 0.19.0', '< 2.0'
   s.add_runtime_dependency 'paperclip', '>= 4.3', '<= 6.0.0'
   s.add_runtime_dependency 'spreadsheet', '~> 1.1'
 
-  s.add_runtime_dependency 'rubyzip', '>= 0.9.9', '< 1.3'
+  s.add_dependency "rubyzip", ">= 1.3.0"
 
   s.add_runtime_dependency 'erubis', '~> 2.7', '>= 2.7.0'
   s.add_runtime_dependency 'thread_safe', '~> 0.3', '>= 0.3'

lib/datashift/generators/config_generator.rb

@@ -20,8 +20,8 @@ def self.title
 
     attr_writer :import_template, :export_template
 
-    def initialize
-      super
+    def initialize(config: nil)
+      super(config: configuration)
     end
 
     def import_template

lib/datashift/generators/csv_generator.rb

@@ -11,8 +11,8 @@ class CsvGenerator < GeneratorBase
 
     attr_accessor :file_name
 
-    def initialize
-      super
+    def initialize(config: nil)
+      super(config: config)
     end
 
     def generate(file_name, klass)

lib/datashift/generators/excel_generator.rb

@@ -18,22 +18,24 @@ class ExcelGenerator < GeneratorBase
     attr_accessor :excel
     attr_accessor :file_name
 
-    def initialize
-      super
+    def initialize(config: nil)
+      super(config: config)
     end
 
     # Create an Excel file template (header row) representing supplied Model
     # file_name => Filename for generated template
     #
     # See DataShift::Exporters::Configuration for options
     #
-    def generate(file_name, klass, options = {})
+    def generate(file_name, klass, associations: false, options: {})
 
       @file_name = file_name
 
       start_excel(klass, options)
 
-      @headers = Headers.klass_to_headers(klass)
+      @config.with = :all if associations
+
+      @headers = Headers.klass_to_headers(klass, config: config)
 
       @excel.set_headers(@headers)
 

lib/datashift/generators/generator_base.rb

@@ -10,25 +10,19 @@ class GeneratorBase
 
     include DataShift::Logging
 
-    attr_accessor :configuration
+    attr_accessor :config
 
-    def initialize; end
+    def initialize(config: nil)
+      @config = config || DataShift::Configuration.call
+    end
 
     # Prepare to generate with associations but then
     # calls a **derived generate** method i.e abstract to this base class
     #
     # file_name => Filename for generated template
     #
     def generate_with_associations(file_name, klass)
-
-      state = DataShift::Configuration.call.with
-
-      DataShift::Configuration.call.with = :all
-
-      generate(file_name, klass)
-    ensure
-      DataShift::Configuration.call.with = state
-
+      generate(file_name, klass, associations: true)
     end
 
   end

lib/datashift/headers.rb

@@ -54,13 +54,13 @@ def index(header)
     #
     class << self
 
-      def klass_to_operators(klass)
+      def klass_to_operators(klass, config: DataShift::Configuration.call)
 
-        headers = Headers.new(klass)
+        headers = Headers.new(klass, config: config)
 
         headers.class_source_to_headers
 
-        DataShift::Transformation::Remove.new.unwanted_headers(headers)
+        DataShift::Transformation::RemoveUnwantedHeaders.call(headers, config: config)
 
         headers
       end
@@ -74,20 +74,18 @@ def klass_to_operators(klass)
     # These can be used to infer an operator to call from an inbound header
     # or provide mapping to an internal method from an external header
     #
-    def class_source_to_operators
+    def class_source_to_operators(config: DataShift::Configuration.call)
 
       raise SourceIsNotAClass, 'Cannot parse source for headers - source must be a Class' unless source.is_a?(Class)
 
       # TODO: This collection can now be sorted
       collection = ModelMethods::Manager.catalog_class(source)
 
-      configuration = DataShift::Configuration.call
-
       if collection
         collection.each do |mm|
           next if(DataShift::Transformation::Remove.new.association?(mm))
 
-          next unless configuration.op_type_in_scope?(mm)
+          next unless config.op_type_in_scope?(mm)
           if(mm.association_type?)
             association_to_headers(mm)
           else
@@ -100,11 +98,9 @@ def class_source_to_operators
 
     alias class_source_to_headers class_source_to_operators
 
-    def association_to_headers( model_method )
-
-      configuration = DataShift::Configuration.call
+    def association_to_headers( model_method, config: DataShift::Configuration.call)
 
-      if(configuration.expand_associations)
+      if(config.expand_associations)
         model_method.association_columns.each do |c|
           add "#{model_method.operator}::#{c.name}"
         end

lib/datashift/loaders/loader_base.rb

@@ -139,10 +139,6 @@ def configure_from(yaml_file, klass = nil, locale_key = 'data_flow_schema')
 
       logger.info("Reading Datashift loader config from: #{yaml_file.inspect}")
 
-      data = Configuration.parse_yaml(yaml_file)
-
-      logger.info("Read Datashift config: #{data.inspect}")
-
       @binder ||= DataShift::Binder.new
 
       data_flow_schema = DataShift::DataFlowSchema.new
@@ -157,6 +153,24 @@ def configure_from(yaml_file, klass = nil, locale_key = 'data_flow_schema')
       logger.info("Loader Options : #{@config.inspect}")
     end
 
+    def configure_from_yaml(yaml, klass: nil, locale_key: 'data_flow_schema')
+
+      setup_load_class(klass) if(klass)
+
+      @binder ||= DataShift::Binder.new
+
+      data_flow_schema = DataShift::DataFlowSchema.new
+
+      # Includes configuring DataShift::Transformation
+      nodes = data_flow_schema.prepare_from_yaml(yaml, locale_key)
+
+      @binder.add_bindings_from_nodes( nodes )
+
+      PopulatorFactory.configure_from_yaml(load_object_class, yaml)
+
+      logger.info("Loader Options : #{@config.inspect}")
+    end
+
   end
 
 end

lib/datashift/node_context.rb

@@ -45,12 +45,8 @@ def next_update?
     def process
       populator.prepare_and_assign(self, doc_context.load_object, data)
     rescue StandardError => x
-      failed = FailureData.new( doc_context.load_object, self, x.message)
-
+      #failed = FailureData.new( doc_context.load_object, self, x.message)
       failed.error_messages <<  "Failed to process node : #{method_binding.pp}"
-
-      doc_context.progress_monitor.failure(failed)
-
       logger.error("#{x.backtrace.first} : #{x.message}")
       raise x
     end

lib/datashift/populators/populator_factory.rb

@@ -24,14 +24,17 @@ def self.populators
     #    populator
     #
     def self.configure(load_object_class, yaml_file)
+      configure_from_yaml(load_object_class, Configuration.parse_yaml(yaml_file))
+    end
 
-      @config = Configuration.parse_yaml(yaml_file)
+    def self.configure_from_yaml(load_object_class, yaml)
+      @config = yaml
 
       if @config[:datashift_populators]
         @config[:datashift_populators].each do |_operator, type|
           populator = ::Object.const_get(type).new
 
-          populator.configure_from(load_object_class, yaml_file)
+          populator.configure_from_yaml(load_object_class, yaml)
 
           populators[@config[:datashift_populators]]
         end

lib/datashift/progress_monitor.rb

@@ -52,7 +52,6 @@ def success(reportable_object)
     # so the load object is invalid
 
     def failure(failure_data)
-
       @current_status = :failure
 
       logger.error 'Failure(s) reported :'
@@ -64,17 +63,21 @@ def failure(failure_data)
     end
 
     def add_loaded_object(object)
+      return if object.nil? || @loaded_objects.include?(object)
+
       @success_inbound_count += 1
       @processed_object_count += 1
 
-      @loaded_objects << object.id unless object.nil? || @loaded_objects.include?(object)
+      @loaded_objects << object
     end
 
     def add_failed_object(object)
+      return if object.nil? || @failed_objects.include?(object)
+
       @failed_inbound_count += 1
       @processed_object_count += 1
 
-      @failed_objects << object unless object.nil? || @failed_objects.include?(object)
+      @failed_objects << object
     end
 
     # The database objects created or rejected

lib/datashift/transformation/remove.rb

@@ -10,15 +10,25 @@ module DataShift
 
   module Transformation
 
+    class RemoveUnwantedHeaders
+      def self.call(headers, config: DataShift::Configuration.call)
+        Remove.new(config: config).unwanted_headers(headers)
+      end
+    end
+
     class Remove
 
+      def initialize(config: DataShift::Configuration.call)
+        @config = config
+      end
+
       def remove_list
-        @remove_list ||= DataShift::Configuration.call.prep_remove_list
+        @remove_list ||= @config.prep_remove_list
       end
 
       def association?(mm)
         return false unless(mm.association_type?)
-        (DataShift::Configuration.call.exclude_associations & [mm.operator, mm.operator.to_sym]).present?
+        (@config.exclude_associations & [mm.operator, mm.operator.to_sym]).present?
       end
 
       # Specify columns to remove via DataShift::Configuration