Start sorting out configuration mess

Author: Thomas Statter

datashift.thor

@@ -75,6 +75,11 @@ module Datashift
         exit(-1)
       end
 
+      if options[:bump] && options[:bump] !~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/
+        puts 'ERROR: bump should be a valid numeric version in form x.x.x'
+        exit(-1)
+      end
+
       version = options[:bump] || DataShift::VERSION
 
       # Bump the VERSION file in library

lib/datashift/column_packer.rb

@@ -43,6 +43,8 @@ def record_to_column(record, json = false)
 
       return '' if record.nil? || (record.respond_to?(:each) && record.empty?)
 
+      # :only and :except options can be used to limit the attributes included
+
       return record.to_json if json # packs associations into single column
 
       data = []
@@ -62,6 +64,7 @@ def record_to_column(record, json = false)
         "#{attribute_list_start}#{data.join(multi_value_delim)}#{attribute_list_end}"
       end
 
+
     end
 
     # Convert an AR instance to a set of CSV columns

lib/datashift/configuration.rb

@@ -19,6 +19,15 @@ def self.parse_yaml( yaml_file )
     end
 
 
+    # @param [Boolean] Export/Import of data expected in JSON format
+    # @return [Boolean]
+    #
+    attr_accessor :json
+
+    def json?
+      !!@json
+    end
+
     # List of association +TYPES+ to INCLUDE [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
     # Defaults to [:assignment, :enum]
     #

lib/datashift/exporters/configuration.rb

@@ -24,11 +24,6 @@ class Configuration < DataShift::Configuration
       #
       attr_accessor :csv_delimiter
 
-      # @param [Boolean] Export association data in single column in JSON format
-      # @return [Boolean]
-      #
-      attr_accessor :json
-
       # @param [String] Name for worksheet, otherwise uses Class name
       # @return [String]
       #

lib/datashift/exporters/excel_exporter.rb

@@ -96,7 +96,13 @@ def prepare_data_flow_schema(klass)
     #
     #   See - lib/exporters/configuration.rb
     #
-    def export_with_associations(file_name, klass, records, options = {})
+    # To export associations as JSON:
+    #
+    #   DataShift::Exporters::Configuration.configure { |config| config.json = true }
+    #
+    def export_with_associations(file_name, klass, export_records, options = {})
+
+      records = [*export_records]
 
       state = DataShift::Configuration.call.with
 
@@ -132,7 +138,7 @@ def export_with_associations(file_name, klass, records, options = {})
             # pack association instances into single column
             if model_method.association_type?
               logger.info("Processing #{model_method.inspect} associations")
-              excel[row, column] = record_to_column( obj.send( model_method.operator ), configuration.json )
+              excel[row, column] = record_to_column( obj.send( model_method.operator ), configuration.json? )
             else
               excel[row, column] = obj.send( model_method.operator )
             end

lib/datashift/exporters/exporter_base.rb

@@ -9,11 +9,13 @@ module DataShift
 
   class ExporterBase
 
-    attr_accessor :configuration
     attr_accessor :file_name
 
     def initialize
-      @configuration = DataShift::Exporters::Configuration.call
+    end
+
+    def configuration
+      DataShift::Configuration.call
     end
 
   end

lib/datashift/header.rb

@@ -18,6 +18,11 @@ def initialize(source:, presentation: nil)
       @presentation = presentation || source
     end
 
+    def ==(rhs)
+      puts "== Called with #{rhs}"
+      @source == rhs
+    end
+
     def to_s
       presentation
     end

lib/datashift/headers.rb

@@ -26,12 +26,24 @@ class Headers
 
     def_delegators :@headers, *Array.delegated_methods_for_fwdable
 
-    def initialize(source, idx = 0, headers = [])
+    def initialize(source, header_row_index = 0, headers = [])
       @source = source
-      @idx = idx
+      @idx = header_row_index
       @headers = headers
     end
 
+    # Check for either string or symbol version of a header
+    def header?( header)
+      h = header.is_a?(ModelMethod) ? header.operator : header
+      (sources & [h, h.to_sym]).present?
+    end
+
+    def index(header)
+      h = header.is_a?(ModelMethod) ? header.operator : header
+      sources.index(h) || sources.index(h.to_sym)
+    end
+
+
     # Factory for dealing with Active Record models and collections
     # Catalogs the supplied Klass and builds set of expected/valid Headers for Klass
     # See config options
@@ -79,6 +91,7 @@ def class_source_to_operators
           if(mm.association_type?)
             association_to_headers(mm)
           else
+            # TODO - can/shoudl we standardise to always store symbols - this is currently String
             add mm.operator
           end
         end

lib/datashift/mapping/data_flow_schema.rb

@@ -82,10 +82,15 @@ def prepare_from_klass( klass, doc_context = nil )
 
       @nodes = create_node_collections(klass, doc_context: doc_context)
 
-      klass_to_model_methods( klass ).each_with_index do |mm, i|
-        @nodes.headers.add(mm.operator) # for a class, the header names, default to the operators (methods)
+      @nodes.headers = Headers.klass_to_headers(klass)
 
-        binding = MethodBinding.new(mm.operator, mm, idx: i)
+      klass_to_model_methods( klass ).each do |mm|
+        # headers define what nodes are actually in scope
+        next unless @nodes.headers.header?(mm)
+
+        i = @nodes.headers.index(mm)
+
+        binding = MethodBinding.new(mm.operator, mm, idx: i)  # TOFIX - why need to pass in both mm.operator AND mm ??
 
         # TODO: - do we really need to pass in the doc context when parent nodes already has it ?
         @nodes << DataShift::NodeContext.new(@nodes.doc_context, binding, i, nil)

lib/datashift/node_collection.rb

@@ -18,7 +18,7 @@ class NodeCollection
 
     def_delegators :@nodes, *Array.delegated_methods_for_fwdable
 
-    def_delegators :@doc_context, :headers, :klass
+    def_delegators :@doc_context, :headers, :'headers=', :klass
 
     def initialize(doc_context: nil)
       @nodes = []

lib/datashift/transformation/remove.rb

@@ -18,7 +18,7 @@ def remove_list
 
       def association?(mm)
         return false unless(mm.association_type?)
-        DataShift::Configuration.call.exclude_associations.include?(mm.operator)
+        (DataShift::Configuration.call.exclude_associations & [mm.operator, mm.operator.to_sym]).present?
       end
 
       # Specify columns to remove via DataShift::Configuration

spec/datashift/exporters/csv_exporter_spec.rb

@@ -221,7 +221,7 @@ module DataShift
 
         expected = result_file('project_and_assoc_in_json_export.csv')
 
-        DataShift::Exporters::Configuration.configure do |config|
+        DataShift::Configuration.configure do |config|
           config.json = true
         end
 

spec/datashift/exporters/excel_exporter_spec.rb

@@ -166,7 +166,7 @@ module  DataShift
 
         expected = result_file('project_and_assoc_in_json_export.xls')
 
-        DataShift::Exporters::Configuration.configure do |config|
+        DataShift::Configuration.configure do |config|
           config.json = true
         end