From a3da0f9763d9f110ac08df3f219fd8dcc4a617b4 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Sun, 11 Aug 2013 13:04:30 +0530 Subject: [PATCH 01/10] Added support for parsing multi-dimensional array string * Used the Sequel::Postgres::PGArray::Parser class from https://github.com/jeremyevans/sequel/blob/master/lib/sequel/extensions/pg_array.rb * The saner method to parse multi-dimensional array-strings is to NOT use regexps and use some sort of parser instead * Changed String.valid_postgres_array? to use constants instead of variables to defined the regexps * Converting timestamps to current Rails timezone using `in_time_zone` --- lib/activerecord-postgres-array.rb | 3 +- lib/activerecord-postgres-array/parser.rb | 112 ++++++++++++++++++++++ lib/activerecord-postgres-array/string.rb | 35 +++---- 3 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 lib/activerecord-postgres-array/parser.rb diff --git a/lib/activerecord-postgres-array.rb b/lib/activerecord-postgres-array.rb index 3a02758..8c98a5c 100644 --- a/lib/activerecord-postgres-array.rb +++ b/lib/activerecord-postgres-array.rb @@ -14,4 +14,5 @@ class ActiveRecordPostgresArray < Rails::Railtie end require "activerecord-postgres-array/string" -require "activerecord-postgres-array/array" \ No newline at end of file +require "activerecord-postgres-array/array" +require "activerecord-postgres-array/parser" \ No newline at end of file diff --git a/lib/activerecord-postgres-array/parser.rb b/lib/activerecord-postgres-array/parser.rb new file mode 100644 index 0000000..4822162 --- /dev/null +++ b/lib/activerecord-postgres-array/parser.rb @@ -0,0 +1,112 @@ +class ActiveRecordPostgresArray < Rails::Railtie + # PostgreSQL array parser that handles all types of input. + # + # This parser is very simple and unoptimized, but should still + # be O(n) where n is the length of the input string. + class Parser + ARRAY = "ARRAY".freeze + DOUBLE_COLON = '::'.freeze + EMPTY_BRACKET = '[]'.freeze + OPEN_BRACKET = '['.freeze + CLOSE_BRACKET = ']'.freeze + COMMA = ','.freeze + BACKSLASH = '\\'.freeze + EMPTY_STRING = ''.freeze + OPEN_BRACE = '{'.freeze + CLOSE_BRACE = '}'.freeze + NULL = 'NULL'.freeze + QUOTE = '"'.freeze + # Current position in the input string. + attr_reader :pos + + # Set the source for the input, and any converter callable + # to call with objects to be created. For nested parsers + # the source may contain text after the end current parse, + # which will be ignored. + def initialize(source, converter=nil) + @source = source + @source_length = source.length + @converter = converter + @pos = -1 + @entries = [] + @recorded = "" + @dimension = 0 + end + + # Return 2 objects, whether the next character in the input + # was escaped with a backslash, and what the next character is. + def next_char + @pos += 1 + if (c = @source[@pos..@pos]) == BACKSLASH + @pos += 1 + [true, @source[@pos..@pos]] + else + [false, c] + end + end + + # Add a new character to the buffer of recorded characters. + def record(c) + @recorded << c + end + + # Take the buffer of recorded characters and add it to the array + # of entries, and use a new buffer for recorded characters. + def new_entry(include_empty=false) + if !@recorded.empty? || include_empty + entry = @recorded + if entry == NULL && !include_empty + entry = nil + elsif @converter + entry = @converter.call(entry) + end + @entries.push(entry) + @recorded = "" + end + end + + # Parse the input character by character, returning an array + # of parsed (and potentially converted) objects. + def parse(nested=false) + # quote sets whether we are inside of a quoted string. + quote = false + until @pos >= @source_length + escaped, char = next_char + if char == OPEN_BRACE && !quote + @dimension += 1 + if (@dimension > 1) + # Multi-dimensional array encounter, use a subparser + # to parse the next level down. + subparser = self.class.new(@source[@pos..-1], @converter) + @entries.push(subparser.parse(true)) + @pos += subparser.pos - 1 + end + elsif char == CLOSE_BRACE && !quote + @dimension -= 1 + if (@dimension == 0) + new_entry + # Exit early if inside a subparser, since the + # text after parsing the current level should be + # ignored as it is handled by the parent parser. + return @entries if nested + end + elsif char == QUOTE && !escaped + # If already inside the quoted string, this is the + # ending quote, so add the entry. Otherwise, this + # is the opening quote, so set the quote flag. + new_entry(true) if quote + quote = !quote + elsif char == COMMA && !quote + # If not inside a string and a comma occurs, it indicates + # the end of the entry, so add the entry. + new_entry + else + # Add the character to the recorded character buffer. + record(char) + end + end + raise "array dimensions not balanced" unless @dimension == 0 + @entries + end + end +end \ No newline at end of file diff --git a/lib/activerecord-postgres-array/string.rb b/lib/activerecord-postgres-array/string.rb index f3fd3c6..0996201 100644 --- a/lib/activerecord-postgres-array/string.rb +++ b/lib/activerecord-postgres-array/string.rb @@ -1,4 +1,10 @@ class String + PgStringRegexp = /[^",\\]+/ + PgQuotedStringRegexp = /"[^"\\]*(?:\\.[^"\\]*)*"/ + PgNumberRegexp = /[-+]?[0-9]*\.?[0-9]+/ + PgValidationRegexp = /\{\s*((#{PgNumberRegexp}|#{PgQuotedStringRegexp}|#{PgStringRegexp})(\s*\,\s*(#{PgNumberRegexp}|#{PgQuotedStringRegexp}|#{PgStringRegexp}))*)?\}/ + + # def to_postgres_array self end @@ -8,11 +14,7 @@ def to_postgres_array # * A string like '{10000, 10000, 10000, 10000}' # * TODO A multi dimensional array string like '{{"meeting", "lunch"}, {"training", "presentation"}}' def valid_postgres_array? - string_regexp = /[^",\\]+/ - quoted_string_regexp = /"[^"\\]*(?:\\.[^"\\]*)*"/ - number_regexp = /[-+]?[0-9]*\.?[0-9]+/ - validation_regexp = /\{\s*((#{number_regexp}|#{quoted_string_regexp}|#{string_regexp})(\s*\,\s*(#{number_regexp}|#{quoted_string_regexp}|#{string_regexp}))*)?\}/ - !!match(/^\s*('#{validation_regexp}'|#{validation_regexp})?\s*$/) + !!match(/^\s*('#{PgValidationRegexp}'|#{PgValidationRegexp})?\s*$/) end # Creates an array from a postgres array string that postgresql spits out. @@ -20,23 +22,16 @@ def from_postgres_array(base_type = :string) if empty? [] else - elements = match(/\{(.*)\}/m).captures.first.gsub(/\\"/, '$ESCAPED_DOUBLE_QUOTE$').split(/(?:,)(?=(?:[^"]|"[^"]*")*$)/m) - elements = elements.map do |e| - res = e.gsub('$ESCAPED_DOUBLE_QUOTE$', '"').gsub("\\\\", "\\").gsub(/^"/, '').gsub(/"$/, '').gsub("''", "'").strip - res == 'NULL' ? nil : res + converter = case base_type + when :decimal then Proc.new {|x| x.to_d } + when :float then Proc.new {|x| x.to_f } + when :intger then Proc.new {|x| x.to_i } + when :timestamp then Proc.new {|x| x.to_time.in_time_zone } + else Proc.new {|x| x } end - if base_type == :decimal - elements.collect(&:to_d) - elsif base_type == :float - elements.collect(&:to_f) - elsif base_type == :integer || base_type == :bigint - elements.collect(&:to_i) - elsif base_type == :timestamp - elements.collect(&:to_time) - else - elements - end + parser = ActiveRecordPostgresArray::Parser.new(self, converter) + return parser.parse end end end From d46eb0245e298fdc454bbca7fda9e669f5beb1e0 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Sun, 11 Aug 2013 13:30:29 +0530 Subject: [PATCH 02/10] Removed String.valid_postgres_array? Removed String.valid_postgres_array? because there is no easy way to validate array-string using Regexps alone. One will have to parse the string into an array to check if it's valid -- a task that the PG database would anyways do. Removed check in `quote_with_array` --- lib/activerecord-postgres-array/activerecord.rb | 2 +- lib/activerecord-postgres-array/string.rb | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/activerecord-postgres-array/activerecord.rb b/lib/activerecord-postgres-array/activerecord.rb index dd8be3b..59a6ad8 100644 --- a/lib/activerecord-postgres-array/activerecord.rb +++ b/lib/activerecord-postgres-array/activerecord.rb @@ -40,7 +40,7 @@ def native_database_types_with_array(*args) # Quotes a value for use in an SQL statement def quote_with_array(value, column = nil) if value && column && column.sql_type =~ /\[\]$/ - raise ArrayTypeMismatch, "#{column.name} must be an Array or have a valid array value (#{value})" unless value.kind_of?(Array) || value.valid_postgres_array? + # raise ArrayTypeMismatch, "#{column.name} must be an Array or have a valid array value (#{value})" unless value.kind_of?(Array) return value.to_postgres_array end quote_without_array(value,column) diff --git a/lib/activerecord-postgres-array/string.rb b/lib/activerecord-postgres-array/string.rb index 0996201..f9681a0 100644 --- a/lib/activerecord-postgres-array/string.rb +++ b/lib/activerecord-postgres-array/string.rb @@ -13,9 +13,10 @@ def to_postgres_array # * An empty string # * A string like '{10000, 10000, 10000, 10000}' # * TODO A multi dimensional array string like '{{"meeting", "lunch"}, {"training", "presentation"}}' - def valid_postgres_array? - !!match(/^\s*('#{PgValidationRegexp}'|#{PgValidationRegexp})?\s*$/) - end + # def valid_postgres_array? + # !!match(/^\s*('#{PgValidationRegexp}'|#{PgValidationRegexp})?\s*$/) + # true + # end # Creates an array from a postgres array string that postgresql spits out. def from_postgres_array(base_type = :string) @@ -27,6 +28,7 @@ def from_postgres_array(base_type = :string) when :float then Proc.new {|x| x.to_f } when :intger then Proc.new {|x| x.to_i } when :timestamp then Proc.new {|x| x.to_time.in_time_zone } + when :boolean then Proc.new {|x| x.downcase=='t' ? true : false } else Proc.new {|x| x } end From 6a007a270c37fc0f7595839f2dbbc0f69617c688 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Sun, 11 Aug 2013 14:17:53 +0530 Subject: [PATCH 03/10] Convertime Time objects to iso8601 representation in UTC before saving to DB --- lib/activerecord-postgres-array/array.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/activerecord-postgres-array/array.rb b/lib/activerecord-postgres-array/array.rb index d737fce..41f1561 100644 --- a/lib/activerecord-postgres-array/array.rb +++ b/lib/activerecord-postgres-array/array.rb @@ -15,6 +15,8 @@ def to_postgres_array(omit_quotes = false) value elsif value.is_a?(NilClass) value = 'NULL' + elsif value.is_a?(Time) + value = "\"#{value.getutc.iso8601}\"" else value end From 5630623e164c46ba629193c0fa55d501d409a113 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Sun, 11 Aug 2013 14:26:49 +0530 Subject: [PATCH 04/10] * Making it play nice with other serialized attributes --- lib/activerecord-postgres-array/activerecord.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/activerecord-postgres-array/activerecord.rb b/lib/activerecord-postgres-array/activerecord.rb index 59a6ad8..7835de5 100644 --- a/lib/activerecord-postgres-array/activerecord.rb +++ b/lib/activerecord-postgres-array/activerecord.rb @@ -16,8 +16,8 @@ def arel_attributes_values(include_primary_key = true, include_readonly_attribut value = read_attribute(name) if column.type.to_s =~ /_array$/ && value && value.is_a?(Array) value = value.to_postgres_array(new_record?) - elsif klass.serialized_attributes.include?(name) - value = @attributes[name].serialized_value + elsif coder = klass.serialized_attributes[name] + value = coder.dump @attributes[name] end attrs[arel_table[name]] = value end From 9deaf08d476ea29a886fa819adaccff3e81f42af Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Sat, 24 Aug 2013 19:32:37 +0530 Subject: [PATCH 05/10] fixed a typo --- lib/activerecord-postgres-array/string.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/activerecord-postgres-array/string.rb b/lib/activerecord-postgres-array/string.rb index f9681a0..0c4f145 100644 --- a/lib/activerecord-postgres-array/string.rb +++ b/lib/activerecord-postgres-array/string.rb @@ -26,7 +26,7 @@ def from_postgres_array(base_type = :string) converter = case base_type when :decimal then Proc.new {|x| x.to_d } when :float then Proc.new {|x| x.to_f } - when :intger then Proc.new {|x| x.to_i } + when :integer then Proc.new {|x| x.to_i } when :timestamp then Proc.new {|x| x.to_time.in_time_zone } when :boolean then Proc.new {|x| x.downcase=='t' ? true : false } else Proc.new {|x| x } From 2ef39d4832f66d833d57ff3753dfdca6ed8d51d9 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Tue, 21 Jan 2014 19:01:11 +0530 Subject: [PATCH 06/10] BUGFIX -- the regex for detecing numeric array in the schema was incorrect --- lib/activerecord-postgres-array/activerecord.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/activerecord-postgres-array/activerecord.rb b/lib/activerecord-postgres-array/activerecord.rb index 7835de5..4e8cf0e 100644 --- a/lib/activerecord-postgres-array/activerecord.rb +++ b/lib/activerecord-postgres-array/activerecord.rb @@ -98,7 +98,7 @@ def type_cast_code_with_array(var_name) # Adds the array type for the column. def simplified_type_with_array(field_type) - if field_type =~ /^numeric.+\[\]$/ + if field_type =~ /^numeric.*\[\]$/ :decimal_array elsif field_type =~ /character varying.*\[\]/ :string_array From 061d43055538bb4ab93248cbec7e8dd53e78cd0b Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Fri, 12 Dec 2014 00:19:13 +0530 Subject: [PATCH 07/10] * Removed the monkeypatch for ActiveRecord::Base#arel_attributes_values because the actual function in Rails 3.2.x has changed. The monkeypatched version was unable to handle serialized attributes and was causing all serializations to barf. * Removed the method_chaing for quote_without_array, because it's not required in the new approach. * Added ActiveRecord::Coders::PgArray to act as a serializer/de-serializer for all array columns. One has to expilicity define how to serialize a column, like: `serialize :tags, ActiveRecord::Coders::PgArray(:string)` --- lib/activerecord-postgres-array.rb | 3 +- .../activerecord.rb | 40 +++++++++---------- lib/activerecord-postgres-array/coder.rb | 27 +++++++++++++ 3 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 lib/activerecord-postgres-array/coder.rb diff --git a/lib/activerecord-postgres-array.rb b/lib/activerecord-postgres-array.rb index 8c98a5c..ff01410 100644 --- a/lib/activerecord-postgres-array.rb +++ b/lib/activerecord-postgres-array.rb @@ -15,4 +15,5 @@ class ActiveRecordPostgresArray < Rails::Railtie require "activerecord-postgres-array/string" require "activerecord-postgres-array/array" -require "activerecord-postgres-array/parser" \ No newline at end of file +require "activerecord-postgres-array/parser" +require "activerecord-postgres-array/coder" diff --git a/lib/activerecord-postgres-array/activerecord.rb b/lib/activerecord-postgres-array/activerecord.rb index 4e8cf0e..ad998f1 100644 --- a/lib/activerecord-postgres-array/activerecord.rb +++ b/lib/activerecord-postgres-array/activerecord.rb @@ -5,27 +5,27 @@ class ArrayTypeMismatch < ActiveRecord::ActiveRecordError end class Base - def arel_attributes_values(include_primary_key = true, include_readonly_attributes = true, attribute_names = @attributes.keys) - attrs = {} - klass = self.class - arel_table = klass.arel_table + # def arel_attributes_values(include_primary_key = true, include_readonly_attributes = true, attribute_names = @attributes.keys) + # attrs = {} + # klass = self.class + # arel_table = klass.arel_table - attribute_names.each do |name| - if (column = column_for_attribute(name)) && (include_primary_key || !column.primary) - if include_readonly_attributes || !self.class.readonly_attributes.include?(name) - value = read_attribute(name) - if column.type.to_s =~ /_array$/ && value && value.is_a?(Array) - value = value.to_postgres_array(new_record?) - elsif coder = klass.serialized_attributes[name] - value = coder.dump @attributes[name] - end - attrs[arel_table[name]] = value - end - end - end + # attribute_names.each do |name| + # if (column = column_for_attribute(name)) && (include_primary_key || !column.primary) + # if include_readonly_attributes || !self.class.readonly_attributes.include?(name) + # value = read_attribute(name) + # if column.type.to_s =~ /_array$/ && value && value.is_a?(Array) + # value = value.to_postgres_array(new_record?) + # elsif coder = klass.serialized_attributes[name] + # value = coder.dump @attributes[name] + # end + # attrs[arel_table[name]] = value + # end + # end + # end - attrs - end + # attrs + # end end module ConnectionAdapters @@ -45,7 +45,7 @@ def quote_with_array(value, column = nil) end quote_without_array(value,column) end - alias_method_chain :quote, :array + # alias_method_chain :quote, :array end class Table diff --git a/lib/activerecord-postgres-array/coder.rb b/lib/activerecord-postgres-array/coder.rb new file mode 100644 index 0000000..ff69d39 --- /dev/null +++ b/lib/activerecord-postgres-array/coder.rb @@ -0,0 +1,27 @@ +module ActiveRecord + module Coders + class PgArray + def self.load(arr) + new({}).load(arr) + end + + def self.dump(arr) + new({}).dump(arr) + end + + def initialize(base_type, default=nil) + @base_type = base_type + @default=default + end + + def dump(obj) + obj.nil? ? (@default.nil? ? nil : @default.to_postgres_array(true)) : obj.to_postgres_array(true) + end + + def load(arr) + arr.nil? ? @default : arr.from_postgres_array(@base_type) + end + end + end +end + From 6b7ff20c54866b67e048b517bd0b88399bd951ff Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Fri, 12 Dec 2014 13:06:45 +0530 Subject: [PATCH 08/10] special cases for boolean arrays --- lib/activerecord-postgres-array/array.rb | 4 ++++ lib/activerecord-postgres-array/string.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/activerecord-postgres-array/array.rb b/lib/activerecord-postgres-array/array.rb index 41f1561..9c0c0e1 100644 --- a/lib/activerecord-postgres-array/array.rb +++ b/lib/activerecord-postgres-array/array.rb @@ -17,6 +17,10 @@ def to_postgres_array(omit_quotes = false) value = 'NULL' elsif value.is_a?(Time) value = "\"#{value.getutc.iso8601}\"" + elsif value.is_a?(TrueClass) + 't' + elsif value.is_a?(FalseClass) + 'f' else value end diff --git a/lib/activerecord-postgres-array/string.rb b/lib/activerecord-postgres-array/string.rb index 0c4f145..670b655 100644 --- a/lib/activerecord-postgres-array/string.rb +++ b/lib/activerecord-postgres-array/string.rb @@ -28,7 +28,7 @@ def from_postgres_array(base_type = :string) when :float then Proc.new {|x| x.to_f } when :integer then Proc.new {|x| x.to_i } when :timestamp then Proc.new {|x| x.to_time.in_time_zone } - when :boolean then Proc.new {|x| x.downcase=='t' ? true : false } + when :boolean then Proc.new {|x| (x.downcase=='t' || x==true) ? true : false } else Proc.new {|x| x } end From a006ed603a7f27c5f53e511c68aeb82b35043e08 Mon Sep 17 00:00:00 2001 From: Saurabh Nanda Date: Fri, 26 Dec 2014 23:19:32 +0530 Subject: [PATCH 09/10] fixes to make sure read_attribute works for postrgres arrays, even if we override the reader method --- lib/activerecord-postgres-array/activerecord.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/activerecord-postgres-array/activerecord.rb b/lib/activerecord-postgres-array/activerecord.rb index 4e8cf0e..be91ea2 100644 --- a/lib/activerecord-postgres-array/activerecord.rb +++ b/lib/activerecord-postgres-array/activerecord.rb @@ -85,6 +85,19 @@ class TableDefinition end class PostgreSQLColumn < Column + + # + def type_cast_with_array(val) + if type.to_s =~ /_array$/ + base_type = type.to_s.gsub(/_array/, '') + val.nil? ? nil : val.from_postgres_array(base_type.parameterize('_').to_sym) + else + type_cast_without_array(val) + end + end + alias_method_chain :type_cast, :array + + # Does the type casting from array columns using String#from_postgres_array or Array#from_postgres_array. def type_cast_code_with_array(var_name) if type.to_s =~ /_array$/ From e5eaef79c98956b48b9c4f0b038210610d3c6603 Mon Sep 17 00:00:00 2001 From: Ashwin Saval Date: Fri, 30 Jan 2015 13:24:51 +0530 Subject: [PATCH 10/10] * Committing small changes to bump up latest commit --- README.textile | 2 +- activerecord-postgres-array.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.textile b/README.textile index 969e52c..420793e 100644 --- a/README.textile +++ b/README.textile @@ -1,4 +1,4 @@ -h2. Postgres array support for activerecord +h2. Postgres array support for activerecord modified by VacationLabs Add basic support for postgres arrays to activerecord, with special attention to getting rails migrations / schema dumps working nicely. diff --git a/activerecord-postgres-array.gemspec b/activerecord-postgres-array.gemspec index bb43266..dde34ba 100644 --- a/activerecord-postgres-array.gemspec +++ b/activerecord-postgres-array.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |s| s.name = "activerecord-postgres-array" s.version = "0.0.9" - s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to?(:required_rubygems_version=) s.authors = ["Tim Connor"] s.date = %q{2012-02-08} s.description = "Adds support for postgres arrays to ActiveRecord"