From d4f6e2d3bc0c89f0218b813444a9e751f62cce9c Mon Sep 17 00:00:00 2001 From: Andrew Konchin Date: Sun, 12 Jan 2025 01:15:11 +0200 Subject: [PATCH] Support condition expressions with `#where` --- README.md | 23 +++- lib/dynamoid/adapter.rb | 2 +- lib/dynamoid/adapter_plugin/aws_sdk_v3.rb | 6 +- .../aws_sdk_v3/filter_expression_convertor.rb | 34 +++++- .../adapter_plugin/aws_sdk_v3/query.rb | 2 +- .../adapter_plugin/aws_sdk_v3/scan.rb | 2 +- lib/dynamoid/criteria/chain.rb | 65 ++++++++-- lib/dynamoid/criteria/where_conditions.rb | 19 ++- spec/dynamoid/criteria/chain_spec.rb | 113 ++++++++++++++++++ 9 files changed, 241 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 83fbe0e6..05184329 100644 --- a/README.md +++ b/README.md @@ -719,7 +719,7 @@ users = User.import([{ name: 'Josh' }, { name: 'Nick' }]) ### Querying -Querying can be done in one of three ways: +Querying can be done in one of the following ways: ```ruby Address.find(address.id) # Find directly by ID. @@ -728,6 +728,27 @@ Address.where(city: 'Chicago').all # Find by any number of matching criteria. Address.find_by_city('Chicago') # The same as above, but using ActiveRecord's older syntax. ``` +There is also a way to `#where` with a condition expression: + +```ruby +Address.where('city = :c', c: 'Chicago') +``` + +A condition expression may contain operators (e.g. `<`, `>=`, `<>`), +keywords (e.g. `AND`, `OR`, `BETWEEN`) and built-in functions (e.g. +`begins_with`, `contains`) (see (documentation +)[https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html] +for full syntax description). + +**Warning:** Values (specified for a String condition expression) are +sent as is so Dynamoid field types that aren't supported natively by +DynamoDB (e.g. `datetime` and `date`) require explicit casting. + +**Warning:** String condition expressions will be used by DynamoDB only +at filtering, so conditions on key attributes should be specified as a +Hash to perform Query operation instead of Scan. Don't use key +attributes in `#where`'s String condition expressions. + And you can also query on associations: ```ruby diff --git a/lib/dynamoid/adapter.rb b/lib/dynamoid/adapter.rb index 664a67c2..0b1a5a96 100644 --- a/lib/dynamoid/adapter.rb +++ b/lib/dynamoid/adapter.rb @@ -118,7 +118,7 @@ def delete(table, ids, options = {}) # @param [Hash] query a hash of attributes: matching records will be returned by the scan # # @since 0.2.0 - def scan(table, query = {}, opts = {}) + def scan(table, query = [], opts = {}) benchmark('Scan', table, query) { adapter.scan(table, query, opts) } end diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb index 593d1d47..029dfd9e 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb @@ -517,7 +517,7 @@ def put_item(table_name, object, options = {}) # @since 1.0.0 # # @todo Provide support for various other options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#query-instance_method - def query(table_name, key_conditions, non_key_conditions = {}, options = {}) + def query(table_name, key_conditions, non_key_conditions = [], options = {}) Enumerator.new do |yielder| table = describe_table(table_name) @@ -550,7 +550,7 @@ def query_count(table_name, key_conditions, non_key_conditions, options) # @since 1.0.0 # # @todo: Provide support for various options http://docs.aws.amazon.com/sdkforruby/api/Aws/DynamoDB/Client.html#scan-instance_method - def scan(table_name, conditions = {}, options = {}) + def scan(table_name, conditions = [], options = {}) Enumerator.new do |yielder| table = describe_table(table_name) @@ -563,7 +563,7 @@ def scan(table_name, conditions = {}, options = {}) end end - def scan_count(table_name, conditions = {}, options = {}) + def scan_count(table_name, conditions = [], options = {}) table = describe_table(table_name) options[:select] = 'COUNT' diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb index 8f9b50ca..015f541c 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3/filter_expression_convertor.rb @@ -20,7 +20,24 @@ def initialize(conditions, name_placeholders, value_placeholders, name_placehold private def build - clauses = @conditions.map do |name, attribute_conditions| + clauses = [] + + @conditions.each do |conditions| + if conditions.is_a? Hash + clauses << build_for_hash(conditions) unless conditions.empty? + elsif conditions.is_a? Array + query, placeholders = conditions + clauses << build_for_string(query, placeholders) + else + raise ArgumentError, "expected Hash or Array but actual value is #{conditions}" + end + end + + @expression = clauses.join(' AND ') + end + + def build_for_hash(hash) + clauses = hash.map do |name, attribute_conditions| attribute_conditions.map do |operator, value| # replace attribute names with placeholders unconditionally to support # - special characters (e.g. '.', ':', and '#') and @@ -62,7 +79,20 @@ def build end end.flatten - @expression = clauses.join(' AND ') + if clauses.empty? + nil + else + clauses.join(' AND ') + end + end + + def build_for_string(query, placeholders) + placeholders.each do |(k, v)| + k = ":#{k}" unless k.start_with?(':') + @value_placeholders[k] = v + end + + "(#{query})" end def name_placeholder_for(name) diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb index 25cf8835..9b6501e0 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb @@ -69,7 +69,7 @@ def build_request limit = [record_limit, scan_limit, batch_size].compact.min # key condition expression - convertor = FilterExpressionConvertor.new(@key_conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence) + convertor = FilterExpressionConvertor.new([@key_conditions], name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence) key_condition_expression = convertor.expression value_placeholders = convertor.value_placeholders name_placeholders = convertor.name_placeholders diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb index d4b1d7ad..fdadc247 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3/scan.rb @@ -13,7 +13,7 @@ class AwsSdkV3 class Scan attr_reader :client, :table, :conditions, :options - def initialize(client, table, conditions = {}, options = {}) + def initialize(client, table, conditions = [], options = {}) @client = client @table = table @conditions = conditions diff --git a/lib/dynamoid/criteria/chain.rb b/lib/dynamoid/criteria/chain.rb index 9d46a934..9e872982 100644 --- a/lib/dynamoid/criteria/chain.rb +++ b/lib/dynamoid/criteria/chain.rb @@ -95,15 +95,45 @@ def initialize(source) # # Internally +where+ performs either +Scan+ or +Query+ operation. # + # Conditions can be specified as an expression as well: + # + # Post.where('links_count = :v', v: 2) + # + # This way complex expressions can be constructed (e.g. with AND, OR, and NOT + # keyword): + # + # Address.where('city = :c AND (post_code = :pc1 OR post_code = :pc2)', city: 'A', pc1: '001', pc2: '002') + # + # See documentation for condition expression's syntax and examples: + # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html + # - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.FilterExpression.html + # # @return [Dynamoid::Criteria::Chain] # @since 0.2.0 - def where(args) - detector = NonexistentFieldsDetector.new(args, @source) + def where(conditions, placeholders = nil) + if conditions.is_a?(Hash) + where_with_hash(conditions) + else + where_with_string(conditions, placeholders) + end + end + + private def where_with_hash(conditions) + detector = NonexistentFieldsDetector.new(conditions, @source) if detector.found? Dynamoid.logger.warn(detector.warning_message) end - @where_conditions.update(args.symbolize_keys) + @where_conditions.update_with_hash(conditions.symbolize_keys) + + # we should re-initialize keys detector every time we change @where_conditions + @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name) + + self + end + + private def where_with_string(query, placeholders) + @where_conditions.update_with_string(query, placeholders) # we should re-initialize keys detector every time we change @where_conditions @key_fields_detector = KeyFieldsDetector.new(@where_conditions, @source, forced_index_name: @forced_index_name) @@ -635,12 +665,12 @@ def query_key_conditions end def query_non_key_conditions - opts = {} + hash_conditions = {} # Honor STI and :type field if it presents if @source.attributes.key?(@source.inheritance_field) && @key_fields_detector.hash_key.to_sym != @source.inheritance_field.to_sym - @where_conditions.update(sti_condition) + @where_conditions.update_with_hash(sti_condition) end # TODO: Separate key conditions and non-key conditions properly: @@ -650,11 +680,17 @@ def query_non_key_conditions .reject { |k, _| k.to_s =~ /^#{@key_fields_detector.range_key}\./ } keys.each do |key| name, condition = field_condition(key, @where_conditions[key]) - opts[name] ||= [] - opts[name] << condition + hash_conditions[name] ||= [] + hash_conditions[name] << condition end - opts + string_conditions = [] + @where_conditions.string_conditions.each do |query, placeholders| + placeholders ||= {} + string_conditions << [query, placeholders] + end + + [hash_conditions] + string_conditions end # TODO: casting should be operator aware @@ -721,16 +757,25 @@ def query_options def scan_conditions # Honor STI and :type field if it presents if sti_condition - @where_conditions.update(sti_condition) + @where_conditions.update_with_hash(sti_condition) end - {}.tap do |opts| + hash_conditions = {} + hash_conditions.tap do |opts| @where_conditions.keys.map(&:to_sym).each do |key| name, condition = field_condition(key, @where_conditions[key]) opts[name] ||= [] opts[name] << condition end end + + string_conditions = [] + @where_conditions.string_conditions.each do |query, placeholders| + placeholders ||= {} + string_conditions << [query, placeholders] + end + + [hash_conditions] + string_conditions end def scan_options diff --git a/lib/dynamoid/criteria/where_conditions.rb b/lib/dynamoid/criteria/where_conditions.rb index 52baadd7..3b1b9e1b 100644 --- a/lib/dynamoid/criteria/where_conditions.rb +++ b/lib/dynamoid/criteria/where_conditions.rb @@ -4,24 +4,31 @@ module Dynamoid module Criteria # @private class WhereConditions + attr_reader :string_conditions + def initialize - @conditions = [] + @hash_conditions = [] + @string_conditions = [] + end + + def update_with_hash(hash) + @hash_conditions << hash.symbolize_keys end - def update(hash) - @conditions << hash.symbolize_keys + def update_with_string(query, placeholders) + @string_conditions << [query, placeholders] end def keys - @conditions.flat_map(&:keys) + @hash_conditions.flat_map(&:keys) end def empty? - @conditions.empty? + @hash_conditions.empty? && @string_conditions.empty? end def [](key) - hash = @conditions.find { |h| h.key?(key) } + hash = @hash_conditions.find { |h| h.key?(key) } hash[key] if hash end end diff --git a/spec/dynamoid/criteria/chain_spec.rb b/spec/dynamoid/criteria/chain_spec.rb index a3ca7bcb..20fe3058 100644 --- a/spec/dynamoid/criteria/chain_spec.rb +++ b/spec/dynamoid/criteria/chain_spec.rb @@ -1338,6 +1338,70 @@ def request_params end end + describe '#where with String query' do + let(:klass) do + new_class do + field :first_name # `name` is a reserved keyword + field :age, :integer + end + end + + it 'filters by specified conditions' do + obj1 = klass.create!(first_name: 'Alex', age: 42) + obj2 = klass.create!(first_name: 'Michael', age: 50) + + expect(klass.where('age > :age', age: 42).all).to contain_exactly(obj2) + expect(klass.where('first_name = :name', name: 'Alex').all).to contain_exactly(obj1) + end + + it 'accepts placeholder names with ":" prefix' do + obj1 = klass.create!(first_name: 'Alex', age: 42) + obj2 = klass.create!(first_name: 'Michael', age: 50) + + expect(klass.where('age > :age', ":age": 42).all).to contain_exactly(obj2) + expect(klass.where('first_name = :name', ":name": 'Alex').all).to contain_exactly(obj1) + end + + it 'combines with a call with String query with logical AND' do + obj1 = klass.create!(first_name: 'Alex', age: 42) + obj2 = klass.create!(first_name: 'Michael', age: 50) + obj3 = klass.create!(first_name: 'Alex', age: 18) + + expect(klass.where('age < :age', age: 40).where('first_name = :name', name: 'Alex').all).to contain_exactly(obj3) + end + + it 'combines with a call with Hash query with logical AND' do + obj1 = klass.create!(first_name: 'Alex', age: 42) + obj2 = klass.create!(first_name: 'Michael', age: 50) + obj3 = klass.create!(first_name: 'Alex', age: 18) + + expect(klass.where('age < :age', age: 40).where(first_name: 'Alex').all).to contain_exactly(obj3) + end + + context 'Query' do + it 'filters by specified conditions' do + obj = klass.create!(first_name: 'Alex', age: 42) + + expect(klass.where(id: obj.id).where('age = :age', age: 42).all.to_a).to eq([obj]) + expect(klass.where(id: obj.id).where('age <> :age', age: 42).all.to_a).to eq([]) + end + end + + context 'Scan' do + it 'filters by specified conditions' do + obj = klass.create!(first_name: 'Alex', age: 42) + expect(klass.where('age = :age', age: 42).all.to_a).to eq([obj]) + end + + it 'performs Scan when key attributes are used only in String query' do + obj = klass.create!(first_name: 'Alex', age: 42) + + expect(Dynamoid.adapter.client).to receive(:scan).and_call_original + expect(klass.where('id = :id', id: obj.id).all.to_a).to eq([obj]) + end + end + end + describe '#find_by_pages' do let(:model) do new_class do @@ -1598,6 +1662,20 @@ def request_params expect { chain.delete_all }.to change { klass.count }.by(-1) end + + it 'works well when #where is called with a String query' do + klass = new_class do + field :title + end + + document = klass.create!(title: 'title#1') + klass.create! + + chain = described_class.new(klass) + chain = chain.where(id: document.id).where('title = :v', v: document.title) + + expect { chain.delete_all }.to change { klass.count }.by(-1) + end end context 'Scan (partition key is not specified)' do @@ -1628,6 +1706,20 @@ def request_params expect { chain.delete_all }.to change { klass.count }.by(-1) end + + it 'works well when #where is called with a String query' do + klass = new_class do + field :title + end + + klass.create!(title: 'Doc #1') + klass.create!(title: 'Doc #2') + + chain = described_class.new(klass) + chain = chain.where('title = :v', v: 'Doc #1') + + expect { chain.delete_all }.to change { klass.count }.by(-1) + end end end @@ -1880,6 +1972,7 @@ def request_params table name: :customer, key: :name range :age, :integer + field :year_of_birth, :integer end end @@ -1890,6 +1983,18 @@ def request_params expect(model.where(name: 'Bob', 'age.lt': 10).count).to eql(2) end + + it 'returns count of filtered documents when #where called with a String query' do + customer1 = model.create(name: 'Bob', age: 5, year_of_birth: 2000) + customer2 = model.create(name: 'Bob', age: 9, year_of_birth: 2010) + customer3 = model.create(name: 'Bob', age: 12, year_of_birth: 2020) + + expect( + model.where(name: 'Bob', 'age.lt': 10) + .where('year_of_birth > :year', year: 2005) + .count + ).to eql(1) + end end context 'Scan' do @@ -1906,6 +2011,14 @@ def request_params expect(model.where('age.lt': 10).count).to eql(2) end + + it 'returns count of filtered documents when #where called with a String query' do + customer1 = model.create(age: 5) + customer2 = model.create(age: 9) + customer3 = model.create(age: 12) + + expect(model.where('age < :age', age: 10).count).to eql(2) + end end end