diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index 01e15fb32..9932ed5ec 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -33,8 +33,6 @@ def enabled? end def update_keywords(language, keywords, keyword, operation, doc_id, context) - alegre_operation = nil - alegre_params = nil common_alegre_params = { doc_id: doc_id, context: { @@ -44,15 +42,11 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) } if operation == 'add' && !keywords.include?(keyword) keywords << keyword - alegre_operation = 'post' - alegre_params = common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }) + Bot::Alegre.index_sync_with_params(common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }), "text") elsif operation == 'remove' keywords -= [keyword] - alegre_operation = 'delete' - alegre_params = common_alegre_params.merge({ quiet: true }) + Bot::Alegre.request_delete_from_raw(common_alegre_params.merge({ quiet: true }), "text") end - # FIXME: Add error handling and better logging - Bot::Alegre.request(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params keywords end @@ -91,19 +85,19 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.query_sync_with_params(params, "text") # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option # So, we are not using this approach right now # Get the `alegre_result_key` of all results returned - # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', alegre_result_key)} + # option_counts = response['result'].to_a.map{|o| o.dig('context', alegre_result_key)} # Count how many of each alegre_result_key we have and sort (high to low) # ranked_options = option_counts.group_by(&:itself).transform_values(&:count).sort_by{|_k,v| v}.reverse() # Second approach is to sort the results from best to worst - sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse - ranked_options = sorted_options.map{ |o| { 'key' => o.dig('_source', 'context', alegre_result_key), 'score' => o['_score'] } } + sorted_options = response['result'].to_a.sort_by{ |result| result['score'] }.reverse + ranked_options = sorted_options.map{ |o| { 'key' => o.dig('context', alegre_result_key), 'score' => o['score'] } } matches = ranked_options # In all cases log for analysis diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index 9471af181..598f2ecc6 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -41,7 +41,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {}) ALL_TEXT_SIMILARITY_FIELDS.each do |field| text = self.send(field) next if text.blank? - threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_similar_texts(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) } + threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_items_from_similar_text(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) } end threads.map(&:join) end @@ -155,10 +155,8 @@ def self.run(body) if ['audio', 'image', 'video'].include?(self.get_pm_type(pm)) self.relate_project_media_async(pm) else - Bot::Alegre.send_to_media_similarity_index(pm) - Bot::Alegre.send_field_to_similarity_index(pm, 'original_title') - Bot::Alegre.send_field_to_similarity_index(pm, 'original_description') - Bot::Alegre.relate_project_media_to_similar_items(pm) + self.relate_project_media_async(pm, 'original_title') + self.relate_project_media_async(pm, 'original_description') end self.get_extracted_text(pm) self.get_flags(pm) @@ -206,7 +204,7 @@ def self.get_items_from_similar_text(team_id, text, fields = nil, threshold = ni threshold ||= self.get_threshold_for_query('text', nil, true) models ||= [self.matching_model_to_use(team_ids)].flatten Hash[self.get_similar_items_from_api( - '/text/similarity/search/', + 'text', self.similar_texts_from_api_conditions(text, models, fuzzy, team_ids, fields, threshold), threshold ).collect{|k,v| [k, v.merge(model: v[:model]||Bot::Alegre.default_matching_model)]}] @@ -716,8 +714,4 @@ def self.is_text_too_short?(pm, length_threshold) is_short end - class < "Claim", @@ -36,6 +36,10 @@ def is_video? def is_audio? self.type == "audio" end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end end module AlegreV2 @@ -55,11 +59,18 @@ def sync_path_for_type(type) end def async_path(project_media) - "/similarity/async/#{get_type(project_media)}" + self.async_path_for_type(get_type(project_media)) + end + + def async_path_for_type(type) + "/similarity/async/#{type}" end def delete_path(project_media) - type = get_type(project_media) + self.delete_path_for_type(get_type(project_media)) + end + + def delete_path_for_type(type) "/#{type}/similarity/" end @@ -122,6 +133,10 @@ def request(method, path, params, retries=3) end end + def request_delete_from_raw(params, type) + request("delete", delete_path_for_type(type), params) + end + def request_delete(data, project_media) request("delete", delete_path(project_media), data) end @@ -148,28 +163,32 @@ def get_type(project_media) type end + def content_hash_for_value(value) + value.nil? ? nil : Digest::MD5.hexdigest(value) + end + def content_hash(project_media, field) if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field) - Digest::MD5.hexdigest(project_media.send(field)) + content_hash_for_value(project_media.send(field)) + elsif project_media.is_link? + return content_hash_for_value(project_media.media.url) + elsif project_media.is_a?(TemporaryProjectMedia) + return Rails.cache.read("url_sha:#{project_media.url}") + elsif project_media.is_uploaded_media? + return project_media.media.file.filename.split(".").first else - if project_media.is_link? - return Digest::MD5.hexdigest(project_media.media.url) - elsif project_media.is_a?(TemporaryProjectMedia) - return Rails.cache.read("url_sha:#{project_media.url}") - elsif !project_media.is_text? - return project_media.media.file.filename.split(".").first - else - return Digest::MD5.hexdigest(project_media.send(field).to_s) - end + return content_hash_for_value(project_media.send(field).to_s) end end def generic_package(project_media, field) - { - content_hash: content_hash(project_media, field), + content_hash_value = content_hash(project_media, field) + params = { doc_id: item_doc_id(project_media, field), context: get_context(project_media, field) } + params[:content_hash] = content_hash_value if !content_hash_value.nil? + params end def delete_package(project_media, field, params={}, quiet=false) @@ -267,6 +286,22 @@ def store_package_text(project_media, field, params) generic_package_text(project_media, field, params) end + def index_async_with_params(params, type, suppress_search_response=true) + request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) + end + + def index_sync_with_params(params, type) + query_sync_with_params(params, type) + end + + def query_sync_with_params(params, type) + request("post", sync_path_for_type(type), params) + end + + def query_async_with_params(params, type) + request("post", async_path_for_type(type), params) + end + def get_sync(project_media, field=nil, params={}) request_sync( store_package(project_media, field, params), @@ -286,6 +321,10 @@ def delete(project_media, field=nil, params={}) delete_package(project_media, field, params), project_media ) + rescue StandardError => e + error = Bot::Alegre::Error.new(e) + Rails.logger.error("[Alegre Bot] Exception on Delete for ProjectMedia ##{project_media.id}: #{error.class} - #{error.message}") + CheckSentry.notify(error, bot: "alegre", project_media: project_media, params: params, field: field) end def get_per_model_threshold(project_media, threshold) @@ -298,7 +337,7 @@ def get_per_model_threshold(project_media, threshold) end def isolate_relevant_context(project_media, result) - result["context"].select{|x| ([x["team_id"]].flatten & [project_media.team_id].flatten).count > 0 && !x["temporary_media"]}.first + (result["contexts"]||result["context"]).select{|x| ([x["team_id"]].flatten & [project_media.team_id].flatten).count > 0 && !x["temporary_media"]}.first end def get_target_field(project_media, field) @@ -485,25 +524,27 @@ def wait_for_results(project_media, args) end def get_items_with_similar_media_v2(args={}) + text = args[:text] + field = args[:field] media_url = args[:media_url] project_media = args[:project_media] threshold = args[:threshold] team_ids = args[:team_ids] type = args[:type] - if ['audio', 'image', 'video'].include?(type) - if project_media.nil? - project_media = TemporaryProjectMedia.new - project_media.url = media_url - project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) - project_media.team_id = team_ids - project_media.type = type - end - get_similar_items_v2_async(project_media, nil, threshold) - wait_for_results(project_media, args) - response = get_similar_items_v2_callback(project_media, nil) - delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) - return response + if project_media.nil? + project_media = TemporaryProjectMedia.new + project_media.text = text + project_media.field = field + project_media.url = media_url + project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) + project_media.team_id = team_ids + project_media.type = type end + get_similar_items_v2_async(project_media, nil, threshold) + wait_for_results(project_media, args) + response = get_similar_items_v2_callback(project_media, nil) + delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) + return response end def process_alegre_callback(params) @@ -512,9 +553,11 @@ def process_alegre_callback(params) should_relate = true if project_media.nil? project_media = TemporaryProjectMedia.new + project_media.text = params.dig('data', 'item', 'raw', 'text') project_media.url = params.dig('data', 'item', 'raw', 'url') project_media.id = params.dig('data', 'item', 'raw', 'context', 'project_media_id') project_media.team_id = params.dig('data', 'item', 'raw', 'context', 'team_id') + project_media.field = params.dig('data', 'item', 'raw', 'context', 'field') project_media.type = params['model_type'] should_relate = false end diff --git a/app/models/concerns/project_media_getters.rb b/app/models/concerns/project_media_getters.rb index b8662224e..0ce240c33 100644 --- a/app/models/concerns/project_media_getters.rb +++ b/app/models/concerns/project_media_getters.rb @@ -11,10 +11,6 @@ def is_link? self.media.type == "Link" end - def is_uploaded_image? - self.media.type == "UploadedImage" - end - def is_blank? self.media.type == "Blank" end @@ -28,7 +24,11 @@ def is_audio? end def is_image? - self.is_uploaded_image? + self.media.type == "UploadedImage" + end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? end def is_text? diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 5e2992bd8..aa25ea9e4 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -73,24 +73,26 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) # Index title params = { + content_hash: Bot::Alegre.content_hash_for_value(explainer.title), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')), + context: base_context.merge({ field: 'title' }), text: explainer.title, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ field: 'title' }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") # Index paragraphs count = 0 explainer.description.to_s.gsub(/\r\n?/, "\n").split(/\n+/).reject{ |paragraph| paragraph.strip.blank? }.each do |paragraph| count += 1 params = { + content_hash: Bot::Alegre.content_hash_for_value(paragraph.strip), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'paragraph', count].join(':')), + context: base_context.merge({ paragraph: count }), text: paragraph.strip, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") end # Remove paragraphs that don't exist anymore (we delete after updating in order to avoid race conditions) @@ -101,7 +103,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) quiet: true, context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('delete', '/text/similarity/', params) + Bot::Alegre.request_delete_from_raw(params, "text") end end @@ -116,9 +118,9 @@ def self.search_by_similarity(text, language, team_id) language: language } } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.query_sync_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } - explainer_ids = results.collect{ |result| result.dig('_source', 'context', 'explainer_id').to_i }.uniq.first(3) + explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(3) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) end diff --git a/app/models/request.rb b/app/models/request.rb index 81c9e5834..8f48f5b80 100644 --- a/app/models/request.rb +++ b/app/models/request.rb @@ -53,7 +53,7 @@ def attach_to_similar_request!(alegre_limit = 20) models_thresholds = self.text_similarity_settings.reject{ |_k, v| v['min_words'] > words } if models_thresholds.count > 0 params = { text: media.quote, models: models_thresholds.keys, per_model_threshold: models_thresholds.transform_values{ |v| v['threshold'] }, limit: alegre_limit, context: context } - similar_request_id = ::Bot::Alegre.request('post', '/text/similarity/search/', params)&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } + similar_request_id = ::Bot::Alegre.query_sync_with_params(params, "text")&.dig('result').to_a.collect{ |result| result&.dig('context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } end # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # threshold = 0.85 #FIXME: Should be feed setting @@ -194,7 +194,7 @@ def self.send_to_alegre(id) models: request.text_similarity_settings.keys(), context: context } - ::Bot::Alegre.request('post', '/text/similarity/', params) + ::Bot::Alegre.index_async_with_params(params, "text") # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # type = media.type.gsub(/^Uploaded/, '').downcase # url = media.file&.file&.public_url diff --git a/app/resources/api/v2/report_resource.rb b/app/resources/api/v2/report_resource.rb index 29ee00cc8..3289b0b61 100644 --- a/app/resources/api/v2/report_resource.rb +++ b/app/resources/api/v2/report_resource.rb @@ -57,7 +57,7 @@ def self.apply_text_similarity_filter(organization_ids, threshold, filters) ids = nil unless text.blank? fields = filters[:similarity_fields].blank? ? nil : filters[:similarity_fields].to_a.flatten - ids_and_scores = Bot::Alegre.get_similar_texts(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0)) + ids_and_scores = Bot::Alegre.get_items_from_similar_text(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0)) RequestStore.store[:scores] = ids_and_scores # Store the scores so we can return them ids = ids_and_scores.keys.uniq ids = [0] if ids.blank? diff --git a/app/workers/reindex_alegre_workspace.rb b/app/workers/reindex_alegre_workspace.rb index 093f8e56a..557a3db08 100644 --- a/app/workers/reindex_alegre_workspace.rb +++ b/app/workers/reindex_alegre_workspace.rb @@ -45,12 +45,15 @@ def get_default_query(team_id, last_id=nil) end def get_request_doc(pm, field, field_value) - Bot::Alegre.send_to_text_similarity_index_package( - pm, - field, - field_value, - Bot::Alegre.item_doc_id(pm, field) - ) + { + doc: Bot::Alegre.send_to_text_similarity_index_package( + pm, + field, + field_value, + Bot::Alegre.item_doc_id(pm, field) + ), + type: Bot::Alegre.get_pm_type(pm) + } end def get_request_docs_for_project_media(pm) @@ -64,11 +67,14 @@ def get_request_docs_for_project_media(pm) end end - def check_for_write(running_bucket, event_id, team_id, write_remains=false, in_processes=3) + def check_for_write(running_bucket, event_id, team_id, write_remains=false) # manage dispatch of documents to bulk similarity api call in parallel if running_bucket.length > 500 || write_remains log(event_id, 'Writing to Alegre...') - Parallel.map(running_bucket.each_slice(30).to_a, in_processes: in_processes) { |bucket_slice| Bot::Alegre.request('post', '/text/bulk_similarity/', { documents: bucket_slice }) } + running_bucket.each do |item| + # FIXME we need to go back to bulk uploads eventually + Bot::Alegre.query_async_with_params(item[:doc], item[:type]) + end log(event_id, 'Wrote to Alegre.') # track state in case job needs to restart write_last_id(event_id, team_id, running_bucket.last[:context][:project_media_id]) if running_bucket.length > 0 && running_bucket.last[:context] diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index 6493472a9..3c22995bd 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -60,7 +60,7 @@ def setup WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json) WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre/image/classification/').with({ body: { uri: 'some/path' } }).to_return(body: { "result": valid_flags_data }.to_json) diff --git a/test/controllers/graphql_controller_5_test.rb b/test/controllers/graphql_controller_5_test.rb index 8264680ec..e85a1918b 100644 --- a/test/controllers/graphql_controller_5_test.rb +++ b/test/controllers/graphql_controller_5_test.rb @@ -57,14 +57,14 @@ def setup pm = create_project_media project: p, media: m pm2 = create_project_media project: p, media: m2 create_claim_description project_media: pm2 - Bot::Alegre.stubs(:get_similar_texts).returns({ pm2.id => 0.9, pm.id => 0.8 }) + Bot::Alegre.stubs(:get_items_from_similar_text).returns({ pm2.id => 0.9, pm.id => 0.8 }) query = 'query { project_media(ids: "' + [pm.id, p.id, t.id].join(',') + '") { similar_items(first: 10000) { edges { node { dbid, claim_description { id, fact_check { id } } } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal pm2.id, JSON.parse(@response.body)['data']['project_media']['similar_items']['edges'][0]['node']['dbid'] - Bot::Alegre.unstub(:get_similar_texts) + Bot::Alegre.unstub(:get_items_from_similar_text) end test "should create and update flags and content warning" do diff --git a/test/controllers/reports_controller_test.rb b/test/controllers/reports_controller_test.rb index 0af43ffdd..45b1785a4 100644 --- a/test/controllers/reports_controller_test.rb +++ b/test/controllers/reports_controller_test.rb @@ -13,17 +13,15 @@ def setup def from_alegre(pm) { - '_index' => 'alegre_similarity', - '_type' => '_doc', + 'index' => 'alegre_similarity', '_id' => 'tMXj53UB36CYclMPXp14', - '_score' => 0.9, - '_source' => { - 'content' => 'Test', - 'context' => { - 'team_id' => pm.team_id.to_s, - 'field' => 'original_title', - 'project_media_id' => pm.id.to_s - } + 'id' => 'tMXj53UB36CYclMPXp14', + 'score' => 0.9, + 'content' => 'Test', + 'context' => { + 'team_id' => pm.team_id.to_s, + 'field' => 'original_title', + 'project_media_id' => pm.id.to_s } } end diff --git a/test/lib/smooch_nlu_test.rb b/test/lib/smooch_nlu_test.rb index 86e01015f..ee73b38ba 100644 --- a/test/lib/smooch_nlu_test.rb +++ b/test/lib/smooch_nlu_test.rb @@ -64,7 +64,7 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) nlu.enable! - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe') expected_output = { 'en' => { @@ -85,7 +85,7 @@ def create_team_with_smooch_bot_installed end test 'should add keyword if it does not exist' do - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -93,9 +93,9 @@ def create_team_with_smooch_bot_installed test 'should not add keyword if it exists' do team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.never + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.never nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -114,8 +114,8 @@ def create_team_with_smooch_bot_installed end test 'should return a menu option if NLU is enabled' do - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => 'test' } } }, + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ + { 'score' => 0.9, 'context' => { 'menu_option_id' => 'test' } }, ]}) team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).enable! diff --git a/test/models/bot/alegre_2_test.rb b/test/models/bot/alegre_2_test.rb index 29e105664..efe63f0d4 100644 --- a/test/models/bot/alegre_2_test.rb +++ b/test/models/bot/alegre_2_test.rb @@ -315,7 +315,7 @@ def teardown WebMock.stub_request(:post, 'http://alegre.test/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre.test/text/similarity/').to_return(body: { success: true }.to_json) WebMock.stub_request(:delete, 'http://alegre.test/image/similarity/').to_return(body: { success: true }.to_json) - WebMock.stub_request(:post, 'http://alegre.test/text/similarity/search/').to_return(body: { success: true }.to_json) + WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/text').to_return(body: { success: true }.to_json) WebMock.stub_request(:post, 'http://alegre.test/image/ocr/').to_return(body: { text: 'Foo bar' }.to_json) WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/image').to_return(body: { result: [ diff --git a/test/models/bot/alegre_3_test.rb b/test/models/bot/alegre_3_test.rb index 5a7a90cfe..1d9b74cec 100644 --- a/test/models/bot/alegre_3_test.rb +++ b/test/models/bot/alegre_3_test.rb @@ -149,7 +149,7 @@ def teardown WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: { "success": true }.to_json) diff --git a/test/models/bot/alegre_test.rb b/test/models/bot/alegre_test.rb index bea721a37..2310322f0 100644 --- a/test/models/bot/alegre_test.rb +++ b/test/models/bot/alegre_test.rb @@ -44,7 +44,8 @@ def teardown WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: 'success') + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: 'success') + WebMock.stub_request(:post, 'http://alegre/similarity/async/text').to_return(body: 'success') WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ Bot::Alegre.any_instance.stubs(:get_language).raises(RuntimeError) assert_nothing_raised do @@ -102,6 +103,7 @@ def teardown test "should unarchive item after running" do WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/async/text').to_return(body: {results: []}.to_json) stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(status: 200, body: '{}') pm = create_project_media @@ -205,7 +207,7 @@ def teardown test "should index report data" do WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) pm = create_project_media team: @team assert_nothing_raised do publish_report(pm) @@ -214,7 +216,7 @@ def teardown test "should use OCR data for similarity matching" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') pm = create_project_media team: @team pm2 = create_project_media team: @team Bot::Alegre.stubs(:get_items_with_similar_description).returns({ pm2.id => {:score=>0.9, :context=>{"team_id"=>@team.id, "field"=>"original_description", "project_media_id"=>pm2.id, "has_custom_id"=>true}, :model=>"elasticsearch"} }) @@ -262,7 +264,7 @@ def teardown # This test to reproduce errbit error CHECK-1218 test "should match to existing parent" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') pm_s = create_project_media team: @team pm = create_project_media team: @team pm2 = create_project_media team: @team @@ -280,7 +282,7 @@ def teardown test "should use transcription data for similarity matching" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') json_schema = { type: 'object', required: ['job_name'], @@ -305,7 +307,7 @@ def teardown end test "should check existing relationship before create a new one" do - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') pm = create_project_media team: @team pm2 = create_project_media team: @team diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb index f99d11b11..79e7ac4a3 100644 --- a/test/models/bot/alegre_v2_test.rb +++ b/test/models/bot/alegre_v2_test.rb @@ -37,7 +37,7 @@ def teardown tpm = TemporaryProjectMedia.new tpm.type = k assert_equal tpm.media.type, v - [:is_blank?, :is_link?, :is_text?, :is_image?, :is_video?, :is_audio?].each do |meth| + [:is_blank?, :is_link?, :is_text?, :is_image?, :is_video?, :is_audio?, :is_uploaded_media?].each do |meth| assert_equal [true, false].include?(tpm.send(meth)), true end end @@ -484,6 +484,15 @@ def teardown assert_equal JSON.parse(Bot::Alegre.delete(pm1).to_json), JSON.parse(response.to_json) end + test "should return false and log error during delete request" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + Bot::Alegre.stubs(:request_delete).raises(StandardError) + Rails.logger.expects(:error).with("[Alegre Bot] Exception on Delete for ProjectMedia ##{pm1.id}: Bot::Alegre::Error - StandardError").returns(nil) + CheckSentry.expects(:notify).with(instance_of(Bot::Alegre::Error), bot: "alegre", project_media: pm1, params: {}, field: nil).returns(false) + result = Bot::Alegre.delete(pm1) + assert_equal false, result + end + test "should get_items" do pm1 = create_project_media team: @team, media: create_uploaded_audio response = { diff --git a/test/models/bot/fetch_2_test.rb b/test/models/bot/fetch_2_test.rb index a457b66bf..3c2ef02d6 100644 --- a/test/models/bot/fetch_2_test.rb +++ b/test/models/bot/fetch_2_test.rb @@ -39,7 +39,7 @@ def setup WebMock.stub_request(:get, 'http://fetch:8000/claim_reviews?end_time=2017-08-11&include_raw=false&offset=0&per_page=100&service=test&start_time=2017-08-06').to_return(body: [@claim_review].to_json) WebMock.stub_request(:post, 'http://fetch:8000/subscribe').with(body: { service: 'foo', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test', language: nil }.to_json).to_return(body: '{}') WebMock.stub_request(:delete, 'http://fetch:8000/subscribe').with(body: { service: 'test', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test'}.to_json).to_return(body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) create_verification_status_stuff diff --git a/test/models/bot/fetch_test.rb b/test/models/bot/fetch_test.rb index 83bf68161..33728ef05 100644 --- a/test/models/bot/fetch_test.rb +++ b/test/models/bot/fetch_test.rb @@ -44,7 +44,7 @@ def setup WebMock.stub_request(:get, 'http://fetch:8000/claim_reviews?end_time=2017-08-11&include_raw=false&offset=0&per_page=100&service=test&start_time=2017-08-06').to_return(body: [@claim_review].to_json) WebMock.stub_request(:post, 'http://fetch:8000/subscribe').with(body: { service: 'foo', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test', language: nil }.to_json).to_return(body: '{}') WebMock.stub_request(:delete, 'http://fetch:8000/subscribe').with(body: { service: 'test', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test'}.to_json).to_return(body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) create_verification_status_stuff diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 5fa32450e..718b018df 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -139,6 +139,8 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should submit query without details on tipline bot v2" do WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1' assert_saved_query_type 'default_requests' @@ -208,6 +210,8 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query with details on tipline bot v2" do + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, '2', random_string, claim, '1' @@ -285,7 +289,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query and handle search error on tipline bot v2" do - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers CheckSearch.any_instance.stubs(:medias).raises(StandardError) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' @@ -384,7 +388,7 @@ def send_message_outside_24_hours_window(template, pm = nil) ProjectMedia.any_instance.stubs(:report_status).returns('published') ProjectMedia.any_instance.stubs(:analysis_published_article_url).returns(random_url) Bot::Alegre.stubs(:get_merged_similar_items).returns({ create_project_media.id => { score: 0.9 } }) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', "Foo bar foo bar #{url} foo bar", '1' end @@ -693,7 +697,7 @@ def send_message_outside_24_hours_window(template, pm = nil) pm = create_project_media team: @team publish_report(pm, {}, nil, { language: 'pt', use_visual_card: false }) Bot::Smooch.stubs(:get_search_results).returns([pm]) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' end @@ -807,9 +811,9 @@ def send_message_outside_24_hours_window(template, pm = nil) test 'should process menu option using NLU' do # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords for the newsletter menu option nlu = SmoochNlu.new(@team.slug) @@ -822,9 +826,9 @@ def send_message_outside_24_hours_window(template, pm = nil) subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id'] # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, - { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ + { 'score' => 0.9, 'context' => { 'menu_option_id' => subscription_option_id } }, + { 'score' => 0.2, 'context' => { 'menu_option_id' => query_option_id } } ]}) # Sending a message about the newsletter should take to the newsletter state, as per configurations done above @@ -836,9 +840,9 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_state 'main' # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ - { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, - { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ + { 'score' => 0.96, 'context' => { 'menu_option_id' => subscription_option_id } }, + { 'score' => 0.91, 'context' => { 'menu_option_id' => query_option_id } } ]}) # Sending a message that returns more than one option (disambiguation) @@ -875,9 +879,9 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.fake! do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords to a new "About Us" resource nlu = SmoochNlu.new(@team.slug) @@ -887,9 +891,9 @@ def send_message_outside_24_hours_window(template, pm = nil) r.add_keyword('who are you') # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /who are you/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'resource_id' => 0 } } }, - { '_score' => 0.8, '_source' => { 'context' => { 'resource_id' => r.id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns({ 'result' => [ + { '_score' => 0.9, 'context' => { 'resource_id' => 0 } }, + { '_score' => 0.8, 'context' => { 'resource_id' => r.id } } ]}) # Sending a message asking about the tipline should send a resource, as per configurations done above diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 4fd46ac40..9b970fb47 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -600,7 +600,8 @@ def teardown end test "should include claim_description_content in smooch search" do - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/async/image').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) RequestStore.store[:skip_cached_field_update] = false t = create_team m = create_uploaded_image diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index a74124c71..7d191e1e7 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -83,6 +83,7 @@ def setup end test "should create tag texts when setting tags" do + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do assert_difference 'TagText.count' do create_explainer tags: ['foo'] @@ -99,12 +100,12 @@ def setup } # Index two paragraphs and title when the explainer is created - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(3) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(3) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).never ex = create_explainer description: description # Update the index when paragraphs change - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(2) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).once ex = Explainer.find(ex.id) ex.description = 'Now this is the only paragraph' diff --git a/test/models/project_media_5_test.rb b/test/models/project_media_5_test.rb index 709aa4450..ef8ba5a0d 100644 --- a/test/models/project_media_5_test.rb +++ b/test/models/project_media_5_test.rb @@ -174,15 +174,6 @@ def setup assert pm.is_link? end - test "checks truthfulness of is_uploaded_image?" do - u = create_user - t = create_team - pm = create_project_media team: t - pm.media.type = "UploadedImage" - pm.media.save! - assert pm.is_uploaded_image? - end - test "checks truthfulness of is_image?" do u = create_user t = create_team diff --git a/test/models/request_test.rb b/test/models/request_test.rb index 354bd184e..565e8fab4 100644 --- a/test/models/request_test.rb +++ b/test/models/request_test.rb @@ -116,7 +116,7 @@ def setup end test "should send text request to Alegre" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) assert_nothing_raised do create_request(media: create_claim_media) end @@ -124,7 +124,6 @@ def setup end test "should send media request to Alegre" do - Bot::Alegre.stubs(:request).returns(true) assert_nothing_raised do create_request(media: create_uploaded_image) end @@ -132,14 +131,14 @@ def setup end test "should attach to similar text long" do - Bot::Alegre.stubs(:request).returns(true) f = create_feed + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns({}) m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar foo bar 2' r2 = create_request media: m2, feed: f - response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + response = { 'result' => [{ 'context' => { 'request_id' => r1.id } }] } + Bot::Alegre.stubs(:request).with('post', '/similarity/sync/text', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should be called with ES and vector model for request with 4 or more words assert_equal r1, r2.reload.similar_to_request @@ -148,14 +147,14 @@ def setup end test "should attach to similar text short" do - Bot::Alegre.stubs(:request).returns(true) f = create_feed + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns({}) m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar 2' r2 = create_request media: m2, feed: f - response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + response = { 'result' => [{ 'context' => { 'request_id' => r1.id } }] } + Bot::Alegre.stubs(:request).with('post', '/similarity/sync/text', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should only be called with vector models for 2 or 3 word request assert_equal r1, r2.reload.similar_to_request @@ -164,7 +163,7 @@ def setup end test "should not attach to similar text short" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) f = create_feed m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f @@ -193,7 +192,7 @@ def setup # end test "should attach to similar link" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) f = create_feed m = create_valid_media create_request request_type: 'text', media: m diff --git a/test/workers/generic_worker_test.rb b/test/workers/generic_worker_test.rb index baf671cec..3c1917feb 100644 --- a/test/workers/generic_worker_test.rb +++ b/test/workers/generic_worker_test.rb @@ -3,6 +3,7 @@ class GenericWorkerTest < ActiveSupport::TestCase def setup require 'sidekiq/testing' + WebMock.disable_net_connect! allow: /http:\/\/bot|#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ Sidekiq::Worker.clear_all end diff --git a/test/workers/reindex_alegre_workspace_test.rb b/test/workers/reindex_alegre_workspace_test.rb index afa42eb9f..a294946fb 100644 --- a/test/workers/reindex_alegre_workspace_test.rb +++ b/test/workers/reindex_alegre_workspace_test.rb @@ -25,7 +25,7 @@ def setup @tbi.save Bot::Alegre.stubs(:get_alegre_tbi).returns(TeamBotInstallation.new) Sidekiq::Testing.inline! - Bot::Alegre.stubs(:request).with('post', '/text/bulk_similarity/', anything).returns("done") + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns("done") end def teardown @@ -58,16 +58,20 @@ def teardown test "checks alegre package in get_request_doc" do package = { - :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), - :text=>"Some text", - :context=>{ - :team_id=>@pm.team_id, - :project_media_id=>@pm.id, - :has_custom_id=>true, - :temporary_media=>false, - :field=>"title" + :doc=>{ + :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), + :text=>"Some text", + :context=>{ + :team_id=>@pm.team_id, + :project_media_id=>@pm.id, + :has_custom_id=>true, + :temporary_media=>false, + :field=>"title" + }, + :models=>["elasticsearch"], + :requires_callback=>true }, - :models=>["elasticsearch"] + :type=>"text" } response = ReindexAlegreWorkspace.new.get_request_doc(@pm, "title", "Some text") assert_equal package, response @@ -93,18 +97,21 @@ def teardown test "tests the parallel request" do package = { - :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), - :text=>"Some text", - :context=>{ - :team_id=>@pm.team_id, - :project_media_id=>@pm.id, - :has_custom_id=>true, - :temporary_media=>false, - :field=>"title" + :doc=>{ + :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), + :text=>"Some text", + :context=>{ + :team_id=>@pm.team_id, + :project_media_id=>@pm.id, + :has_custom_id=>true, + :temporary_media=>false, + :field=>"title" + }, + :models=>["elasticsearch"] }, - :models=>["elasticsearch"] + :type=>"text" } - response = ReindexAlegreWorkspace.new.check_for_write(1.upto(30).collect{|x| package}, "a", @team.id, true, 1) + response = ReindexAlegreWorkspace.new.check_for_write(1.upto(30).collect{|x| package}, "a", @team.id, true) assert_equal Array, response.class end