From 1f4672f4a30f865f98ec8edf31d14f9ae96be09a Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 10 Dec 2024 21:41:39 +0200 Subject: [PATCH 01/52] CV2-5628: fix OR condition and remove projects filter (#2125) * CV2-5628: fix OR condition and remove projects filter * CV2-5628: fix tests 1/2 * CV2-5628: fix tests 2/2 * CV2-5628: fix feed query * CV2-5628: fix tests * CV2-5628: cleanup * CV2-5628: fix feed conditions * CV2-5628: cleanup --- lib/check_search.rb | 70 +++++++++---------- test/controllers/elastic_search_10_test.rb | 12 +--- test/controllers/elastic_search_4_test.rb | 54 +++----------- test/controllers/elastic_search_7_test.rb | 10 ++- test/controllers/elastic_search_test.rb | 34 --------- .../controllers/graphql_controller_10_test.rb | 29 -------- test/controllers/graphql_controller_3_test.rb | 36 ++++------ test/controllers/graphql_controller_6_test.rb | 60 ++++------------ test/lib/team_statistics_test.rb | 2 +- test/models/bot/smooch_5_test.rb | 16 ++--- test/models/project_media_test.rb | 16 ++--- 11 files changed, 85 insertions(+), 254 deletions(-) diff --git a/lib/check_search.rb b/lib/check_search.rb index 3953f762e7..17e64282db 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -2,7 +2,7 @@ class CheckSearch include SearchHelper def initialize(options, file = nil, team_id = Team.current&.id) - # Options include keywords, projects, tags, status, report status + # Options include search filters options = begin JSON.parse(options) rescue {} end @options = options.to_h.clone.with_indifferent_access @options['input'] = options.clone @@ -22,9 +22,6 @@ def initialize(options, file = nil, team_id = Team.current&.id) @options['esoffset'] ||= 0 adjust_es_window_size - # Check for non project - @options['none_project'] = @options['projects'].include?('-1') unless @options['projects'].blank? - adjust_project_filter adjust_channel_filter adjust_numeric_range_filter adjust_archived_filter @@ -37,8 +34,6 @@ def initialize(options, file = nil, team_id = Team.current&.id) # Apply feed filters @feed_view = @options['feed_view'] || :fact_check @options.merge!(@feed.get_feed_filters(@feed_view)) if feed_query? - - (Project.current ||= Project.where(id: @options['projects'].last).last) if @options['projects'].to_a.size == 1 @file = file end @@ -244,9 +239,7 @@ def get_pg_results_for_media core_conditions = {} core_conditions['team_id'] = @options['team_id'] if @options['team_id'].is_a?(Array) # Add custom conditions for array values - { - 'project_id' => 'projects', 'user_id' => 'users', 'source_id' => 'sources', 'read' => 'read', 'unmatched' => 'unmatched' - }.each do |k, v| + { 'user_id' => 'users', 'source_id' => 'sources', 'read' => 'read', 'unmatched' => 'unmatched'}.each do |k, v| custom_conditions[k] = [@options[v]].flatten if @options.has_key?(v) end core_conditions.merge!({ archived: @options['archived'] }) @@ -286,6 +279,17 @@ def get_search_field end def medias_query + return build_feed_conditions if feed_query? + and_conditions, or_conditions, not_conditions = build_es_medias_query + # Build ES query using this format: `bool: { must: [{and_conditions}], should: [{or_conditions}, must_not: [{not_conditions}]] }` + query = {} + { must: and_conditions, should: or_conditions, must_not: not_conditions }.each do |k, v| + query[k] = v.flatten unless v.blank? + end + { bool: query } + end + + def build_es_medias_query core_conditions = [] custom_conditions = [] core_conditions << { terms: { get_search_field => @options['project_media_ids'] } } unless @options['project_media_ids'].blank? @@ -313,16 +317,16 @@ def medias_query custom_conditions.concat request_language_conditions custom_conditions.concat report_language_conditions custom_conditions.concat team_tasks_conditions - feed_conditions = build_feed_conditions - conditions = [] + and_conditions = core_conditions + or_conditions = [] + not_conditions = [] if @options['operator'].upcase == 'OR' - and_conditions = { bool: { must: core_conditions } } - or_conditions = { bool: { should: custom_conditions } } - conditions = [and_conditions, or_conditions, feed_conditions] + or_conditions << custom_conditions + not_conditions << { term: { associated_type: { value: "Blank" } } } else - conditions = [{ bool: { must: (core_conditions + custom_conditions) } }, feed_conditions] + and_conditions.concat(custom_conditions) end - { bool: { must: conditions.reject{ |c| c.blank? } } } + return and_conditions, or_conditions, not_conditions end def medias_get_search_result(query) @@ -418,25 +422,6 @@ def adjust_es_window_size @options['eslimit'] = window_size - @options['esoffset'].to_i if current_size > window_size end - def adjust_project_filter - team_id = [@options['team_id']].flatten.first - project_group_ids = [@options['project_group_id']].flatten.reject{ |pgid| pgid.blank? }.map(&:to_i) - unless project_group_ids.empty? - project_ids = @options['projects'].to_a.map(&:to_i) - project_groups_project_ids = Project.where(project_group_id: project_group_ids, team_id: team_id).map(&:id) - - project_ids = project_ids.blank? ? project_groups_project_ids : (project_ids & project_groups_project_ids) - - # Invalidate the search if empty... otherwise, adjust the projects filter - @options['projects'] = project_ids.empty? ? [0] : project_ids - end - if Team.current && !feed_query? && [@options['team_id']].flatten.size == 1 - t = Team.find(team_id) - @options['projects'] = @options['projects'].blank? ? (Project.where(team_id: t.id).map(&:id) + [nil]) : Project.where(id: @options['projects'], team_id: t.id).map(&:id) - end - @options['projects'] += [nil] if @options['none_project'] - end - def adjust_channel_filter if @options['channels'].is_a?(Array) && @options['channels'].include?('any_tipline') channels = @options['channels'] - ['any_tipline'] @@ -700,7 +685,7 @@ def doc_conditions doc_c << { terms: { 'associated_type': types } } end - fields = { 'project_id' => 'projects', 'user_id' => 'users' } + fields = { 'user_id' => 'users' } status_search_fields.each do |field| fields[field] = field end @@ -780,12 +765,21 @@ def hit_es_for_range_filter end def build_feed_conditions - return {} unless feed_query? + return [] unless feed_query? conditions = [] + feed_options = @options.clone + feed_options.delete('feed_id') + feed_options.delete('input') + and_conditions, or_conditions, not_conditions = CheckSearch.new(feed_options.to_json, nil, @options['team_id']).build_es_medias_query @feed.get_team_filters(@options['feed_team_ids']).each do |filters| team_id = filters['team_id'].to_i conditions << CheckSearch.new(filters.merge({ show_similar: !!@options['show_similar'] }).to_json, nil, team_id).medias_query end - { bool: { should: conditions } } + or_conditions.concat(conditions) + query = [] + { must: and_conditions, should: or_conditions, must_not: not_conditions}.each do |k, v| + query << { bool: { "#{k}": v } } unless v.blank? + end + { bool: { must: query } } end end diff --git a/test/controllers/elastic_search_10_test.rb b/test/controllers/elastic_search_10_test.rb index 82199c9852..b5a42a3c98 100644 --- a/test/controllers/elastic_search_10_test.rb +++ b/test/controllers/elastic_search_10_test.rb @@ -140,22 +140,14 @@ def setup end end - test "should filter items by non project and read-unread" do + test "should filter items by read-unread" do t = create_team - p = create_project team: t u = create_user create_team_user team: t, user: u, role: 'admin' with_current_user_and_team(u ,t) do pm = create_project_media team: t, disable_es_callbacks: false - pm2 = create_project_media project: p, disable_es_callbacks: false + pm2 = create_project_media team: t, disable_es_callbacks: false pm3 = create_project_media team: t, quote: 'claim a', disable_es_callbacks: false - results = CheckSearch.new({ projects: ['-1'] }.to_json) - # result should return empty as now all items should have a project CHECK-1150 - assert_empty results.medias.map(&:id) - results = CheckSearch.new({ projects: [p.id, '-1'] }.to_json) - assert_equal [pm2.id], results.medias.map(&:id) - results = CheckSearch.new({ keyword: 'claim', projects: ['-1'] }.to_json) - assert_empty results.medias.map(&:id) # test read/unread pm.read = true pm.save! diff --git a/test/controllers/elastic_search_4_test.rb b/test/controllers/elastic_search_4_test.rb index 5276dc01cb..0cb5351b99 100644 --- a/test/controllers/elastic_search_4_test.rb +++ b/test/controllers/elastic_search_4_test.rb @@ -8,10 +8,8 @@ def setup test "should search with multiple filters" do t = create_team - p = create_project team: t - p2 = create_project team: t - pm = create_project_media project: p, quote: 'report_title', disable_es_callbacks: false - pm2 = create_project_media project: p2, quote: 'report_title', disable_es_callbacks: false + pm = create_project_media team: t, quote: 'report_title', disable_es_callbacks: false + pm2 = create_project_media team: t, quote: 'report_title', disable_es_callbacks: false create_tag tag: 'sports', annotated: pm, disable_es_callbacks: false create_tag tag: 'sports', annotated: pm2, disable_es_callbacks: false create_status status: 'verified', annotated: pm, disable_es_callbacks: false @@ -20,37 +18,16 @@ def setup Team.current = t result = CheckSearch.new({keyword: 'report_title', tags: ['sports']}.to_json) assert_equal [pm2.id, pm.id], result.medias.map(&:id) - # keyword & context - result = CheckSearch.new({keyword: 'report_title', projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) # keyword & status result = CheckSearch.new({keyword: 'report_title', verification_status: ['verified']}.to_json) assert_equal [pm.id], result.medias.map(&:id) - # tags & context - result = CheckSearch.new({projects: [p.id], tags: ['sports']}.to_json) - assert_equal [pm.id], result.medias.map(&:id) - # status & context - result = CheckSearch.new({projects: [p.id], verification_status: ['verified']}.to_json) - assert_equal [pm.id], result.medias.map(&:id) - # keyword & tags & context - result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) - # keyword & status & context - result = CheckSearch.new({keyword: 'report_title', verification_status: ['verified'], projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) - # tags & context & status - result = CheckSearch.new({tags: ['sports'], verification_status: ['verified'], projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) # keyword & tags & status result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], verification_status: ['verified']}.to_json) assert_equal [pm.id], result.medias.map(&:id) - # keyword & tags & context & status - result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], verification_status: ['verified'], projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) # search keyword in comments create_comment text: 'add_comment', annotated: pm, disable_es_callbacks: false sleep 1 - result = CheckSearch.new({keyword: 'add_comment', projects: [p.id]}.to_json) + result = CheckSearch.new({keyword: 'add_comment'}.to_json) assert_equal [pm.id], result.medias.map(&:id) end @@ -200,34 +177,21 @@ def setup c2 = create_claim_media m = create_valid_media t1 = create_team - p1a = create_project team: t1 - p1b = create_project team: t1 - pm1a = create_project_media project: p1a, media: c, disable_es_callbacks: false - sleep 1 - pm1b = create_project_media project: p1b, media: c2, disable_es_callbacks: false - + pm1a = create_project_media team: t1, media: c, disable_es_callbacks: false + pm1b = create_project_media team: t1, media: c2, disable_es_callbacks: false t2 = create_team - p2a = create_project team: t2 - p2b = create_project team: t2 - pm2a = create_project_media project: p2a, media: m, disable_es_callbacks: false - sleep 1 - pm2b = create_project_media project: p2b, disable_es_callbacks: false - + pm2a = create_project_media team: t2, media: m, disable_es_callbacks: false + pm2b = create_project_media team: t2, disable_es_callbacks: false + sleep 2 Team.current = t1 assert_equal [pm1b, pm1a], CheckSearch.new('{}').medias assert_equal 2, CheckSearch.new('{}').project_medias.count - assert_equal 1, CheckSearch.new({ projects: [p1a.id], show: ['claims']}.to_json).project_medias.count - assert_equal [pm1a], CheckSearch.new({ projects: [p1a.id] }.to_json).medias - assert_equal 1, CheckSearch.new({ projects: [p1a.id] }.to_json).project_medias.count + assert_equal 2, CheckSearch.new({ show: ['claims']}.to_json).project_medias.count assert_equal [pm1a, pm1b], CheckSearch.new({ sort_type: 'ASC' }.to_json).medias assert_equal 2, CheckSearch.new({ sort_type: 'ASC' }.to_json).project_medias.count - Team.current = nil - Team.current = t2 assert_equal [pm2b, pm2a], CheckSearch.new('{}').medias assert_equal 2, CheckSearch.new('{}').project_medias.count - assert_equal [pm2a], CheckSearch.new({ projects: [p2a.id] }.to_json).medias - assert_equal 1, CheckSearch.new({ projects: [p2a.id] }.to_json).project_medias.count assert_equal [pm2a, pm2b], CheckSearch.new({ sort_type: 'ASC' }.to_json).medias assert_equal 2, CheckSearch.new({ sort_type: 'ASC' }.to_json).project_medias.count Team.current = nil diff --git a/test/controllers/elastic_search_7_test.rb b/test/controllers/elastic_search_7_test.rb index dca282fad6..128556df40 100644 --- a/test/controllers/elastic_search_7_test.rb +++ b/test/controllers/elastic_search_7_test.rb @@ -159,16 +159,14 @@ def setup test "should parse search options" do t = create_team - p = create_project team: t - p2 = create_project team: t - pm = create_project_media project: p, disable_es_callbacks: false - pm2 = create_project_media project: p2, disable_es_callbacks: false + pm = create_project_media team: t, archived: 0, disable_es_callbacks: false + pm2 = create_project_media team: t, archived: 2, disable_es_callbacks: false sleep 1 Team.current = t - result = CheckSearch.new({projects: [p.id]}.to_json) + result = CheckSearch.new({archived: [0]}.to_json) assert_equal [pm.id], result.medias.map(&:id) # pass wrong format should map to all items - result = CheckSearch.new({projects: [p.id]}) + result = CheckSearch.new({archived: [0]}) assert_equal [pm.id, pm2.id], result.medias.map(&:id).sort end diff --git a/test/controllers/elastic_search_test.rb b/test/controllers/elastic_search_test.rb index 71bddc19f2..16d7f4e6b1 100644 --- a/test/controllers/elastic_search_test.rb +++ b/test/controllers/elastic_search_test.rb @@ -157,40 +157,6 @@ def setup assert_equal [pm.id], result.medias.map(&:id) end - test "should search with context" do - t = create_team - p = create_project team: t - pender_url = CheckConfig.get('pender_url_private') + '/api/medias' - url = 'http://test.com' - response = '{"type":"media","data":{"url":"' + url + '/normalized","type":"item", "title": "search_title", "description":"search_desc"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response) - url2 = 'http://test2.com' - response = '{"type":"media","data":{"url":"' + url2 + '/normalized","type":"item", "title": "search_title", "description":"search_desc"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url2 } }).to_return(body: response) - m = create_media(account: create_valid_account, url: url) - m1 = create_media(account: create_valid_account, url: url2) - pm = create_project_media project: p, media: m, disable_es_callbacks: false - keyword = { projects: [0,0,0] }.to_json - sleep 1 - Team.current = t - result = CheckSearch.new(keyword) - assert_empty result.medias - result = CheckSearch.new({projects: [p.id]}.to_json) - assert_equal [pm.id], result.medias.map(&:id) - # add a new context to existing media - p2 = create_project team: t - pm2 = create_project_media project: p2, media: m1, disable_es_callbacks: false - sleep 1 - result = CheckSearch.new({projects: [p.id]}.to_json) - assert_equal [pm.id].sort, result.medias.map(&:id).sort - # add a new media to same context - m2 = create_valid_media - pm2 = create_project_media project: p, media: m2, disable_es_callbacks: false - sleep 1 - result = CheckSearch.new({projects: [p.id]}.to_json) - assert_equal [pm.id, pm2.id].sort, result.medias.map(&:id).sort - end - test "should search with tags or status" do t = create_team p = create_project team: t diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb index b7ad00320f..fb61cf884e 100644 --- a/test/controllers/graphql_controller_10_test.rb +++ b/test/controllers/graphql_controller_10_test.rb @@ -515,35 +515,6 @@ def setup assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } end - test "should search by project group" do - u = create_user is_admin: true - t = create_team - create_team_user user: u, team: t, role: 'admin' - authenticate_with_user(u) - - pg = create_project_group team: t - p1 = create_project team: t - p1.project_group = pg - p1.save! - create_project_media project: p1 - p2 = create_project team: t - p2.project_group = pg - p2.save! - create_project_media project: p2 - p3 = create_project team: t - create_project_media project: p3 - - query = 'query CheckSearch { search(query: "{}") { number_of_results } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal 3, JSON.parse(@response.body)['data']['search']['number_of_results'] - - query = 'query CheckSearch { search(query: "{\"project_group_id\":' + pg.id.to_s + '}") { number_of_results } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] - end - test "should not access GraphQL mutation if not authenticated" do post :create, params: { query: 'mutation Test' } assert_response 401 diff --git a/test/controllers/graphql_controller_3_test.rb b/test/controllers/graphql_controller_3_test.rb index 267a64dc83..c8183e8eb4 100644 --- a/test/controllers/graphql_controller_3_test.rb +++ b/test/controllers/graphql_controller_3_test.rb @@ -18,22 +18,17 @@ def setup u = create_user is_admin: true authenticate_with_user(u) t1 = create_team - p1a = create_project team: t1 - p1b = create_project team: t1 - pm1a = create_project_media project: p1a, disable_es_callbacks: false ; sleep 1 - pm1b = create_project_media project: p1b, disable_es_callbacks: false ; sleep 1 + pm1a = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 + pm1b = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 pm1b.disable_es_callbacks = false ; pm1b.updated_at = Time.now ; pm1b.save! ; sleep 1 pm1a.disable_es_callbacks = false ; pm1a.updated_at = Time.now ; pm1a.save! ; sleep 1 - pm1c = create_project_media project: p1a, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::TRASHED ; sleep 1 + pm1c = create_project_media team: t1, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::TRASHED ; sleep 1 t2 = create_team - p2 = create_project team: t2 pm2 = [] 6.times do - pm2 << create_project_media(project: p2, disable_es_callbacks: false) - sleep 1 + pm2 << create_project_media(team: t2, disable_es_callbacks: false) end - - sleep 10 + sleep 2 # Default sort criteria and order: recent added, descending query = 'query CheckSearch { search(query: "{}") {medias(first:20){edges{node{dbid}}}}}' @@ -63,13 +58,6 @@ def setup results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } assert_equal [pm1b.id, pm1a.id], results - # Filter by project - query = 'query CheckSearch { search(query: "{\"projects\":[' + p1b.id.to_s + ']}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1b.id], results - # Get archived items query = 'query CheckSearch { search(query: "{\"archived\":1}") {medias(first:20){edges{node{dbid}}}}}' post :create, params: { query: query, team: t1.slug } @@ -78,10 +66,10 @@ def setup assert_equal [pm1c.id], results # Relationships - pm1e = create_project_media project: p1a, disable_es_callbacks: false ; sleep 1 - pm1f = create_project_media project: p1a, disable_es_callbacks: false, media: nil, quote: 'Test 1' ; sleep 1 - pm1g = create_project_media project: p1a, disable_es_callbacks: false, media: nil, quote: 'Test 2' ; sleep 1 - pm1h = create_project_media project: p1a, disable_es_callbacks: false, media: nil, quote: 'Test 3' ; sleep 1 + pm1e = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 + pm1f = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 1' ; sleep 1 + pm1g = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 2' ; sleep 1 + pm1h = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 3' ; sleep 1 create_relationship source_id: pm1e.id, target_id: pm1f.id, disable_es_callbacks: false ; sleep 1 create_relationship source_id: pm1e.id, target_id: pm1g.id, disable_es_callbacks: false ; sleep 1 create_relationship source_id: pm1e.id, target_id: pm1h.id, disable_es_callbacks: false ; sleep 1 @@ -94,21 +82,21 @@ def setup assert_equal [pm1f.id, pm1g.id, pm1h.id].sort, results.sort # Paginate, page 1 - query = 'query CheckSearch { search(query: "{\"projects\":[' + p2.id.to_s + '],\"eslimit\":2,\"esoffset\":0}") {medias(first:20){edges{node{dbid}}}}}' + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":0}") {medias(first:20){edges{node{dbid}}}}}' post :create, params: { query: query, team: t2.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } assert_equal [pm2[5].id, pm2[4].id], results # Paginate, page 2 - query = 'query CheckSearch { search(query: "{\"projects\":[' + p2.id.to_s + '],\"eslimit\":2,\"esoffset\":2}") {medias(first:20){edges{node{dbid}}}}}' + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":2}") {medias(first:20){edges{node{dbid}}}}}' post :create, params: { query: query, team: t2.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } assert_equal [pm2[3].id, pm2[2].id], results # Paginate, page 3 - query = 'query CheckSearch { search(query: "{\"projects\":[' + p2.id.to_s + '],\"eslimit\":2,\"esoffset\":4}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":4}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' post :create, params: { query: query, team: t2.slug } assert_response :success response = JSON.parse(@response.body)['data']['search'] diff --git a/test/controllers/graphql_controller_6_test.rb b/test/controllers/graphql_controller_6_test.rb index 6c752783a3..ca283d99ea 100644 --- a/test/controllers/graphql_controller_6_test.rb +++ b/test/controllers/graphql_controller_6_test.rb @@ -39,73 +39,40 @@ def teardown assert_not_nil json_response.dig('data', 'team', 'team_bot_installation', 'smooch_enabled_integrations') end - test "should search using OR or AND on PG" do + test "should search using OR or AND" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false t = create_team - p1 = create_project team: t - p2 = create_project team: t u = create_user + u2 = create_user create_team_user team: t, user: u, role: 'admin' authenticate_with_user(u) - - pm1 = create_project_media team: t, project: p1, read: true - pm2 = create_project_media team: t, project: p2, read: false - - query = 'query CheckSearch { search(query: "{\"operator\":\"AND\",\"read\":true,\"projects\":[' + p2.id.to_s + ']}") { medias(first: 20) { edges { node { dbid } } } } }' + pm1 = create_project_media team: t, user: u, read: true, disable_es_callbacks: false + pm2 = create_project_media team: t, user: u2, read: false, disable_es_callbacks: false + # PG + query = 'query CheckSearch { search(query: "{\"operator\":\"AND\",\"read\":true,\"users\":[' + u2.id.to_s + ']}") { medias(first: 20) { edges { node { dbid } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] } - - query = 'query CheckSearch { search(query: "{\"operator\":\"OR\",\"read\":true,\"projects\":[' + p2.id.to_s + ']}") { medias(first: 20) { edges { node { dbid } } } } }' + query = 'query CheckSearch { search(query: "{\"operator\":\"OR\",\"read\":true,\"users\":[' + u2.id.to_s + ']}") { medias(first: 20) { edges { node { dbid } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal [pm1.id, pm2.id].sort, JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] }.sort - end - - test "should search using OR or AND on ES" do - setup_elasticsearch - RequestStore.store[:skip_cached_field_update] = false - t = create_team - p1 = create_project team: t - p2 = create_project team: t - u = create_user - create_team_user team: t, user: u, role: 'admin' - authenticate_with_user(u) - - pm1 = create_project_media team: t, project: p1, read: true, disable_es_callbacks: false - pm2 = create_project_media team: t, project: p2, read: false, disable_es_callbacks: false - - query = 'query CheckSearch { search(query: "{\"operator\":\"AND\",\"read\":[1],\"projects\":[' + p2.id.to_s + '],\"report_status\":\"unpublished\"}") { medias(first: 20) { edges { node { dbid } } } } }' + # ES + query = 'query CheckSearch { search(query: "{\"operator\":\"AND\",\"read\":[1],\"users\":[' + u2.id.to_s + '],\"report_status\":\"unpublished\"}") { medias(first: 20) { edges { node { dbid } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] } - - query = 'query CheckSearch { search(query: "{\"operator\":\"OR\",\"read\":[1],\"projects\":[' + p2.id.to_s + '],\"report_status\":\"unpublished\"}") { medias(first: 20) { edges { node { dbid } } } } }' + query = 'query CheckSearch { search(query: "{\"operator\":\"OR\",\"read\":[1],\"users\":[' + u2.id.to_s + '],\"report_status\":\"unpublished\"}") { medias(first: 20) { edges { node { dbid } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal [pm1.id, pm2.id].sort, JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] }.sort end - test "should search by project" do - t = create_team - p = create_project team: t - u = create_user - create_team_user team: t, user: u, role: 'admin' - authenticate_with_user(u) - - create_project_media team: t, project: nil, project_id: nil - create_project_media project: p - - query = 'query CheckSearch { search(query: "{}") { medias(first: 20) { edges { node { dbid } } } } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal 2, JSON.parse(@response.body)['data']['search']['medias']['edges'].size - end - test "should search by similar image on PG" do t = create_team u = create_user is_admin: true authenticate_with_user(u) - pm = create_project_media team: t Bot::Alegre.stubs(:get_items_with_similar_media_v2).returns({ pm.id => 0.8 }) @@ -124,13 +91,11 @@ def teardown t = create_team u = create_user is_admin: true authenticate_with_user(u) - m = create_claim_media quote: 'Test' m2 = create_claim_media quote: 'Another Test' pm = create_project_media team: t, media: m pm2 = create_project_media team: t, media: m2 sleep 2 - Bot::Alegre.stubs(:get_items_with_similar_media_v2).returns({ pm.id => 0.8 }) path = File.join(Rails.root, 'test', 'data', 'rails.png') file = Rack::Test::UploadedFile.new(path, 'image/png') @@ -146,7 +111,6 @@ def teardown t = create_team u = create_user is_admin: true authenticate_with_user(u) - path = File.join(Rails.root, 'test', 'data', 'rails.png') file = Rack::Test::UploadedFile.new(path, 'image/png') query = 'mutation { searchUpload(input: {}) { file_handle, file_url } }' diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 7cd7a46f61..ca164c628e 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -96,7 +96,7 @@ def teardown sleep 2 object = TeamStatistics.new(@team, 'past_week', 'en') - expected = [{ id: fc2.id, label: 'Foo', value: 2 }, { id: fc1.id, label: 'Bar', value: 1 }] + expected = [{ id: pm2.fact_check_id, label: 'Foo', value: 2 }, { id: pm1.fact_check_id, label: 'Bar', value: 1 }] assert_equal expected, object.top_articles_sent end diff --git a/test/models/bot/smooch_5_test.rb b/test/models/bot/smooch_5_test.rb index 0321dca79f..93eef03afe 100644 --- a/test/models/bot/smooch_5_test.rb +++ b/test/models/bot/smooch_5_test.rb @@ -67,18 +67,18 @@ def teardown # Get feed data scoped by teams that are part of the feed, taking into account the filters for the feed # and for each team participating in the feed with_current_user_and_team(u, t1) do - # Keyword search - assert_equal [pm1a, pm1f, pm2a].sort, Bot::Smooch.search_for_similar_published_fact_checks('text', 'Test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).to_a.sort - + result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'Test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + assert_equal [pm1a.id, pm1f.id, pm2a.id].sort, result.sort # Text similarity search - assert_equal [pm1a, pm1d, pm2a], Bot::Smooch.search_for_similar_published_fact_checks('text', 'This is a test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).to_a - + result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'This is a test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + assert_equal [pm1a.id, pm1d.id, pm2a.id].sort, result.sort # Media similarity search - assert_equal [pm1a, pm1d, pm2a], Bot::Smooch.search_for_similar_published_fact_checks('image', random_url, [t1.id, t2.id, t3.id, t4.id], nil, f1.id).to_a - + result = Bot::Smooch.search_for_similar_published_fact_checks('image', random_url, [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + assert_equal [pm1a.id, pm1d.id, pm2a.id], result.sort # URL search - assert_equal [pm1g, pm2b].sort, Bot::Smooch.search_for_similar_published_fact_checks('text', "Test with URL: #{url}", [t1.id, t2.id, t3.id, t4.id], nil, f1.id).to_a.sort + result = Bot::Smooch.search_for_similar_published_fact_checks('text', "Test with URL: #{url}", [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + assert_equal [pm1g.id, pm2b.id].sort, result.sort end Bot::Alegre.unstub(:get_merged_similar_items) diff --git a/test/models/project_media_test.rb b/test/models/project_media_test.rb index cb8de03b95..21ebbda9fe 100644 --- a/test/models/project_media_test.rb +++ b/test/models/project_media_test.rb @@ -12,20 +12,14 @@ def setup test "should query media" do setup_elasticsearch t = create_team - p = create_project team: t - p1 = create_project team: t - p2 = create_project team: t - pm = create_project_media team: t, project_id: p.id, disable_es_callbacks: false - create_project_media team: t, project_id: p1.id, disable_es_callbacks: false - create_project_media team: t, archived: CheckArchivedFlags::FlagCodes::TRASHED, project_id: p.id, disable_es_callbacks: false - pm = create_project_media team: t, project_id: p1.id, disable_es_callbacks: false - create_relationship source_id: pm.id, target_id: create_project_media(team: t, project_id: p.id, disable_es_callbacks: false).id, relationship_type: Relationship.confirmed_type + pm = create_project_media team: t, disable_es_callbacks: false + create_project_media team: t, disable_es_callbacks: false + create_project_media team: t, archived: CheckArchivedFlags::FlagCodes::TRASHED, disable_es_callbacks: false + pm = create_project_media team: t, disable_es_callbacks: false + create_relationship source_id: pm.id, target_id: create_project_media(team: t, disable_es_callbacks: false).id, relationship_type: Relationship.confirmed_type sleep 2 assert_equal 3, CheckSearch.new({ team_id: t.id }.to_json, nil, t.id).medias.size assert_equal 4, CheckSearch.new({ show_similar: true, team_id: t.id }.to_json, nil, t.id).medias.size - assert_equal 2, CheckSearch.new({ team_id: t.id, projects: [p1.id] }.to_json, nil, t.id).medias.size - assert_equal 0, CheckSearch.new({ team_id: t.id, projects: [p2.id] }.to_json, nil, t.id).medias.size - assert_equal 1, CheckSearch.new({ team_id: t.id, projects: [p1.id], eslimit: 1 }.to_json, nil, t.id).medias.size end test "should handle indexing conflicts" do From 5020cbc2b6969860938b2ad301203acfc1ced0e6 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 11 Dec 2024 10:48:39 +0200 Subject: [PATCH 02/52] CV2-5373: Most relevant articles (#2130) * CV2-5373: add a new graphql fields & * CV2-5373: update relay files * CV2-5730: return dummy relevant articles (#2136) * CV2-5731 Refactoring smooch search (#2137) * CV2-5731: call tipline search_for_articles method and append Explainers if no articles exists * CV2-5731: fix graphql query and add more tests * CV2-5731: apply PR comments and add more tests * CV2-5761 list most relevant articles fact check and explainer for project media item (#2142) * CV2-5761: include FactCheck & Explainer for item most relevant * CV2-5751: fix articles sort and change the limit * CV2-5761: keep default sort (sort by score) * CV2-5761: enforce limit value as a method args * CV2-5761: cleanup * CV2-5761: add more tests * CV2-5761: add missing test to back coverage 100% * CV2-5761: apply PR comments * CV2-5373: check language exists for fc_language condition * CV2-5617: use workspace similarity settings for explainers (#2145) * CV2-5617: use workspace similarity settings for explainers * Refactor threshold getters * CV2-5617: fix tests * CV2-5617: return models_and_thresholds in Hash formatt --------- Co-authored-by: Devin Gaffney * Add Helper Method to Create and Publish Standalone Fact Checks for Check Web Testing (#2151) * Add new test helper method for creating standalone and published fact check - create_imported_standalone_fact_check method in TestController to create a standalone and published fact check and associate it with a team. - Updated `routes.rb` to include the new endpoint for fact checks. - Added tests for the create_imported_standalone_fact_check Reference: CV2-5737 * CV2-5373: fix tests --------- Co-authored-by: Devin Gaffney Co-authored-by: Daniele Valverde <34126648+danielevalverde@users.noreply.github.com> --- app/controllers/test_controller.rb | 33 ++++++++ app/graph/types/project_media_type.rb | 12 +++ app/models/bot/alegre.rb | 21 ++++-- app/models/concerns/smooch_search.rb | 45 +++++------ app/models/explainer.rb | 41 +++++----- app/models/project_media.rb | 16 ++++ app/models/team.rb | 24 ++++++ app/resources/api/v2/feed_resource.rb | 15 ++-- config/routes.rb | 1 + lib/relay.idl | 22 ++++++ public/relay.json | 75 +++++++++++++++++++ test/controllers/feeds_controller_test.rb | 16 ++-- test/controllers/graphql_controller_5_test.rb | 19 +++++ test/controllers/test_controller_test.rb | 47 ++++++++++++ test/models/bot/smooch_3_test.rb | 4 +- test/models/bot/smooch_4_test.rb | 6 +- test/models/bot/smooch_5_test.rb | 8 +- test/models/bot/smooch_7_test.rb | 30 ++++---- test/models/explainer_test.rb | 6 ++ test/models/project_media_7_test.rb | 48 ++++++++++++ test/models/team_2_test.rb | 34 +++++++++ 21 files changed, 439 insertions(+), 84 deletions(-) diff --git a/app/controllers/test_controller.rb b/app/controllers/test_controller.rb index efc12a92a0..69ec297339 100644 --- a/app/controllers/test_controller.rb +++ b/app/controllers/test_controller.rb @@ -225,6 +225,39 @@ def suggest_similarity_item render_success 'suggest_similarity', pm1 end + def create_imported_standalone_fact_check + team = Team.current = Team.find(params[:team_id]) + user = User.where(email: params[:email]).last + description = params[:description] + context = params[:context] + title = params[:title] + summary = params[:summary] + url = params[:url] + language = params[:language] || 'en' + + # Create ClaimDescription + claim_description = ClaimDescription.create!( + description: description, + context: context, + user: user, + team: team + ) + + # Set up FactCheck + fact_check = FactCheck.new( + claim_description: claim_description, + title: title, + summary: summary, + url: url, + language: language, + user: user, + publish_report: true, + report_status: 'published' + ) + fact_check.save! + render_success 'fact_check', fact_check + end + def random render html: "Test #{rand(100000).to_i}Test".html_safe end diff --git a/app/graph/types/project_media_type.rb b/app/graph/types/project_media_type.rb index 794c033264..986cd4097c 100644 --- a/app/graph/types/project_media_type.rb +++ b/app/graph/types/project_media_type.rb @@ -394,4 +394,16 @@ def articles_count count += 1 if object.fact_check count end + + field :relevant_articles, ::ArticleUnion.connection_type, null: true + + def relevant_articles + object.get_similar_articles + end + + field :relevant_articles_count, GraphQL::Types::Int, null: true + + def relevant_articles_count + object.get_similar_articles.count + end end diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index a1d05d8950..e2b9b393b0 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -256,11 +256,12 @@ def self.merge_suggested_and_confirmed(suggested_or_confirmed, confirmed, pm) end end - def self.get_matching_key_value(pm, media_type, similarity_method, automatic, model_name) + def self.get_threshold_given_model_settings(team_id, media_type, similarity_method, automatic, model_name) + tbi = nil + tbi = self.get_alegre_tbi(team_id) unless team_id.nil? similarity_level = automatic ? 'matching' : 'suggestion' generic_key = "#{media_type}_#{similarity_method}_#{similarity_level}_threshold" specific_key = "#{media_type}_#{similarity_method}_#{model_name}_#{similarity_level}_threshold" - tbi = self.get_alegre_tbi(pm&.team_id) settings = tbi.alegre_settings unless tbi.nil? outkey = "" value = nil @@ -274,17 +275,25 @@ def self.get_matching_key_value(pm, media_type, similarity_method, automatic, mo return [outkey, value] end - def self.get_threshold_for_query(media_type, pm, automatic = false) + def self.get_matching_key_value(pm, media_type, similarity_method, automatic, model_name) + self.get_threshold_given_model_settings(pm&.team_id, media_type, similarity_method, automatic, model_name) + end + + def self.get_similarity_methods_and_models_given_media_type_and_team_id(media_type, team_id, get_vector_settings) similarity_methods = media_type == 'text' ? ['elasticsearch'] : ['hash'] models = similarity_methods.dup - if media_type == 'text' && !pm.nil? - models_to_use = [self.matching_model_to_use(pm.team_id)].flatten-[Bot::Alegre::ELASTICSEARCH_MODEL] + if media_type == 'text' && get_vector_settings + models_to_use = [self.matching_model_to_use(team_id)].flatten-[Bot::Alegre::ELASTICSEARCH_MODEL] models_to_use.each do |model| similarity_methods << 'vector' models << model end end - similarity_methods.zip(models).collect do |similarity_method, model_name| + return similarity_methods.zip(models) + end + + def self.get_threshold_for_query(media_type, pm, automatic = false) + self.get_similarity_methods_and_models_given_media_type_and_team_id(media_type, pm&.team_id, !pm.nil?).collect do |similarity_method, model_name| key, value = self.get_matching_key_value(pm, media_type, similarity_method, automatic, model_name) { value: value.to_f, key: key, automatic: automatic, model: model_name} end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index ca6941093b..b306a4dc41 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -9,21 +9,22 @@ module ClassMethods def search(app_id, uid, language, message, team_id, workflow, provider = nil) platform = self.get_platform_from_message(message) begin + limit = CheckConfig.get('most_relevant_team_limit', 3, :integer) sm = CheckStateMachine.new(uid) self.get_installation(self.installation_setting_id_keys, app_id) if self.config.blank? RequestStore.store[:smooch_bot_provider] = provider unless provider.blank? query = self.get_search_query(uid, message) - results = self.get_search_results(uid, query, team_id, language).collect{ |pm| Relationship.confirmed_parent(pm) }.uniq + results = self.get_search_results(uid, query, team_id, language, limit).collect{ |pm| Relationship.confirmed_parent(pm) }.uniq reports = results.select{ |pm| pm.report_status == 'published' }.collect{ |pm| pm.get_dynamic_annotation('report_design') }.reject{ |r| r.nil? }.collect{ |r| r.report_design_to_tipline_search_result }.select{ |r| r.should_send_in_language?(language) } # Extract explainers from matched media if they don't have published fact-checks but they have explainers - reports = results.collect{ |pm| pm.explainers.to_a }.flatten.uniq.first(3).map(&:as_tipline_search_result) if !results.empty? && reports.empty? + reports = results.collect{ |pm| pm.explainers.to_a }.flatten.uniq.first(limit).map(&:as_tipline_search_result) if !results.empty? && reports.empty? # Search for explainers if fact-checks were not found if reports.empty? && query['type'] == 'text' - explainers = self.search_for_explainers(uid, query['text'], team_id, language).first(3).select{ |explainer| explainer.as_tipline_search_result.should_send_in_language?(language) } + explainers = self.search_for_explainers(uid, query['text'], team_id, limit, language).select{ |explainer| explainer.as_tipline_search_result.should_send_in_language?(language) } Rails.logger.info "[Smooch Bot] Text similarity search got #{explainers.count} explainers while looking for '#{query['text']}' for team #{team_id}" - results = explainers.collect{ |explainer| explainer.project_medias.to_a }.flatten.uniq.reject{ |pm| pm.blank? }.first(3) + results = explainers.collect{ |explainer| explainer.project_medias.to_a }.flatten.uniq.reject{ |pm| pm.blank? }.first(limit) reports = explainers.map(&:as_tipline_search_result) end @@ -100,9 +101,9 @@ def reject_temporary_results(results) end end - def parse_search_results_from_alegre(results, after = nil, feed_id = nil, team_ids = nil) + def parse_search_results_from_alegre(results, limit, after = nil, feed_id = nil, team_ids = nil) pms = reject_temporary_results(results).sort_by{ |a| [a[1][:model] != Bot::Alegre::ELASTICSEARCH_MODEL ? 1 : 0, a[1][:score]] }.to_h.keys.reverse.collect{ |id| Relationship.confirmed_parent(ProjectMedia.find_by_id(id)) } - filter_search_results(pms, after, feed_id, team_ids).uniq(&:id).sort_by{ |pm| pm.report_status == 'published' ? 0 : 1 }.first(3) + filter_search_results(pms, after, feed_id, team_ids).uniq(&:id).first(limit) end def date_filter(team_id) @@ -127,14 +128,14 @@ def get_search_query(uid, last_message) self.bundle_list_of_messages(list, last_message, true) end - def get_search_results(uid, message, team_id, language) + def get_search_results(uid, message, team_id, language, limit) results = [] begin type = message['type'] after = self.date_filter(team_id) query = message['text'] query = CheckS3.rewrite_url(message['mediaUrl']) unless type == 'text' - results = self.search_for_similar_published_fact_checks(type, query, [team_id], after, nil, language).select{ |pm| is_a_valid_search_result(pm) } + results = self.search_for_similar_published_fact_checks(type, query, [team_id], limit, after, nil, language).select{ |pm| is_a_valid_search_result(pm) } rescue StandardError => e self.handle_search_error(uid, e, language) end @@ -148,19 +149,19 @@ def normalized_query_hash(type, query, team_ids, after, feed_id, language) # "type" is text, video, audio or image # "query" is either a piece of text of a media URL - def search_for_similar_published_fact_checks(type, query, team_ids, after = nil, feed_id = nil, language = nil, skip_cache = false) + def search_for_similar_published_fact_checks(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil, skip_cache = false) if skip_cache - self.search_for_similar_published_fact_checks_no_cache(type, query, team_ids, after, feed_id, language) + self.search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after, feed_id, language) else Rails.cache.fetch("smooch:search_results:#{self.normalized_query_hash(type, query, team_ids, after, feed_id, language)}", expires_in: 2.hours) do - self.search_for_similar_published_fact_checks_no_cache(type, query, team_ids, after, feed_id, language) + self.search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after, feed_id, language) end end end # "type" is text, video, audio or image # "query" is either a piece of text of a media URL - def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, after = nil, feed_id = nil, language = nil) + def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil) results = [] pm = nil pm = ProjectMedia.new(team_id: team_ids[0]) if team_ids.size == 1 # We'll use the settings of a team instead of global settings when there is only one team @@ -179,10 +180,10 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, aft words = text.split(/\s+/) Rails.logger.info "[Smooch Bot] Search query (text): #{text}" if Bot::Alegre.get_number_of_words(text) <= self.max_number_of_words_for_keyword_search - results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id, language) + results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language) else alegre_results = Bot::Alegre.get_merged_similar_items(pm, [{ value: self.get_text_similarity_threshold }], Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, text, team_ids) - results = self.parse_search_results_from_alegre(alegre_results, after, feed_id, team_ids) + results = self.parse_search_results_from_alegre(alegre_results, limit, after, feed_id, team_ids) Rails.logger.info "[Smooch Bot] Text similarity search got #{results.count} results while looking for '#{text}' after date #{after.inspect} for teams #{team_ids}" end else @@ -192,7 +193,7 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, aft media_url = self.save_locally_and_return_url(media_url, type, feed_id) threshold = Bot::Alegre.get_threshold_for_query(type, pm)[0][:value] alegre_results = Bot::Alegre.get_items_with_similar_media_v2(media_url: media_url, threshold: [{ value: threshold }], team_ids: team_ids, type: type) - results = self.parse_search_results_from_alegre(alegre_results, after, feed_id, team_ids) + results = self.parse_search_results_from_alegre(alegre_results, limit, after, feed_id, team_ids) Rails.logger.info "[Smooch Bot] Media similarity search got #{results.count} results while looking for '#{query}' after date #{after.inspect} for teams #{team_ids}" end results @@ -245,11 +246,11 @@ def should_restrict_by_language?(team_ids) !!tbi&.alegre_settings&.dig('single_language_fact_checks_enabled') end - def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id = nil, language = nil) + def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id = nil, language = nil) types = CheckSearch::MEDIA_TYPES.clone.push('blank') search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content) - filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3, show: types } - filters.merge!({ fc_language: [language] }) if should_restrict_by_language?(team_ids) + filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: limit, show: types } + filters.merge!({ fc_language: [language] }) if !language.blank? && should_restrict_by_language?(team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed feed_id.blank? ? filters.merge!({ report_status: ['published'] }) : filters.merge!({ feed_id: feed_id }) filters.merge!({ range: { updated_at: { start_time: after.strftime('%Y-%m-%dT%H:%M:%S.%LZ') } } }) unless after.blank? @@ -304,19 +305,19 @@ def ask_for_feedback_when_all_search_results_are_received(app_id, language, work end end - def search_for_explainers(uid, query, team_id, language) + def search_for_explainers(uid, query, team_id, limit, language = nil) results = nil begin text = ::Bot::Smooch.extract_claim(query) if Bot::Alegre.get_number_of_words(text) == 1 results = Explainer.where(team_id: team_id).where('description ILIKE ? OR title ILIKE ?', "%#{text}%", "%#{text}%") - results = results.where(language: language) if should_restrict_by_language?([team_id]) + results = results.where(language: language) if !language.nil? && should_restrict_by_language?([team_id]) results = results.order('updated_at DESC') else - results = Explainer.search_by_similarity(text, language, team_id) + results = Explainer.search_by_similarity(text, language, team_id, limit) end rescue StandardError => e - self.handle_search_error(uid, e, language) + self.handle_search_error(uid, e, language) unless uid.blank? end results.joins(:project_medias) end diff --git a/app/models/explainer.rb b/app/models/explainer.rb index aa25ea9e42..6303c157e0 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -1,12 +1,6 @@ class Explainer < ApplicationRecord include Article - # FIXME: Read from workspace settings - ALEGRE_MODELS_AND_THRESHOLDS = { - # Bot::Alegre::ELASTICSEARCH_MODEL => 0.8 # Sometimes this is easier for local development - Bot::Alegre::PARAPHRASE_MULTILINGUAL_MODEL => 0.7 - } - belongs_to :team has_annotations @@ -71,13 +65,14 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) explainer_id: explainer.id } + models_thresholds = Explainer.get_alegre_models_and_thresholds(explainer.team_id).keys # Index title params = { content_hash: Bot::Alegre.content_hash_for_value(explainer.title), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')), context: base_context.merge({ field: 'title' }), text: explainer.title, - models: ALEGRE_MODELS_AND_THRESHOLDS.keys, + models: models_thresholds, } Bot::Alegre.index_async_with_params(params, "text") @@ -90,7 +85,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'paragraph', count].join(':')), context: base_context.merge({ paragraph: count }), text: paragraph.strip, - models: ALEGRE_MODELS_AND_THRESHOLDS.keys, + models: models_thresholds, } Bot::Alegre.index_async_with_params(params, "text") end @@ -107,23 +102,35 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) end end - def self.search_by_similarity(text, language, team_id) + def self.search_by_similarity(text, language, team_id, limit) + models_thresholds = Explainer.get_alegre_models_and_thresholds(team_id) + context = { + type: 'explainer', + team: Team.find(team_id).slug + } + context[:language] = language unless language.nil? params = { text: text, - models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - per_model_threshold: ALEGRE_MODELS_AND_THRESHOLDS, - context: { - type: 'explainer', - team: Team.find(team_id).slug, - language: language - } + models: models_thresholds.keys, + per_model_threshold: models_thresholds, + context: context + } response = Bot::Alegre.query_sync_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } - explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(3) + explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(limit) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) end + def self.get_alegre_models_and_thresholds(team_id) + models_thresholds = {} + Bot::Alegre.get_similarity_methods_and_models_given_media_type_and_team_id("text", team_id, true).map do |similarity_method, model_name| + _, value = Bot::Alegre.get_threshold_given_model_settings(team_id, "text", similarity_method, true, model_name) + models_thresholds[model_name] = value + end + models_thresholds + end + private def set_team diff --git a/app/models/project_media.rb b/app/models/project_media.rb index ab169298d3..e29adddb82 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -455,6 +455,22 @@ def replace_with_blank_media self.save! end + def get_similar_articles + # Get search query based on Media type + # Quote for Claim + # Transcription for UploadedVideo , UploadedAudio and UploadedImage + # Title and/or description for Link + media = self.media + search_query = case media.type + when 'Claim' + media.quote + when 'UploadedVideo', 'UploadedAudio', 'UploadedImage' + self.transcription + end + search_query ||= self.title + self.team.search_for_similar_articles(search_query, self) + end + protected def add_extra_elasticsearch_data(ms) diff --git a/app/models/team.rb b/app/models/team.rb index 329b044e62..8b9e762740 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -563,6 +563,30 @@ def filter_by_keywords(query, filters, type = 'FactCheck') query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end + def search_for_similar_articles(query, pm = nil) + # query: expected to be text + # pm: to request a most relevant to specific item and also include both FactCheck & Explainer + limit = pm.nil? ? CheckConfig.get('most_relevant_team_limit', 3, :integer) : CheckConfig.get('most_relevant_item_limit', 10, :integer) + result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit).map(&:id) + items = [] + unless result_ids.blank? + # I depend on FactCheck to filter result instead of report_design + items = FactCheck.where(report_status: 'published') + .joins(claim_description: :project_media) + .where('project_medias.id': result_ids) + # Exclude the ones already applied to a target item if exsits + items = items.where.not('fact_checks.id' => pm.fact_check_id) unless pm&.fact_check_id.nil? + end + if items.blank? || !pm.nil? + # Get Explainers if no fact-check returned or get similar_articles for a ProjectMedia + ex_items = Bot::Smooch.search_for_explainers(nil, query, self.id, limit) + # Exclude the ones already applied to a target item + ex_items = ex_items.where.not(id: pm.explainer_ids) unless pm&.explainer_ids.blank? + items = items + ex_items + end + items + end + # private # # Please add private methods to app/models/concerns/team_private.rb diff --git a/app/resources/api/v2/feed_resource.rb b/app/resources/api/v2/feed_resource.rb index ffabf372ad..83f4c0212c 100644 --- a/app/resources/api/v2/feed_resource.rb +++ b/app/resources/api/v2/feed_resource.rb @@ -38,26 +38,27 @@ def self.records(options = {}, skip_save_request = false, skip_cache = false) skip_cache = skip_cache || filters.dig(:skip_cache, 0) == 'true' return ProjectMedia.none if team_ids.blank? || query.blank? - + limit = CheckConfig.get('most_relevant_team_limit', 3, :integer) if feed_id > 0 - return get_results_from_feed_teams(team_ids, feed_id, query, type, after, webhook_url, skip_save_request, skip_cache) + return get_results_from_feed_teams(team_ids, feed_id, query, type, after, webhook_url, skip_save_request, skip_cache, limit) elsif ApiKey.current - return get_results_from_api_key_teams(type, query, after, skip_cache) + return get_results_from_api_key_teams(type, query, after, skip_cache, limit) end end - def self.get_results_from_api_key_teams(type, query, after, skip_cache) + def self.get_results_from_api_key_teams(type, query, after, skip_cache, limit) RequestStore.store[:pause_database_connection] = true # Release database connection during Bot::Alegre.request_api team_ids = ApiKey.current.bot_user.team_ids - Bot::Smooch.search_for_similar_published_fact_checks(type, query, team_ids, after, skip_cache) + limit = CheckConfig.get('most_relevant_team_limit', 3, :integer) + Bot::Smooch.search_for_similar_published_fact_checks(type, query, team_ids, limit, after, skip_cache) end - def self.get_results_from_feed_teams(team_ids, feed_id, query, type, after, webhook_url, skip_save_request, skip_cache) + def self.get_results_from_feed_teams(team_ids, feed_id, query, type, after, webhook_url, skip_save_request, skip_cache, limit) return ProjectMedia.none unless can_read_feed?(feed_id, team_ids) feed = Feed.find(feed_id) RequestStore.store[:pause_database_connection] = true # Release database connection during Bot::Alegre.request_api RequestStore.store[:smooch_bot_settings] = feed.get_smooch_bot_settings.to_h - results = Bot::Smooch.search_for_similar_published_fact_checks(type, query, feed.team_ids, after, feed_id, skip_cache) + results = Bot::Smooch.search_for_similar_published_fact_checks(type, query, feed.team_ids, limit, after, feed_id, skip_cache) Feed.delay({ retry: 0, queue: 'feed' }).save_request(feed_id, type, query, webhook_url, results.to_a.map(&:id)) unless skip_save_request results end diff --git a/config/routes.rb b/config/routes.rb index e4bc0fa4e2..e2c6f3d51e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -79,5 +79,6 @@ match '/test/suggest_similarity' => 'test#suggest_similarity_item', via: :get match '/test/install_bot' => 'test#install_bot', via: :get match '/test/add_team_user' => 'test#add_team_user', via: :get + match '/test/create_imported_standalone_fact_check' => 'test#create_imported_standalone_fact_check', via: :get match '/test/random' => 'test#random', via: :get end diff --git a/lib/relay.idl b/lib/relay.idl index b672574b3a..f65bcb1b2d 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -11666,6 +11666,28 @@ type ProjectMedia implements Node { published: String pusher_channel: String quote: String + relevant_articles( + """ + Returns the elements in the list that come after the specified cursor. + """ + after: String + + """ + Returns the elements in the list that come before the specified cursor. + """ + before: String + + """ + Returns the first _n_ elements from the list. + """ + first: Int + + """ + Returns the last _n_ elements from the list. + """ + last: Int + ): ArticleUnionConnection + relevant_articles_count: Int report_status: String report_type: String requests( diff --git a/public/relay.json b/public/relay.json index 2fc923085e..c2387c8288 100644 --- a/public/relay.json +++ b/public/relay.json @@ -61591,6 +61591,81 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "relevant_articles", + "description": null, + "args": [ + { + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "first", + "description": "Returns the first _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "last", + "description": "Returns the last _n_ elements from the list.", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "ArticleUnionConnection", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "relevant_articles_count", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "report_status", "description": null, diff --git a/test/controllers/feeds_controller_test.rb b/test/controllers/feeds_controller_test.rb index bced9ac47d..534872db7d 100644 --- a/test/controllers/feeds_controller_test.rb +++ b/test/controllers/feeds_controller_test.rb @@ -26,7 +26,7 @@ def setup @f = create_feed published: true @f.teams = [@t1, @t2] FeedTeam.update_all(shared: true) - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false).returns([@pm1, @pm2]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false).returns([@pm1, @pm2]) end def teardown @@ -44,7 +44,7 @@ def teardown b.api_key = a b.save! create_team_user team: @t1, user: b - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id], nil, false).returns([@pm1]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id], 3, nil, false).returns([@pm1]) authenticate_with_token a get :index, params: { filter: { type: 'text', query: 'Foo' } } @@ -55,7 +55,7 @@ def teardown test "should request feed data" do Bot::Smooch.stubs(:search_for_similar_published_fact_checks) - .with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false) + .with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false) .returns([@pm1, @pm2]) authenticate_with_token @a get :index, params: { filter: { type: 'text', query: 'Foo', feed_id: @f.id } } @@ -68,13 +68,13 @@ def teardown Sidekiq::Testing.fake! do authenticate_with_token @a - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false).returns([@pm1, @pm2]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false).returns([@pm1, @pm2]) get :index, params: { filter: { type: 'text', query: 'Foo', feed_id: @f.id } } assert_response :success assert_equal 'Foo', json_response['data'][0]['attributes']['organization'] assert_equal 'Bar', json_response['data'][1]['attributes']['organization'] - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false).returns([@pm2, @pm1]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false).returns([@pm2, @pm1]) get :index, params: { filter: { type: 'text', query: 'Foo', feed_id: @f.id } } assert_response :success assert_equal 'Bar', json_response['data'][0]['attributes']['organization'] @@ -111,7 +111,7 @@ def teardown test "should save request query" do Bot::Alegre.stubs(:request).returns({}) Bot::Smooch.stubs(:search_for_similar_published_fact_checks) - .with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false) + .with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false) .returns([@pm1, @pm2]) Sidekiq::Testing.inline! authenticate_with_token @a @@ -129,7 +129,7 @@ def teardown test "should save relationship between request and results" do Bot::Alegre.stubs(:request).returns({}) Bot::Smooch.stubs(:search_for_similar_published_fact_checks) - .with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false) + .with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false) .returns([@pm1, @pm2]) Sidekiq::Testing.inline! authenticate_with_token @a @@ -143,7 +143,7 @@ def teardown test "should not save request when skip_save_request is true" do Bot::Alegre.stubs(:request).returns({}) Bot::Smooch.stubs(:search_for_similar_published_fact_checks) - .with('text', 'Foo', [@t1.id, @t2.id], nil, @f.id, false) + .with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false) .returns([@pm1, @pm2]) Sidekiq::Testing.inline! authenticate_with_token @a diff --git a/test/controllers/graphql_controller_5_test.rb b/test/controllers/graphql_controller_5_test.rb index e85a1918b3..78e7788270 100644 --- a/test/controllers/graphql_controller_5_test.rb +++ b/test/controllers/graphql_controller_5_test.rb @@ -405,6 +405,25 @@ def setup assert_equal 'Test', JSON.parse(@response.body)['data']['createTag']['tag_text_object']['text'] end + test "should get relevant articles for ProjectMedia item" do + t = create_team + pm = create_project_media quote: 'Foo Bar', team: t + ex = create_explainer language: 'en', team: t, title: 'Foo Bar' + pm.explainers << ex + cd = create_claim_description description: pm.title, project_media: pm + fc = create_fact_check claim_description: cd, title: pm.title + items = FactCheck.where(id: fc.id) + Explainer.where(id: ex.id) + ProjectMedia.any_instance.stubs(:get_similar_articles).returns(items) + query = "query { project_media(ids: \"#{pm.id}\") { relevant_articles_count, relevant_articles { edges { node { ... on FactCheck { dbid }, ... on Explainer { dbid } } } } } }" + post :create, params: { query: query, team: t.slug } + assert_response :success + data = JSON.parse(@response.body)['data']['project_media'] + assert_equal 2, data['relevant_articles_count'] + item_ids = data['relevant_articles']['edges'].collect{ |i| i['node']['dbid'] } + assert_equal items.map(&:id).sort, item_ids.sort + ProjectMedia.any_instance.unstub(:get_similar_articles) + end + protected def assert_error_message(expected) diff --git a/test/controllers/test_controller_test.rb b/test/controllers/test_controller_test.rb index eb9fef1b83..94eaed6209 100644 --- a/test/controllers/test_controller_test.rb +++ b/test/controllers/test_controller_test.rb @@ -493,6 +493,53 @@ class TestControllerTest < ActionController::TestCase Rails.unstub(:env) end + test "should create standalone fact check and associate with the team" do + # Test setup + team = create_team + user = create_user + create_team_user(user: user, team: team) + + assert_difference 'FactCheck.count' do + get :create_imported_standalone_fact_check, params: { + team_id: team.id, + email: user.email, + description: 'Test description', + context: 'Test context', + title: 'Test title', + summary: 'Test summary', + url: 'http://example.com', + language: 'en' + } + end + + assert_response :success + end + + test "should not create standalone fact check and associate with the team" do + Rails.stubs(:env).returns('development') + + # Test setup + team = create_team + user = create_user + create_team_user(user: user, team: team) + + assert_no_difference 'FactCheck.count' do + get :create_imported_standalone_fact_check, params: { + team_id: team.id, + email: user.email, + description: 'Test description', + context: 'Test context', + title: 'Test title', + summary: 'Test summary', + url: 'http://example.com', + language: 'en' + } + end + + assert_response 400 + Rails.unstub(:env) + end + test "should get a random number in HTML" do get :random assert_response :success diff --git a/test/models/bot/smooch_3_test.rb b/test/models/bot/smooch_3_test.rb index 843b40602d..f3cb76cdda 100644 --- a/test/models/bot/smooch_3_test.rb +++ b/test/models/bot/smooch_3_test.rb @@ -691,10 +691,10 @@ def teardown 'Segurança das urna', 'Seguranca das urnas' ].each do |query| - assert_equal [pm1.id], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id]).to_a.map(&:id) + assert_equal [pm1.id], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], 3).to_a.map(&:id) end - assert_equal [], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Segurando', [t.id]).to_a.map(&:id) + assert_equal [], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Segurando', [t.id], 3).to_a.map(&:id) end test "should get turn.io installation" do diff --git a/test/models/bot/smooch_4_test.rb b/test/models/bot/smooch_4_test.rb index 34167fa5f6..611b0b07a1 100644 --- a/test/models/bot/smooch_4_test.rb +++ b/test/models/bot/smooch_4_test.rb @@ -671,10 +671,10 @@ def teardown uid = random_string query = Bot::Smooch.get_search_query(uid, {}) - assert_equal [pm2], Bot::Smooch.get_search_results(uid, query, t.id, 'en') + assert_equal [pm2], Bot::Smooch.get_search_results(uid, query, t.id, 'en', 3) Bot::Smooch.stubs(:bundle_list_of_messages).returns({ 'type' => 'text', 'text' => "Test #{url}" }) query = Bot::Smooch.get_search_query(uid, {}) - assert_equal [pm1], Bot::Smooch.get_search_results(uid, query, t.id, 'en') + assert_equal [pm1], Bot::Smooch.get_search_results(uid, query, t.id, 'en', 3) ProjectMedia.any_instance.unstub(:report_status) CheckSearch.any_instance.unstub(:medias) @@ -694,7 +694,7 @@ def teardown assert_nothing_raised do with_current_user_and_team(nil, t) do - Bot::Smooch.search_for_similar_published_fact_checks('text', 'https://projetocomprova.com.br/publicações/tuite-engana-ao-dizer-que-o-stf-decidiu-que-voto-impresso-e-inconstitucional/ ', [t.id], nil, f.id) + Bot::Smooch.search_for_similar_published_fact_checks('text', 'https://projetocomprova.com.br/publicações/tuite-engana-ao-dizer-que-o-stf-decidiu-que-voto-impresso-e-inconstitucional/ ', [t.id], 3, nil, f.id) end end end diff --git a/test/models/bot/smooch_5_test.rb b/test/models/bot/smooch_5_test.rb index 93eef03afe..49f90522d7 100644 --- a/test/models/bot/smooch_5_test.rb +++ b/test/models/bot/smooch_5_test.rb @@ -68,16 +68,16 @@ def teardown # and for each team participating in the feed with_current_user_and_team(u, t1) do # Keyword search - result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'Test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'Test', [t1.id, t2.id, t3.id, t4.id], 3, nil, f1.id).map(&:id) assert_equal [pm1a.id, pm1f.id, pm2a.id].sort, result.sort # Text similarity search - result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'This is a test', [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + result = Bot::Smooch.search_for_similar_published_fact_checks('text', 'This is a test', [t1.id, t2.id, t3.id, t4.id], 3, nil, f1.id).map(&:id) assert_equal [pm1a.id, pm1d.id, pm2a.id].sort, result.sort # Media similarity search - result = Bot::Smooch.search_for_similar_published_fact_checks('image', random_url, [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + result = Bot::Smooch.search_for_similar_published_fact_checks('image', random_url, [t1.id, t2.id, t3.id, t4.id], 3, nil, f1.id).map(&:id) assert_equal [pm1a.id, pm1d.id, pm2a.id], result.sort # URL search - result = Bot::Smooch.search_for_similar_published_fact_checks('text', "Test with URL: #{url}", [t1.id, t2.id, t3.id, t4.id], nil, f1.id).map(&:id) + result = Bot::Smooch.search_for_similar_published_fact_checks('text', "Test with URL: #{url}", [t1.id, t2.id, t3.id, t4.id], 3, nil, f1.id).map(&:id) assert_equal [pm1g.id, pm2b.id].sort, result.sort end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 8aee9c2afe..10217674c3 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -219,7 +219,7 @@ def teardown uid = random_string query = Bot::Smooch.get_search_query(uid, {}) - assert_equal [pm], Bot::Smooch.get_search_results(uid, query, pm.team_id, 'en') + assert_equal [pm], Bot::Smooch.get_search_results(uid, query, pm.team_id, 'en', 3) Bot::Smooch.unstub(:bundle_list_of_messages) CheckSearch.any_instance.unstub(:medias) @@ -242,7 +242,7 @@ def teardown uid = random_string query = Bot::Smooch.get_search_query(uid, {}) - assert_equal [pm], Bot::Smooch.get_search_results(uid, query, pm.team_id, 'en') + assert_equal [pm], Bot::Smooch.get_search_results(uid, query, pm.team_id, 'en', 3) Bot::Smooch.unstub(:bundle_list_of_messages) ProjectMedia.any_instance.unstub(:report_status) @@ -266,7 +266,7 @@ def teardown Bot::Alegre.stubs(:get_items_with_similar_media_v2).returns({ pm.id => { score: 0.9, model: 'elasticsearch', context: {foo: :bar} } }) CheckS3.stubs(:rewrite_url).returns(random_url) - assert_equal [pm], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en') + assert_equal [pm], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en', 3) Bot::Smooch.unstub(:bundle_list_of_messages) ProjectMedia.any_instance.unstub(:report_status) @@ -316,7 +316,7 @@ def teardown # Create more project media if needed results = { pm1.id => { model: 'elasticsearch', score: 10.8, context: {foo: :bar}}, pm2.id => { model: 'elasticsearch', score: 15.2, context: {foo: :bar}}, pm3.id => { model: 'anything-else', score: 1.98, context: {foo: :bar}}, pm4.id => { model: 'anything-else', score: 1.8, context: {foo: :bar}}} - assert_equal [pm3, pm4, pm2], Bot::Smooch.parse_search_results_from_alegre(results, t.id) + assert_equal [pm3, pm4, pm2], Bot::Smooch.parse_search_results_from_alegre(results, 3, t.id) ProjectMedia.any_instance.unstub(:report_status) end @@ -330,7 +330,7 @@ def teardown # Create more project media if needed results = { pm1.id => { model: 'elasticsearch', score: 10.8, context: {blah: 1} }, pm2.id => { model: 'elasticsearch', score: 15.2, context: {blah: 1} }, pm3.id => { model: 'anything-else', score: 1.98, context: {temporary_media: true} }, pm4.id => { model: 'anything-else', score: 1.8, context: {temporary_media: false}}} - assert_equal [pm4, pm2, pm1], Bot::Smooch.parse_search_results_from_alegre(results, t.id) + assert_equal [pm4, pm2, pm1], Bot::Smooch.parse_search_results_from_alegre(results, 3, t.id) ProjectMedia.any_instance.unstub(:report_status) end @@ -346,7 +346,7 @@ def teardown Bot::Smooch.stubs(:bundle_list_of_messages).returns({ 'type' => 'text', 'text' => url }) CheckSearch.any_instance.stubs(:medias).returns([pm]) - assert_equal [], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en') + assert_equal [], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en', 3) ProjectMedia.any_instance.unstub(:report_status) CheckSearch.any_instance.unstub(:medias) @@ -362,11 +362,11 @@ def teardown query = 'foo bar' assert_queries '>', 1 do - assert_equal [pm], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], nil) + assert_equal [pm], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], 3, nil) end assert_queries '=', 0 do - assert_equal [pm], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], nil) + assert_equal [pm], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], 3, nil) end ProjectMedia.any_instance.unstub(:report_status) @@ -389,7 +389,7 @@ def teardown 'ward', #Fuzzy match (non-emoji) '🤣 ward', #Fuzzy match (non-emoji) ].each do |query| - assert_equal [pm.id], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id]).to_a.map(&:id) + assert_equal [pm.id], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], 3).to_a.map(&:id) end [ @@ -397,7 +397,7 @@ def teardown '🌞', #No match '🤣 🌞' #No match (we only perform AND) ].each do |query| - assert_equal [], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id]).to_a.map(&:id) + assert_equal [], Bot::Smooch.search_for_similar_published_fact_checks('text', query, [t.id], 3).to_a.map(&:id) end end @@ -412,9 +412,9 @@ def teardown [pm1, pm2, pm3].each { |pm| publish_report(pm) } sleep 2 # Wait for ElasticSearch to index content - assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id]).to_a.map(&:id) + assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id], 3).to_a.map(&:id) # Calling wiht skip_cache true - assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id], nil, nil, nil, true).to_a.map(&:id) + assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id], 3, nil, nil, nil, true).to_a.map(&:id) end test "should store media" do @@ -441,7 +441,7 @@ def teardown test "should not return cache search result if report is not published anymore" do pm = create_project_media Bot::Smooch.stubs(:search_for_similar_published_fact_checks).returns([pm]) - assert_equal [], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en') + assert_equal [], Bot::Smooch.get_search_results(random_string, {}, pm.team_id, 'en', 3) end test "should store sent tipline message in background" do @@ -609,12 +609,12 @@ def teardown m = create_uploaded_image pm = create_project_media team: t, media: m, disable_es_callbacks: false query = "Claim content" - results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id]) + results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id], 3) assert_empty results cd = create_claim_description project_media: pm, description: query publish_report(pm) assert_equal query, pm.claim_description_content - results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id]) + results = Bot::Smooch.search_by_keywords_for_similar_published_fact_checks(query.split(), nil, [t.id], 3) assert_equal [pm.id], results.map(&:id) end diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index 7d191e1e7b..c694152dd5 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -153,4 +153,10 @@ def setup end end end + + test "should get alegre models_and_thresholds in hash format" do + ex = create_explainer + models_thresholds = Explainer.get_alegre_models_and_thresholds(ex.team_id) + assert_kind_of Hash, models_thresholds + end end diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index 73553e8b81..d83e68c18b 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -115,4 +115,52 @@ def setup 2.times { create_project_media(team: t, set_original_claim: 'This is a claim.') } end end + + test "should search for item similar articles" do + RequestStore.store[:skip_cached_field_update] = false + setup_elasticsearch + t = create_team + pm1 = create_project_media quote: 'Foo Bar', team: t + pm2 = create_project_media quote: 'Foo Bar Test', team: t + pm3 = create_project_media quote: 'Foo Bar Test Testing', team: t + ex1 = create_explainer language: 'en', team: t, title: 'Foo Bar' + ex2 = create_explainer language: 'en', team: t, title: 'Foo Bar Test' + ex3 = create_explainer language: 'en', team: t, title: 'Foo Bar Test Testing' + pm1.explainers << ex1 + pm2.explainers << ex2 + pm3.explainers << ex3 + ex_ids = [ex1.id, ex2.id, ex3.id] + Bot::Smooch.stubs(:search_for_explainers).returns(Explainer.where(id: ex_ids)) + # Should get explainer + assert_equal [ex2.id, ex3.id], pm1.get_similar_articles.map(&:id).sort + fact_checks = [] + [pm1, pm2, pm3].each do |pm| + cd = create_claim_description description: pm.title, project_media: pm + fc = create_fact_check claim_description: cd, title: pm.title + fact_checks << fc.id + end + [pm1, pm2, pm3].each { |pm| publish_report(pm) } + sleep 1 + fact_checks.delete(pm1.fact_check_id) + # Should get both explainer and FactCheck + assert_equal fact_checks.concat([ex2.id, ex3.id]).sort, pm1.get_similar_articles.map(&:id).sort + Bot::Smooch.unstub(:search_for_explainers) + # Test with media item + json_schema = { + type: 'object', + required: ['job_name'], + properties: { + text: { type: 'string' }, + job_name: { type: 'string' }, + last_response: { type: 'object' } + } + } + create_annotation_type_and_fields('Transcription', {}, json_schema) + img = create_uploaded_image + pm_i = create_project_media team: t, media: img + data = { 'job_status' => 'COMPLETED', 'transcription' => 'Foo Bar'} + a = create_dynamic_annotation annotation_type: 'transcription', annotated: pm_i, set_fields: { text: 'Foo Bar', job_name: '0c481e87f2774b1bd41a0a70d9b70d11', last_response: data }.to_json + sleep 1 + assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].sort, pm_i.get_similar_articles.map(&:id).sort + end end diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 5b2df95442..82a0f237f5 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -1549,4 +1549,38 @@ def setup assert_equal 1, t.filtered_explainers(text: 'Foo Bar Alpha').count assert_equal 0, t.filtered_fact_checks(text: 'Foo Bar Delta').count end + + test "should search for similar articles" do + RequestStore.store[:skip_cached_field_update] = false + setup_elasticsearch + t = create_team + pm1 = create_project_media quote: 'Foo Bar', team: t + pm2 = create_project_media quote: 'Foo Bar Test', team: t + pm3 = create_project_media quote: 'Foo Bar Test Testing', team: t + ex1 = create_explainer language: 'en', team: t, title: 'Foo Bar' + ex2 = create_explainer language: 'en', team: t, title: 'Foo Bar Test' + ex3 = create_explainer language: 'en', team: t, title: 'Foo Bar Test Testing' + pm1.explainers << ex1 + pm2.explainers << ex2 + pm3.explainers << ex3 + ex_ids = [ex1.id, ex2.id, ex3.id] + Bot::Smooch.stubs(:search_for_explainers).returns(Explainer.where(id: ex_ids)) + # Return Explainer if no FactCheck exists + assert_equal ex_ids, t.search_for_similar_articles('Foo Bar').map(&:id).sort + fact_checks = [] + [pm1, pm2, pm3].each do |pm| + cd = create_claim_description description: pm.title, project_media: pm + fc = create_fact_check claim_description: cd, title: pm.title + fact_checks << fc.id + end + [pm1, pm2, pm3].each { |pm| publish_report(pm) } + sleep 2 + # Should return FactCheck even there is an Explainer exists + assert_equal fact_checks.sort, t.search_for_similar_articles('Foo Bar').map(&:id).sort + # Verirfy limit option + stub_configs({ 'most_relevant_team_limit' => 1 }) do + assert_equal [fact_checks.first], t.search_for_similar_articles('Foo Bar').map(&:id).sort + end + Bot::Smooch.unstub(:search_for_explainers) + end end From cb54d3af244cd97d336d7400fac0d9db22a3aaf3 Mon Sep 17 00:00:00 2001 From: Alexandre Amoedo Amorim Date: Wed, 11 Dec 2024 17:09:02 -0300 Subject: [PATCH 03/52] Epic/5571 bot preview (#2152) * Bot Preview: Initial backend implementation (#2132) * Add new field to TeamType to allow bot queries Add a bot_query field to TeamType which will send queries to the backend simulating tipline bot queries. Initially, this is returning the most recent Explainers/FactChecks as TiplineSearchResults, but eventually it will be hooked to the actual bot implementation. * Add reviewer feedback Add reviewer feedback * Add id field to TiplineSearchResult Add id field to TiplineSearchResult * Regenerate GraphQL Api Schema Regenerate GraphQL Api Schema * CV2-5703 bot query preview (#2148) * CV2-5761 list most relevant articles fact check and explainer for project media item CV2-5761 list most relevant articles fact check and explainer for project media item commit 5c952735063c2d1157464c2b7654ff8e1a28fd67 Author: Mohamed El-Sawy Date: Tue Dec 3 07:11:22 2024 +0200 CV2-5761 list most relevant articles fact check and explainer for project media item (#2142) * CV2-5761: include FactCheck & Explainer for item most relevant * CV2-5751: fix articles sort and change the limit * CV2-5761: keep default sort (sort by score) * CV2-5761: enforce limit value as a method args * CV2-5761: cleanup * CV2-5761: add more tests * CV2-5761: add missing test to back coverage 100% * CV2-5761: apply PR comments commit f87308de877f36a3fbe30ee8fb91a3e4321ba115 Merge: fe3eda9a0 8383796bb Author: Sawy Date: Mon Dec 2 08:42:17 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit fe3eda9a0e1773c3ff0c1b00610095bb1f77df42 Merge: 236367edf e5018c4f0 Author: Sawy Date: Fri Nov 29 17:41:20 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit 236367edf1bb46386870b1cd2f060c4576295bfd Author: Mohamed El-Sawy Date: Tue Nov 26 20:50:22 2024 +0200 CV2-5731 Refactoring smooch search (#2137) * CV2-5731: call tipline search_for_articles method and append Explainers if no articles exists * CV2-5731: fix graphql query and add more tests * CV2-5731: apply PR comments and add more tests commit 03d9074a710dda012e44bd8a0d0d218c317b16d9 Merge: 58d0adea9 06a7bf8d9 Author: Sawy Date: Tue Nov 26 17:38:26 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit 58d0adea96c94499646d70d5bca596c12881bd28 Merge: b22f02251 160baeb74 Author: Sawy Date: Mon Nov 25 15:45:25 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit b22f0225154b697ce4be1b8caefdb534a1ae7da9 Author: Mohamed El-Sawy Date: Sat Nov 23 15:50:01 2024 +0200 CV2-5730: return dummy relevant articles (#2136) commit 464eb3ed24fdc34699ec2203585000e3e6c91d3a Merge: cb53dfa85 0fab2bfd3 Author: Sawy Date: Sat Nov 23 07:28:29 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit cb53dfa85af93bafb314c51cee328e8858fda770 Merge: cfc7370a9 5770afe7c Author: Sawy Date: Wed Nov 20 19:28:39 2024 +0200 Merge branch 'develop' into epic/CV2-5373-most-recent-articles-to-most-relevant-articles commit cfc7370a9bdce7565ac2e7226b938d16c599d985 Author: Sawy Date: Tue Nov 19 10:41:09 2024 +0200 CV2-5373: update relay files commit 33bd366db03ea849e41360526af6afd4ea276e91 Author: Sawy Date: Mon Nov 18 12:37:18 2024 +0200 CV2-5373: add a new graphql fields & * Return actual matching explainers and fact checks when using the bot_query method Currently the `bot_query` method in `TeamType` returns the 3 most recent explainers/fact checks as `TiplineSearchResult`s. Now we are changing this to actually return matching explainers and fact checks. * CV2-5373: check language exists for fc_language condition --------- Co-authored-by: Sawy --------- Co-authored-by: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Co-authored-by: Sawy Co-authored-by: Caio <117518+caiosba@users.noreply.github.com> --- app/graph/types/team_type.rb | 11 + app/graph/types/tipline_search_result_type.rb | 11 + app/lib/tipline_search_result.rb | 5 +- app/models/explainer.rb | 2 +- app/models/fact_check.rb | 14 ++ config/initializers/report_designer.rb | 1 + lib/relay.idl | 18 ++ public/relay.json | 220 ++++++++++++++++++ .../controllers/graphql_controller_11_test.rb | 54 +++++ 9 files changed, 333 insertions(+), 3 deletions(-) create mode 100644 app/graph/types/tipline_search_result_type.rb diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 80ad198b0e..3e031b3662 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -400,4 +400,15 @@ def statistics(period:, language: nil, platform: nil) return nil unless User.current&.is_admin TeamStatistics.new(object, period, language, platform) end + + field :bot_query, [TiplineSearchResultType], null: true do + argument :search_text, GraphQL::Types::String, required: true + end + + def bot_query(search_text:) + return nil unless User.current&.is_admin + + results = object.search_for_similar_articles(search_text) + results.map(&:as_tipline_search_result) + end end diff --git a/app/graph/types/tipline_search_result_type.rb b/app/graph/types/tipline_search_result_type.rb new file mode 100644 index 0000000000..15cefacc6f --- /dev/null +++ b/app/graph/types/tipline_search_result_type.rb @@ -0,0 +1,11 @@ +class TiplineSearchResultType < DefaultObject + description "Represents a search result for the tipline" + + field :title, GraphQL::Types::String, null: false + field :body, GraphQL::Types::String, null: true + field :image_url, GraphQL::Types::String, null: true + field :language, GraphQL::Types::String, null: true + field :url, GraphQL::Types::String, null: true + field :type, GraphQL::Types::String, null: false + field :format, GraphQL::Types::String, null: false +end diff --git a/app/lib/tipline_search_result.rb b/app/lib/tipline_search_result.rb index 7482872661..5983ec01cf 100644 --- a/app/lib/tipline_search_result.rb +++ b/app/lib/tipline_search_result.rb @@ -1,7 +1,8 @@ class TiplineSearchResult - attr_accessor :team, :title, :body, :image_url, :language, :url, :type, :format + attr_accessor :id, :team, :title, :body, :image_url, :language, :url, :type, :format - def initialize(team:, title:, body:, image_url:, language:, url:, type:, format:) + def initialize(id:, team:, title:, body:, image_url:, language:, url:, type:, format:) + self.id = id self.team = team self.title = title self.body = body diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 6303c157e0..feee0df35c 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -25,6 +25,7 @@ def send_to_alegre def as_tipline_search_result TiplineSearchResult.new( + id: self.id, team: self.team, title: self.title, body: self.description, @@ -114,7 +115,6 @@ def self.search_by_similarity(text, language, team_id, limit) models: models_thresholds.keys, per_model_threshold: models_thresholds, context: context - } response = Bot::Alegre.query_sync_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } diff --git a/app/models/fact_check.rb b/app/models/fact_check.rb index 7804c51eb9..947f011fa5 100644 --- a/app/models/fact_check.rb +++ b/app/models/fact_check.rb @@ -63,6 +63,20 @@ def clean_fact_check_tags self.tags = clean_tags(self.tags) end + def as_tipline_search_result + TiplineSearchResult.new( + id: self.id, + team: self.team, + title: self.title, + body: self.summary, + language: self.language, + url: self.url, + image_url: nil, + type: :fact_check, + format: :text + ) + end + private def set_language diff --git a/config/initializers/report_designer.rb b/config/initializers/report_designer.rb index ffbf4d7d2b..8cebe2c1d1 100644 --- a/config/initializers/report_designer.rb +++ b/config/initializers/report_designer.rb @@ -100,6 +100,7 @@ def report_design_team_setting_value(field, language) def report_design_to_tipline_search_result if self.annotation_type == 'report_design' TiplineSearchResult.new( + id: self.id, type: :fact_check, team: self.annotated.team, title: self.report_design_field_value('title'), diff --git a/lib/relay.idl b/lib/relay.idl index f65bcb1b2d..cf9c3f1168 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13198,6 +13198,7 @@ type Team implements Node { articles_count(article_type: String, imported: Boolean, language: [String], publisher_ids: [Int], rating: [String], report_status: [String], standalone: Boolean, tags: [String], target_id: Int, text: String, trashed: Boolean = false, updated_at: String, user_ids: [Int]): Int available_newsletter_header_types: JsonStringType avatar: String + bot_query(searchText: String!): [TiplineSearchResult!] check_search_spam: CheckSearch check_search_trash: CheckSearch check_search_unconfirmed: CheckSearch @@ -14110,6 +14111,23 @@ type TiplineResourceEdge { node: TiplineResource } +""" +Represents a search result for the tipline +""" +type TiplineSearchResult { + body: String + created_at: String + format: String! + id: ID! + image_url: String + language: String + permissions: String + title: String! + type: String! + updated_at: String + url: String +} + """ Autogenerated input type of TranscribeAudio """ diff --git a/public/relay.json b/public/relay.json index c2387c8288..19244acc86 100644 --- a/public/relay.json +++ b/public/relay.json @@ -69435,6 +69435,43 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "bot_query", + "description": null, + "args": [ + { + "name": "searchText", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "OBJECT", + "name": "TiplineSearchResult", + "ofType": null + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "check_search_spam", "description": null, @@ -74887,6 +74924,189 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "TiplineSearchResult", + "description": "Represents a search result for the tipline", + "fields": [ + { + "name": "body", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "format", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "image_url", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "language", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "permissions", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "title", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "type", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "url", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "INPUT_OBJECT", "name": "TranscribeAudioInput", diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 5e5c544f65..890868719f 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -323,4 +323,58 @@ def teardown assert_response :success assert_equal 2, JSON.parse(@response.body).dig('data', 'team', 'tipline_requests', 'edges').size end + + test "super admin user should receive the 3 matching FactChecks or Explainers based on search_text" do + t = create_team + # Create a super admin user + super_admin = create_user(is_admin: true) + create_team_user team: t, user: super_admin, role: 'admin' + + # Authenticate with super admin user + authenticate_with_user(super_admin) + + # Create a project under the team + project = create_project(team: t) + + # Create ProjectMedia instances + pm1 = create_project_media quote: 'Foo Bar', team: t + pm2 = create_project_media quote: 'Foo Bar Test', team: t + pm3 = create_project_media quote: 'Foo Bar Test Testing', team: t + + # Create Explainers and attach them to PMs + ex1 = create_explainer language: 'en', team: t, title: 'Foo Bar' + ex2 = create_explainer language: 'en', team: t, title: 'Foo Bar Test' + ex3 = create_explainer language: 'en', team: t, title: 'Foo Bar Test Testing' + ex4 = create_explainer language: 'en', team: t, title: 'Explainer Test Testing' + ex5 = create_explainer language: 'en', team: t, title: 'Explainer 5 Test Testing' + ex6 = create_explainer language: 'en', team: t, title: 'Explainer 6 Test Testing' + pm1.explainers << ex1 + pm2.explainers << ex2 + pm3.explainers << ex3 + pm3.explainers << ex4 + pm3.explainers << ex5 + pm3.explainers << ex6 + + # Perform the GraphQL query with searchText "123" + query = <<~GRAPHQL + query { + team(slug: "#{t.slug}") { + bot_query(searchText: "Foo") { + title + type + } + } + } + GRAPHQL + + post :create, params: { query: query, team: t.slug } + assert_response :success + + data = JSON.parse(@response.body)['data']['team']['bot_query'] + assert_equal 3, data.size, "Expected 3 matching results" + + expected_titles = [ex1.title, ex2.title, ex3.title] + actual_titles = data.map { |result| result['title'] } + assert_equal expected_titles.sort, actual_titles.sort, "Results should match the search query" + end end From b3f5756d94c2f63e9d9b460d9294eed6b1adadc6 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:08:04 -0300 Subject: [PATCH 04/52] Make sure that `null` is not returned as the item when creating an imported fact-check with original media in GraphQL API. (#2153) When handling an existing media to decide if the imported fact-check is going to be added or appended to the existing media, there is a case where the media already exists with a fact-check in the same language that is trying to be imported. That case was returning nil. This PR fixes it by making sure that the new object is returned in that case, and then let the validations take care of returning a persisted or failed object. Fixes: CV2-5804. --- app/models/project_media.rb | 3 ++- test/models/project_media_7_test.rb | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/models/project_media.rb b/app/models/project_media.rb index e29adddb82..e1c18a7a85 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -430,7 +430,7 @@ def apply_rules_and_actions_on_update self.team.apply_rules_and_actions(self, rule_ids) end - def self.handle_fact_check_for_existing_claim(existing_pm,new_pm) + def self.handle_fact_check_for_existing_claim(existing_pm, new_pm) if existing_pm.fact_check.blank? existing_pm.append_fact_check_from(new_pm) return existing_pm @@ -440,6 +440,7 @@ def self.handle_fact_check_for_existing_claim(existing_pm,new_pm) return new_pm end end + new_pm end def append_fact_check_from(new_pm) diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index d83e68c18b..9466ddadcb 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -163,4 +163,13 @@ def setup sleep 1 assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].sort, pm_i.get_similar_articles.map(&:id).sort end + + test "should not return null when handling fact-check for existing media" do + t = create_team + pm1 = create_project_media team: t + c = create_claim_description project_media: pm1 + create_fact_check claim_description: c, language: 'en' + pm2 = ProjectMedia.new team: t, set_fact_check: { 'language' => 'en' } + assert_not_nil ProjectMedia.handle_fact_check_for_existing_claim(pm1, pm2) + end end From d08f344b5e2655e28e0cde9d24203865754b2a88 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 15 Dec 2024 09:20:24 -0300 Subject: [PATCH 05/52] Adding missing tests. (#2154) Adding missing tests in order to complete code coverage. Reference: CV2-5830. --- test/lib/check_search_test.rb | 5 +++++ test/models/fact_check_test.rb | 5 +++++ test/models/project_test.rb | 7 +++++++ test/workers/project_media_trash_worker_test.rb | 2 +- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/test/lib/check_search_test.rb b/test/lib/check_search_test.rb index 9288ba3db3..327be12690 100644 --- a/test/lib/check_search_test.rb +++ b/test/lib/check_search_test.rb @@ -17,4 +17,9 @@ def teardown search = CheckSearch.new({ keyword: query }.to_json, nil, @team.id) assert_equal 'Something is going to happen on 04 11 reportedly', search.instance_variable_get('@options')['keyword'] end + + test "should search for array field containing nil values" do + search = CheckSearch.new({ users: [1, nil] }.to_json, nil, @team.id) + assert_not_nil search.send(:doc_conditions) + end end diff --git a/test/models/fact_check_test.rb b/test/models/fact_check_test.rb index 88b8a70d0c..5e5ad74131 100644 --- a/test/models/fact_check_test.rb +++ b/test/models/fact_check_test.rb @@ -728,4 +728,9 @@ def setup assert_nil pm.reload.fact_check_id end end + + test "should be formatted as tipline search result" do + fc = create_fact_check + assert_kind_of TiplineSearchResult, fc.as_tipline_search_result + end end diff --git a/test/models/project_test.rb b/test/models/project_test.rb index e278d7ef50..15a4046ce3 100644 --- a/test/models/project_test.rb +++ b/test/models/project_test.rb @@ -645,4 +645,11 @@ def setup p = create_project team: t assert p.inactive end + + test "should get and set current project" do + p = create_project + assert_nil Project.current + Project.current = p + assert_equal p, Project.current + end end diff --git a/test/workers/project_media_trash_worker_test.rb b/test/workers/project_media_trash_worker_test.rb index 4b86da65b9..0833eeefae 100644 --- a/test/workers/project_media_trash_worker_test.rb +++ b/test/workers/project_media_trash_worker_test.rb @@ -2,9 +2,9 @@ class ProjectMediaTrashWorkerTest < ActiveSupport::TestCase def setup - super require 'sidekiq/testing' Sidekiq::Testing.inline! + Team.current = User.current = nil end test "should destroy trashed items" do From eaa870f34bca0f90ba188da0179d27272d9290eb Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Mon, 16 Dec 2024 05:35:52 -0800 Subject: [PATCH 06/52] CV2-5789 dont send null values out to presto from check-api (#2155) --- app/models/bot/alegre.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index e2b9b393b0..43abff02eb 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -153,10 +153,10 @@ def self.run(body) Rails.logger.info("[Alegre Bot] [ProjectMedia ##{pm.id}] This item was just created, processing...") self.get_language(pm) if ['audio', 'image', 'video'].include?(self.get_pm_type(pm)) - self.relate_project_media_async(pm) + self.relate_project_media_async(pm) if self.media_file_url(pm) else - self.relate_project_media_async(pm, 'original_title') - self.relate_project_media_async(pm, 'original_description') + self.relate_project_media_async(pm, 'original_title') if pm.original_title + self.relate_project_media_async(pm, 'original_description') if pm.original_description end self.get_extracted_text(pm) self.get_flags(pm) From 0d6bdfe3c89768b39daea1773321c0ea8ef41560 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:47:57 -0300 Subject: [PATCH 07/52] Handle single quote when searching by keyword. (#2158) Similarly to other characters, add single quotes to the list of characters to handle when searching by keyword. Fixes: CV2-5808. --- lib/check_search.rb | 2 +- test/lib/check_search_test.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/check_search.rb b/lib/check_search.rb index 17e64282db..509570c82e 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -55,7 +55,7 @@ def adjust_keyword_filter unless @options['keyword'].blank? # This regex removes all characters except letters, numbers, hashtag, search operators, emojis and whitespace # in any language - stripping out special characters can improve match results - @options['keyword'].gsub!(/[^[:word:]\s#~+\-|()"\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F700}-\u{1F77F}\u{1F780}-\u{1F7FF}\u{1F800}-\u{1F8FF}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{1F1E6}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/, ' ') + @options['keyword'].gsub!(/[^[:word:]\s#'~+\-|()"\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F700}-\u{1F77F}\u{1F780}-\u{1F7FF}\u{1F800}-\u{1F8FF}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{1F1E6}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/, ' ') # Set fuzzy matching for keyword search, right now with automatic Levenshtein Edit Distance # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html diff --git a/test/lib/check_search_test.rb b/test/lib/check_search_test.rb index 327be12690..8a59c5bd24 100644 --- a/test/lib/check_search_test.rb +++ b/test/lib/check_search_test.rb @@ -16,6 +16,10 @@ def teardown query = 'Something is going to happen on 04/11, reportedly' search = CheckSearch.new({ keyword: query }.to_json, nil, @team.id) assert_equal 'Something is going to happen on 04 11 reportedly', search.instance_variable_get('@options')['keyword'] + + query = "Something is going to happen on Foo's house" + search = CheckSearch.new({ keyword: query }.to_json, nil, @team.id) + assert_equal "Something is going to happen on Foo's house", search.instance_variable_get('@options')['keyword'] end test "should search for array field containing nil values" do From cd5d9e3a842d30b7f0657c0807287889a933c1c8 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 17 Dec 2024 15:12:05 +0200 Subject: [PATCH 08/52] CV2-5779: use same sort for explainer (#2156) --- app/models/explainer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/explainer.rb b/app/models/explainer.rb index feee0df35c..b26f247eb6 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -117,7 +117,7 @@ def self.search_by_similarity(text, language, team_id, limit) context: context } response = Bot::Alegre.query_sync_with_params(params, "text") - results = response['result'].to_a.sort_by{ |result| result['_score'] } + results = response['result'].to_a.sort_by{ |result| [result['model'] != Bot::Alegre::ELASTICSEARCH_MODEL ? 1 : 0, result['_score']] }.reverse explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(limit) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) end From 621e94773ee17b458e18e671705048a145327b35 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:02:35 -0300 Subject: [PATCH 09/52] Fixes and changes for the data dashboard backend (#2159) Some fixes and changes for the data dashboard backend: - CV2-5812: Expand the "matched media" data point to include all tipline request types. - CV2-5813: Use the default language when an explainer is created without language (includes migration rake task). - CV2-5842: Use updated_at instead of created_at for article tags analytics. - CV2-5835: For messages analytics, include only the states sent and received. - CV2-5836: Use the top_media_tags method from CheckDataPoints lib in the TeamStatistics lib. --- app/models/explainer.rb | 11 ++++++++--- lib/check_data_points.rb | 2 +- ...1216215050_set_language_for_explainers.rake | 18 ++++++++++++++++++ lib/team_statistics.rb | 8 ++++---- test/lib/team_statistics_test.rb | 5 +++-- test/models/explainer_test.rb | 9 +++++++++ 6 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 lib/tasks/migrate/20241216215050_set_language_for_explainers.rake diff --git a/app/models/explainer.rb b/app/models/explainer.rb index b26f247eb6..c6f9da7e71 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -7,10 +7,10 @@ class Explainer < ApplicationRecord has_many :explainer_items, dependent: :destroy has_many :project_medias, through: :explainer_items - before_validation :set_team + before_validation :set_team, :set_language validates_format_of :url, with: URI.regexp, allow_blank: true, allow_nil: true validates_presence_of :team, :title, :description - validate :language_in_allowed_values, unless: proc { |e| e.language.blank? } + validate :language_in_allowed_values after_save :update_paragraphs_in_alegre after_update :detach_explainer_if_trashed @@ -137,8 +137,13 @@ def set_team self.team ||= Team.current end + def set_language + default_language = self.team&.get_language || 'und' + self.language ||= default_language + end + def language_in_allowed_values - allowed_languages = self.team.get_languages || ['en'] + allowed_languages = self.team&.get_languages || ['en'] allowed_languages << 'und' errors.add(:language, I18n.t(:"errors.messages.invalid_article_language_value")) unless allowed_languages.include?(self.language) end diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 83c70d7df5..5f4025bff0 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -6,7 +6,7 @@ class << self # Number of tipline messages def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) - query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date) + query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date, state: ['sent', 'received']) query_based_on_granularity(query, platform, language, granularity) end diff --git a/lib/tasks/migrate/20241216215050_set_language_for_explainers.rake b/lib/tasks/migrate/20241216215050_set_language_for_explainers.rake new file mode 100644 index 0000000000..f137113c18 --- /dev/null +++ b/lib/tasks/migrate/20241216215050_set_language_for_explainers.rake @@ -0,0 +1,18 @@ +namespace :check do + namespace :migrate do + task set_language_for_explainers: :environment do + started = Time.now.to_i + query = Explainer.where(language: nil) + n = query.count + i = 0 + query.find_each do |explainer| + i += 1 + language = explainer.team&.get_language || 'und' + explainer.update_column(:language, language) + puts "[#{Time.now}] [#{i}/#{n}] Setting language for explainer ##{explainer.id} as #{language}" + end + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes. Number of explainers without language: #{query.count}" + end + end +end diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 28b32010b5..4c58dedfe1 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -83,10 +83,10 @@ def top_articles_tags FROM ( SELECT unnest(fcs.tags) AS tag FROM fact_checks fcs INNER JOIN claim_descriptions cds ON fcs.claim_description_id = cds.id - WHERE cds.team_id = :team_id AND fcs.created_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language) + WHERE cds.team_id = :team_id AND fcs.updated_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language) UNION ALL SELECT unnest(explainers.tags) AS tag FROM explainers - WHERE explainers.team_id = :team_id AND explainers.created_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language) + WHERE explainers.team_id = :team_id AND explainers.updated_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language) ) AS all_tags GROUP BY tag ORDER BY tag_count DESC @@ -198,7 +198,7 @@ def top_requested_media_clusters # FIXME: The "demand" is across languages and platforms def top_media_tags tags = {} - clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'language', @platform) + clusters = CheckDataPoints.top_media_tags(@team.id, @start_date, @end_date, 20, 'last_seen', @language || @all_languages, 'language', @platform) clusters.each do |pm_id, demand| item = ProjectMedia.find(pm_id) item.tags_as_sentence.split(',').map(&:strip).each do |tag| @@ -218,7 +218,7 @@ def number_of_articles_sent end def number_of_matched_results_by_article_type - query = TiplineRequest.where(team_id: @team.id, smooch_request_type: ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'], created_at: @start_date..@end_date) + query = TiplineRequest.where(team_id: @team.id, created_at: @start_date..@end_date) query = query.where(platform: @platform) unless @platform.blank? query = query.where(language: @language) unless @language.blank? { 'FactCheck' => query.joins(project_media: { claim_description: :fact_check }).count, 'Explainer' => query.joins(project_media: :explainers).count } diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index ca164c628e..6e3d7744f3 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -2,6 +2,7 @@ class TeamStatisticsTest < ActiveSupport::TestCase def setup + Explainer.delete_all @team = create_team @team.set_languages = ['en', 'pt'] @team.save! @@ -52,7 +53,7 @@ def teardown create_fact_check(tags: ['foo', 'bar'], language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) exp = create_explainer team: @team, language: 'en', tags: ['foo'] - create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer team: @team, tags: ['foo', 'bar'], language: 'pt' create_explainer language: 'en', team: team, tags: ['foo', 'bar'] end @@ -60,7 +61,7 @@ def teardown create_fact_check(tags: ['bar'], report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) create_explainer team: @team, language: 'en', tags: ['foo'] - create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer team: @team, tags: ['foo', 'bar'], language: 'pt' create_explainer language: 'en', team: team, tags: ['foo', 'bar'] exp.updated_at = Time.now exp.save! diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index c694152dd5..7c40dce9d4 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -5,6 +5,10 @@ def setup Explainer.delete_all end + def teardown + User.current = Team.current = nil + end + test "should create explainer" do assert_difference 'Explainer.count' do create_explainer @@ -159,4 +163,9 @@ def setup models_thresholds = Explainer.get_alegre_models_and_thresholds(ex.team_id) assert_kind_of Hash, models_thresholds end + + test "should set default language when language is not set" do + ex = create_explainer language: nil + assert_equal 'en', ex.reload.language + end end From 3910fb8211d4460f5d060e734b08efb61eb2de3c Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:40:52 -0300 Subject: [PATCH 10/52] Revert "Feature flag for data dashboard (#2123)" (#2162) This reverts commit ca41cdcf52fd41a91c2d1b827dc9f4c20781fadd. --- app/graph/types/team_type.rb | 1 - .../controllers/graphql_controller_11_test.rb | 26 ++----------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 3e031b3662..01e56ba6d0 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -397,7 +397,6 @@ def api_keys end def statistics(period:, language: nil, platform: nil) - return nil unless User.current&.is_admin TeamStatistics.new(object, period, language, platform) end diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 890868719f..a41fa4a002 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -201,8 +201,8 @@ def teardown end end - test "should get team statistics if super admin" do - user = create_user is_admin: true + test "should get team statistics" do + user = create_user team = create_team create_team_user user: user, team: team, role: 'admin' @@ -244,28 +244,6 @@ def teardown post :create, params: { query: query } assert_response :success - assert_not_nil JSON.parse(@response.body).dig('data', 'team', 'statistics') - end - - test "should not get team statistics if not super admin" do - user = create_user is_admin: false - team = create_team - create_team_user user: user, team: team, role: 'admin' - - authenticate_with_user(user) - query = <<~GRAPHQL - query { - team(slug: "#{team.slug}") { - statistics(period: "past_week", platform: "whatsapp", language: "en") { - number_of_articles_created_by_date - } - } - } - GRAPHQL - - post :create, params: { query: query } - assert_response :success - assert_nil JSON.parse(@response.body).dig('data', 'team', 'statistics') end test "should not get requests if interval is more than one month" do From 534fc113465e6684ac66b9880324f07ab2e2fa0b Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 18 Dec 2024 17:10:39 +0200 Subject: [PATCH 11/52] CV2-5846 updates to back-end functions to support recent to relevant (#2161) * CV2-5846: cache relevant results * CV2-5846: get FactCheck & Explainers in parallel requests * CV2-5846: include unpublished articles * CV2-5846: fix tests * Revert "CV2-5846: fix tests" This reverts commit 823541a0020cfd15ae49c4944db96b769d8079ce. * CV2-5846: apply PR comments * CV2-5846: fix tests --- app/models/concerns/smooch_search.rb | 12 ++++--- app/models/project_media.rb | 32 +++++++++++++------ app/models/team.rb | 39 ++++++++++++++--------- app/resources/api/v2/feed_resource.rb | 2 +- test/controllers/feeds_controller_test.rb | 4 +-- test/models/project_media_7_test.rb | 13 +++++--- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index b306a4dc41..f24bd1119f 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -161,7 +161,7 @@ def search_for_similar_published_fact_checks(type, query, team_ids, limit, after # "type" is text, video, audio or image # "query" is either a piece of text of a media URL - def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil) + def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil, published_only = true) results = [] pm = nil pm = ProjectMedia.new(team_id: team_ids[0]) if team_ids.size == 1 # We'll use the settings of a team instead of global settings when there is only one team @@ -180,7 +180,7 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, lim words = text.split(/\s+/) Rails.logger.info "[Smooch Bot] Search query (text): #{text}" if Bot::Alegre.get_number_of_words(text) <= self.max_number_of_words_for_keyword_search - results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language) + results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language, published_only) else alegre_results = Bot::Alegre.get_merged_similar_items(pm, [{ value: self.get_text_similarity_threshold }], Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, text, team_ids) results = self.parse_search_results_from_alegre(alegre_results, limit, after, feed_id, team_ids) @@ -246,13 +246,17 @@ def should_restrict_by_language?(team_ids) !!tbi&.alegre_settings&.dig('single_language_fact_checks_enabled') end - def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id = nil, language = nil) + def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id = nil, language = nil, published_only = true) types = CheckSearch::MEDIA_TYPES.clone.push('blank') search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content) filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: limit, show: types } filters.merge!({ fc_language: [language] }) if !language.blank? && should_restrict_by_language?(team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed - feed_id.blank? ? filters.merge!({ report_status: ['published'] }) : filters.merge!({ feed_id: feed_id }) + if feed_id.blank? + filters.merge!({ report_status: ['published'] }) if published_only + else + filters.merge!({ feed_id: feed_id }) + end filters.merge!({ range: { updated_at: { start_time: after.strftime('%Y-%m-%dT%H:%M:%S.%LZ') } } }) unless after.blank? results = CheckSearch.new(filters.to_json, nil, team_ids).medias Rails.logger.info "[Smooch Bot] Keyword search got #{results.count} results (only main items) while looking for '#{words}' after date #{after.inspect} for teams #{team_ids}" diff --git a/app/models/project_media.rb b/app/models/project_media.rb index e1c18a7a85..9e8b2e3b4e 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -461,15 +461,29 @@ def get_similar_articles # Quote for Claim # Transcription for UploadedVideo , UploadedAudio and UploadedImage # Title and/or description for Link - media = self.media - search_query = case media.type - when 'Claim' - media.quote - when 'UploadedVideo', 'UploadedAudio', 'UploadedImage' - self.transcription - end - search_query ||= self.title - self.team.search_for_similar_articles(search_query, self) + results = [] + items = Rails.cache.fetch("relevant-items-#{self.id}", expires_in: 2.hours) do + media = self.media + search_query = case media.type + when 'Claim' + media.quote + when 'UploadedVideo', 'UploadedAudio', 'UploadedImage' + self.transcription + end + search_query ||= self.title + results = self.team.search_for_similar_articles(search_query, self) + fact_check_ids = results.select{|article| article.is_a?(FactCheck)}.map(&:id) + explainer_ids = results.select{|article| article.is_a?(Explainer)}.map(&:id) + { fact_check: fact_check_ids, explainer: explainer_ids }.to_json + end + if results.blank? + # This indicates a cache hit, so we should retrieve the items according to the cached values while maintaining the same sort order. + items = JSON.parse(items) + items.each do |klass, ids| + results += klass.camelize.constantize.where(id: ids).sort_by { |result| ids.index(result.id) } + end + end + results end protected diff --git a/app/models/team.rb b/app/models/team.rb index 8b9e762740..18d0f45f52 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -567,23 +567,32 @@ def search_for_similar_articles(query, pm = nil) # query: expected to be text # pm: to request a most relevant to specific item and also include both FactCheck & Explainer limit = pm.nil? ? CheckConfig.get('most_relevant_team_limit', 3, :integer) : CheckConfig.get('most_relevant_item_limit', 10, :integer) - result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit).map(&:id) - items = [] - unless result_ids.blank? - # I depend on FactCheck to filter result instead of report_design - items = FactCheck.where(report_status: 'published') - .joins(claim_description: :project_media) - .where('project_medias.id': result_ids) - # Exclude the ones already applied to a target item if exsits - items = items.where.not('fact_checks.id' => pm.fact_check_id) unless pm&.fact_check_id.nil? - end - if items.blank? || !pm.nil? - # Get Explainers if no fact-check returned or get similar_articles for a ProjectMedia - ex_items = Bot::Smooch.search_for_explainers(nil, query, self.id, limit) + threads = [] + fc_items = [] + ex_items = [] + threads << Thread.new { + result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit, nil, nil, nil, false).map(&:id) + unless result_ids.blank? + fc_items = FactCheck.joins(claim_description: :project_media).where('project_medias.id': result_ids) + if pm.nil? + # This means we obtain relevant items for the Bot preview, so we should limit FactChecks to published articles; + # otherwise, relevant articles for ProjectMedia should include all FactChecks. + fc_items = fc_items.where(report_status: 'published') + elsif !pm.fact_check_id.nil? + # Exclude the ones already applied to a target item if exists. + fc_items = fc_items.where.not('fact_checks.id' => pm.fact_check_id) unless pm&.fact_check_id.nil? + end + end + } + threads << Thread.new { + ex_items = Bot::Smooch.search_for_explainers(nil, query, self.id, limit).distinct # Exclude the ones already applied to a target item ex_items = ex_items.where.not(id: pm.explainer_ids) unless pm&.explainer_ids.blank? - items = items + ex_items - end + } + threads.map(&:join) + items = fc_items + # Get Explainers if no fact-check returned or get similar_articles for a ProjectMedia + items += ex_items if items.blank? || !pm.nil? items end diff --git a/app/resources/api/v2/feed_resource.rb b/app/resources/api/v2/feed_resource.rb index 83f4c0212c..dd3e8519db 100644 --- a/app/resources/api/v2/feed_resource.rb +++ b/app/resources/api/v2/feed_resource.rb @@ -50,7 +50,7 @@ def self.get_results_from_api_key_teams(type, query, after, skip_cache, limit) RequestStore.store[:pause_database_connection] = true # Release database connection during Bot::Alegre.request_api team_ids = ApiKey.current.bot_user.team_ids limit = CheckConfig.get('most_relevant_team_limit', 3, :integer) - Bot::Smooch.search_for_similar_published_fact_checks(type, query, team_ids, limit, after, skip_cache) + Bot::Smooch.search_for_similar_published_fact_checks(type, query, team_ids, limit, after, nil, nil, skip_cache) end def self.get_results_from_feed_teams(team_ids, feed_id, query, type, after, webhook_url, skip_save_request, skip_cache, limit) diff --git a/test/controllers/feeds_controller_test.rb b/test/controllers/feeds_controller_test.rb index 534872db7d..337dce30c4 100644 --- a/test/controllers/feeds_controller_test.rb +++ b/test/controllers/feeds_controller_test.rb @@ -26,7 +26,7 @@ def setup @f = create_feed published: true @f.teams = [@t1, @t2] FeedTeam.update_all(shared: true) - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, false).returns([@pm1, @pm2]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id, @t2.id], 3, nil, @f.id, nil, false).returns([@pm1, @pm2]) end def teardown @@ -44,7 +44,7 @@ def teardown b.api_key = a b.save! create_team_user team: @t1, user: b - Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id], 3, nil, false).returns([@pm1]) + Bot::Smooch.stubs(:search_for_similar_published_fact_checks).with('text', 'Foo', [@t1.id], 3, nil, nil, nil, false).returns([@pm1]) authenticate_with_token a get :index, params: { filter: { type: 'text', query: 'Foo' } } diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index 9466ddadcb..18dcbedea0 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -139,12 +139,16 @@ def setup fc = create_fact_check claim_description: cd, title: pm.title fact_checks << fc.id end - [pm1, pm2, pm3].each { |pm| publish_report(pm) } + [pm1, pm2].each { |pm| publish_report(pm) } sleep 1 fact_checks.delete(pm1.fact_check_id) # Should get both explainer and FactCheck - assert_equal fact_checks.concat([ex2.id, ex3.id]).sort, pm1.get_similar_articles.map(&:id).sort - Bot::Smooch.unstub(:search_for_explainers) + Rails.cache.delete("relevant-items-#{pm1.id}") + expected_result = fact_checks.concat([ex2.id, ex3.id]).sort + assert_equal expected_result, pm1.get_similar_articles.map(&:id).sort + assert_queries '=', 0 do + assert_equal expected_result, pm1.get_similar_articles.map(&:id).sort + end # Test with media item json_schema = { type: 'object', @@ -161,7 +165,8 @@ def setup data = { 'job_status' => 'COMPLETED', 'transcription' => 'Foo Bar'} a = create_dynamic_annotation annotation_type: 'transcription', annotated: pm_i, set_fields: { text: 'Foo Bar', job_name: '0c481e87f2774b1bd41a0a70d9b70d11', last_response: data }.to_json sleep 1 - assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].sort, pm_i.get_similar_articles.map(&:id).sort + assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].concat([ex1.id, ex2.id, ex3.id]).sort, pm_i.get_similar_articles.map(&:id).sort + Bot::Smooch.unstub(:search_for_explainers) end test "should not return null when handling fact-check for existing media" do From e5166b41e4f06f17185da4e8099eb28e83ba74e2 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:08:51 -0300 Subject: [PATCH 12/52] Smooch Bot should not create a relationship between Media and TiplineResource (#2160) We saw an error in Sentry where the creation of a Relationship between a ProjectMedia and a TiplineResource was attempted. The creation fails, but it should not be attempted in the first place. Error - StandardError: Unable to create new relationship as requested: Related items must exist in the same workspace - exception_class: ActiveRecord::RecordInvalid - exception_message: Related items must exist in the same workspace For this to happen - The media can't be of type text, because we do check if it's a ProjectMedia in that case, - it needs to have a caption, - there needs to be an existing ProjectMedia with the same id as the TiplineResource, - if there isn't a ProjectMedia with the same id as the TiplineResource we get a different error: #. Note on the error - The error is rescued, Sentry is notified and it's logged, - so we need to check if Sentry was notified or/and if the error was logged to confirm it happened. Solution We return early, and not even attempt to relate items if `associated` it's not a ProjectMedia, since Relationship's are only for ProjectMedia. Note There was some confusion on the difference between associated and associated_obj: - associated: is the the "first media" and will be the source of the Relationship - associated_obj: is used for TiplineRequest (smooch_resource_id field) References: CV2-5676 PR: 2160 --- app/models/concerns/smooch_messages.rb | 9 ++++++--- test/models/bot/smooch_7_test.rb | 28 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index 88913b58ce..f23daea9c8 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -395,6 +395,8 @@ def default_archived_flag end def save_message(message_json, app_id, author = nil, request_type = 'default_requests', associated_id = nil, associated_class = nil) + # associated: is the the "first media" and will be the source of the Relationship + # associated_obj: is used for TiplineRequest (smooch_resource_id field) message = JSON.parse(message_json) return if TiplineRequest.where(smooch_message_id: message['_id']).exists? associated_obj = nil @@ -413,13 +415,13 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req associated = self.create_project_media_from_message(message) end unless associated.nil? - self.smoooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj) + self.smooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj) self.smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj) end end end - def smoooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj) + def smooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj) # Remember that we received this message. hash = self.message_hash(message) Rails.cache.write("smooch:message:#{hash}", associated.id) @@ -430,6 +432,7 @@ def smoooch_post_save_message_actions(message, associated, app_id, author, reque end def smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj) + return unless associated.is_a?(ProjectMedia) if !message['caption'].blank? # Check if message contains caption then create an item and force relationship self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.confirmed_type) @@ -455,7 +458,7 @@ def relate_item_and_text(message, associated, app_id, author, request_type, asso message.delete('mediaUrl') target = self.create_project_media_from_message(message) unless target.nil? - smoooch_post_save_message_actions(message, target, app_id, author, request_type, associated_obj) + smooch_post_save_message_actions(message, target, app_id, author, request_type, associated_obj) Relationship.create_unless_exists(associated.id, target.id, relationship_type) end end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 10217674c3..a2f17ed9f5 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -632,4 +632,32 @@ def teardown end TiplineRequest.any_instance.unstub(:save!) end + + test "should not try to create relationship between media and tipline resource" do + t2 = create_team + pm = create_project_media team: t2 + + t = create_team + tipline_resource = create_tipline_resource team: t + tipline_resource.update_column(:id, pm.id) + + # It should not try to match at all, so we should never get to this notification + CheckSentry.expects(:notify).never + Rails.logger.expects(:notify).never + + Sidekiq::Testing.inline! do + message = { + type: 'video', + source: { type: "whatsapp" }, + text: 'Something', + caption: 'For this to happen, it needs a caption', + mediaUrl: @video_url, + '_id': random_string, + language: 'en', + } + assert_no_difference 'Relationship.count' do + Bot::Smooch.save_message(message.to_json, @app_id, @bot, 'resource_requests', tipline_resource.id, 'TiplineResource') + end + end + end end From e691ce25fb700d79bbeef8ebd75bf5e56d50f721 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Fri, 20 Dec 2024 08:13:56 -0300 Subject: [PATCH 13/52] =?UTF-8?q?5857=20=E2=80=93=20apply=5Freplace=5Fby?= =?UTF-8?q?=20should=20save=20history=20version=20and=20not=20error=20if?= =?UTF-8?q?=20the=20old=5Fpm=20does=20not=20exist=20(#2163)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we run apply_replace_by, we look for the original project media id, but sometimes that doesn't exist anymore. This only causes an error when we try to save that information to the history versioning. Since we already have access to that original project media id, we can just grab that value and save that information. Note: We are not sure why that happens, but there are places in that area of the code that expect the possibility of old_pm being nil. So for now, we just update to get the information we need and avoid this specific error. References: 5857 PR: 2163 --- app/models/project_media.rb | 2 +- test/models/project_media_5_test.rb | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 9e8b2e3b4e..8cfc39e82d 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -334,7 +334,7 @@ def self.apply_replace_by(old_pm_id, new_pm_id, options_json) item_id: new_pm.id.to_s, event: 'replace', whodunnit: options['author_id'].to_s, - object_changes: { pm_id: [old_pm.id, new_pm.id] }.to_json, + object_changes: { pm_id: [old_pm_id, new_pm.id] }.to_json, associated_id: new_pm.id, associated_type: 'ProjectMedia', team_id: new_pm.team_id, diff --git a/test/models/project_media_5_test.rb b/test/models/project_media_5_test.rb index ef8ba5a0d2..8ef306782b 100644 --- a/test/models/project_media_5_test.rb +++ b/test/models/project_media_5_test.rb @@ -885,6 +885,16 @@ def setup end end + test "should save history version even if the original project media does not exist anymore" do + t = create_team + old_pm_id = 123456 # something that does not exist anymore + new = create_project_media team: t + ProjectMedia.apply_replace_by(old_pm_id, new.id, "{\"author_id\":1234,\"assignments_ids\":[],\"skip_send_report\":true}") + + history = new.versions.first.object_changes + assert_equal history, { pm_id: [123456,new.id]}.to_json + end + test "should replace a blank project media by another project media" do setup_elasticsearch t = create_team From eb1d6d42aea3679b9e362d32e25b4df08f4bc84a Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Fri, 20 Dec 2024 21:52:17 +0100 Subject: [PATCH 14/52] Fix N+1 query issue in bot_user method by using find_by (#2164) * Fix N+1 query issue in bot_user method by using find_by The bot_user method in the TeamBotInstallation model was previously using the `where(...).last` approach to fetch the BotUser, which could result in unnecessary database queries when called multiple times. This fix optimizes the method by replacing `where(...).last` with `find_by(id: ...)`, ensuring only a single query is executed per call. Additionally, a test has been added to verify that no additional queries are triggered when accessing the `bot_user` method multiple times on the same instance, ensuring that the optimized code works as expected and prevents potential N+1 query issues. Changes: - Replaced `where(...).last` with `find_by(id: ...)` in the `bot_user` method. - Added a test to verify that no extra queries are triggered when calling `bot_user`. * Fix failing test --- app/models/team_bot_installation.rb | 2 +- test/models/team_bot_installation_test.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/app/models/team_bot_installation.rb b/app/models/team_bot_installation.rb index 8227d11c2a..e359f43cce 100644 --- a/app/models/team_bot_installation.rb +++ b/app/models/team_bot_installation.rb @@ -65,7 +65,7 @@ def alegre_settings end def bot_user - BotUser.where(id: self.user_id).last + @bot_user ||= BotUser.find_by(id: self.user_id) end def apply_default_settings diff --git a/test/models/team_bot_installation_test.rb b/test/models/team_bot_installation_test.rb index d55e5d0750..04bc6ba487 100644 --- a/test/models/team_bot_installation_test.rb +++ b/test/models/team_bot_installation_test.rb @@ -247,4 +247,17 @@ def setup assert_equal 'def456', tbi.reload.get_turnio_token end end + + test "should not trigger additional queries when accessing bot_user" do + team_bot = create_team_bot set_approved: true + team_bot_installation = create_team_bot_installation(user_id: team_bot.id) + + initial_query_count = ActiveRecord::Base.connection.query_cache.size + + assert_queries(0) do + team_bot_installation.bot_user + end + + assert_equal initial_query_count, ActiveRecord::Base.connection.query_cache.size + end end From 9a3263c15d5b0b9696b8bf7532cbb50e8fbb5054 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 20 Dec 2024 22:06:01 -0300 Subject: [PATCH 15/52] Make sure that the current relationship user overwrites the previous user when moving targets to a new source (#2165) Imagine we have items A and B. B has a similar item C. If B is added as similar to A, then C will also be moved from B to A. Previously, in that operation, the user who created the relationship between B and C would be set as the user who created the relationship between A and C, but that's problematic - for example, it can prevent reports from being sent. The fix here is to set that the user who created the new relationship between A and C is not the same user who created the relationship between B and C, but instead, it's the user who created the relationship between A and B. Fixes: CV2-5837. --- app/models/relationship.rb | 2 +- config/initializers/version.rb | 2 +- test/models/relationship_2_test.rb | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/models/relationship.rb b/app/models/relationship.rb index 868b74fe09..91449f583d 100644 --- a/app/models/relationship.rb +++ b/app/models/relationship.rb @@ -312,7 +312,7 @@ def point_targets_to_new_source Relationship.where(source_id: self.target_id).find_each do |old_relationship| old_relationship.delete options = { - user_id: old_relationship.user_id, + user_id: User.current&.id || old_relationship.user_id, weight: old_relationship.weight } Relationship.create_unless_exists(self.source_id, old_relationship.target_id, old_relationship.relationship_type, options) diff --git a/config/initializers/version.rb b/config/initializers/version.rb index a4e55ec0ac..749280e526 100644 --- a/config/initializers/version.rb +++ b/config/initializers/version.rb @@ -1 +1 @@ -VERSION = '0.0.1' +VERSION = 'v0.185.3' diff --git a/test/models/relationship_2_test.rb b/test/models/relationship_2_test.rb index 4e8d976f70..e15917d4b7 100644 --- a/test/models/relationship_2_test.rb +++ b/test/models/relationship_2_test.rb @@ -488,4 +488,19 @@ def teardown assert Relationship.where(source: a, target: b).exists? assert !Relationship.where(source: a, target: c).exists? end + + test "should set current user when moving targets to new source" do + t = create_team + u1 = create_user is_admin: true + u2 = create_user is_admin: true + a = create_project_media team: t + b = create_project_media team: t + c = create_project_media team: t + create_relationship source: b, target: c, user: u1 + with_current_user_and_team(u2, t) do + create_relationship source: a, target: b + end + r = Relationship.where(source: a, target: c).last + assert_equal u2, r.reload.user + end end From 71fcb6c44eadc8fabaf3c19dea7fbbc7c616f3f5 Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:21:29 +0300 Subject: [PATCH 16/52] Add missing translations (#2168) Add missing translations for the `no_results_in_language` string. --- config/tipline_strings.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 6a3108390a..116607747f 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -11,7 +11,7 @@ ar: main_menu: القائمة الرئيسية main_state_button_label: إلغاء navigation_button: استخدم الأزرار للتنقل - no_results_in_language: '' + no_results_in_language: 'لم يتم العثور على نتائج في %{language}. إليك بعض النتائج بلغات أخرى قد تكون ذات صلة.' privacy_and_purpose: |- الخصوصية والغرض @@ -276,7 +276,7 @@ zh_CN: main_menu: 主菜单 main_state_button_label: 取消 navigation_button: 使用按键浏览 - no_results_in_language: '' + no_results_in_language: '在%{language}中未找到结果。以下是可能相关的其他语言的结果。' privacy_and_purpose: |- 隐私与用途 @@ -390,7 +390,7 @@ fr: main_menu: Menu principal main_state_button_label: Annuler navigation_button: Utilisez les boutons pour naviguer - no_results_in_language: '' + no_results_in_language: 'Aucun résultat trouvé en %{language}. Voici quelques résultats dans d'autres langues qui pourraient être pertinents.' privacy_and_purpose: |- Confidentialité et objectif @@ -428,7 +428,7 @@ de: main_menu: Hauptmenü main_state_button_label: Abbrechen navigation_button: Verwenden Sie die Schaltflächen zum Navigieren - no_results_in_language: '' + no_results_in_language: 'Keine Ergebnisse in %{language} gefunden. Hier sind einige Ergebnisse in anderen Sprachen, die relevant sein könnten.' privacy_and_purpose: |- Datenschutz und Zweck @@ -540,7 +540,7 @@ id: main_menu: Menu utama main_state_button_label: Batalkan navigation_button: Gunakan tombol untuk mendapatkan navigasi - no_results_in_language: '' + no_results_in_language: 'Tidak ada hasil yang ditemukan dalam %{language}. Berikut beberapa hasil dalam bahasa lain yang mungkin relevan.' privacy_and_purpose: |- Privasi dan Tujuan @@ -954,7 +954,7 @@ es: main_menu: Menú principal main_state_button_label: Cancelar navigation_button: Usa el menú para ver más opciones - no_results_in_language: '' + no_results_in_language: 'No se encontraron resultados en %{language}. Aquí hay algunos resultados en otros idiomas que pueden ser relevantes.' privacy_and_purpose: |- Privacidad y Propósito From f11838f9fc21000286da0612897c220b4fae3aeb Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Mon, 6 Jan 2025 21:50:00 +0300 Subject: [PATCH 17/52] Revert translation updates (#2171) --- config/tipline_strings.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 116607747f..6a3108390a 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -11,7 +11,7 @@ ar: main_menu: القائمة الرئيسية main_state_button_label: إلغاء navigation_button: استخدم الأزرار للتنقل - no_results_in_language: 'لم يتم العثور على نتائج في %{language}. إليك بعض النتائج بلغات أخرى قد تكون ذات صلة.' + no_results_in_language: '' privacy_and_purpose: |- الخصوصية والغرض @@ -276,7 +276,7 @@ zh_CN: main_menu: 主菜单 main_state_button_label: 取消 navigation_button: 使用按键浏览 - no_results_in_language: '在%{language}中未找到结果。以下是可能相关的其他语言的结果。' + no_results_in_language: '' privacy_and_purpose: |- 隐私与用途 @@ -390,7 +390,7 @@ fr: main_menu: Menu principal main_state_button_label: Annuler navigation_button: Utilisez les boutons pour naviguer - no_results_in_language: 'Aucun résultat trouvé en %{language}. Voici quelques résultats dans d'autres langues qui pourraient être pertinents.' + no_results_in_language: '' privacy_and_purpose: |- Confidentialité et objectif @@ -428,7 +428,7 @@ de: main_menu: Hauptmenü main_state_button_label: Abbrechen navigation_button: Verwenden Sie die Schaltflächen zum Navigieren - no_results_in_language: 'Keine Ergebnisse in %{language} gefunden. Hier sind einige Ergebnisse in anderen Sprachen, die relevant sein könnten.' + no_results_in_language: '' privacy_and_purpose: |- Datenschutz und Zweck @@ -540,7 +540,7 @@ id: main_menu: Menu utama main_state_button_label: Batalkan navigation_button: Gunakan tombol untuk mendapatkan navigasi - no_results_in_language: 'Tidak ada hasil yang ditemukan dalam %{language}. Berikut beberapa hasil dalam bahasa lain yang mungkin relevan.' + no_results_in_language: '' privacy_and_purpose: |- Privasi dan Tujuan @@ -954,7 +954,7 @@ es: main_menu: Menú principal main_state_button_label: Cancelar navigation_button: Usa el menú para ver más opciones - no_results_in_language: 'No se encontraron resultados en %{language}. Aquí hay algunos resultados en otros idiomas que pueden ser relevantes.' + no_results_in_language: '' privacy_and_purpose: |- Privacidad y Propósito From 43fbfa56ff68ecee99490150196994f5b2381f7c Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:23:57 +0300 Subject: [PATCH 18/52] Regenerate Transifex translations (#2170) Add missing translations Add missing translations for the `no_results_in_language` string. --- config/tipline_strings.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/tipline_strings.yml b/config/tipline_strings.yml index 6a3108390a..19a9531276 100644 --- a/config/tipline_strings.yml +++ b/config/tipline_strings.yml @@ -11,7 +11,7 @@ ar: main_menu: القائمة الرئيسية main_state_button_label: إلغاء navigation_button: استخدم الأزرار للتنقل - no_results_in_language: '' + no_results_in_language: لم يتم العثور على نتائج في %{language}. إليك بعض النتائج بلغات أخرى قد تكون ذات صلة. privacy_and_purpose: |- الخصوصية والغرض @@ -276,7 +276,7 @@ zh_CN: main_menu: 主菜单 main_state_button_label: 取消 navigation_button: 使用按键浏览 - no_results_in_language: '' + no_results_in_language: 在%{language}中未找到结果。以下是可能相关的其他语言的结果。 privacy_and_purpose: |- 隐私与用途 @@ -390,7 +390,7 @@ fr: main_menu: Menu principal main_state_button_label: Annuler navigation_button: Utilisez les boutons pour naviguer - no_results_in_language: '' + no_results_in_language: Aucun résultat trouvé en %{language}. Voici quelques résultats dans d’autres langues qui pourraient être pertinents. privacy_and_purpose: |- Confidentialité et objectif @@ -428,7 +428,7 @@ de: main_menu: Hauptmenü main_state_button_label: Abbrechen navigation_button: Verwenden Sie die Schaltflächen zum Navigieren - no_results_in_language: '' + no_results_in_language: Keine Ergebnisse in %{language} gefunden. Hier sind einige Ergebnisse in anderen Sprachen, die relevant sein könnten. privacy_and_purpose: |- Datenschutz und Zweck @@ -540,7 +540,7 @@ id: main_menu: Menu utama main_state_button_label: Batalkan navigation_button: Gunakan tombol untuk mendapatkan navigasi - no_results_in_language: '' + no_results_in_language: Tidak ada hasil yang ditemukan dalam %{language}. Berikut beberapa hasil dalam bahasa lain yang mungkin relevan. privacy_and_purpose: |- Privasi dan Tujuan @@ -954,7 +954,7 @@ es: main_menu: Menú principal main_state_button_label: Cancelar navigation_button: Usa el menú para ver más opciones - no_results_in_language: '' + no_results_in_language: No se encontraron resultados en %{language}. Aquí hay algunos resultados en otros idiomas que pueden ser relevantes. privacy_and_purpose: |- Privacidad y Propósito From d5bf9b7c7d8db3c136f556a7ffea6b7165aeb276 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 7 Jan 2025 16:53:00 +0200 Subject: [PATCH 19/52] User join query to get max date (#2169) * User join query to get max date * CV2-5856: fix tests --- .../concerns/project_media_cached_fields.rb | 22 +++++++++++++++---- test/models/bot/alegre_test.rb | 1 + test/models/tipline_newsletter_test.rb | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb index bfad29395c..96561d9c13 100644 --- a/app/models/concerns/project_media_cached_fields.rb +++ b/app/models/concerns/project_media_cached_fields.rb @@ -550,10 +550,24 @@ def recalculate_demand def recalculate_last_seen # If it’s a main/parent item, last_seen is related to any tipline request to that own ProjectMedia or any similar/child ProjectMedia # If it’s not a main item (so, single or child, a.k.a. “confirmed match” or “suggestion”), then last_seen is related only to tipline requests related to that ProjectMedia. - ids = self.is_parent ? self.related_items_ids : self.id - v1 = TiplineRequest.where(associated_type: 'ProjectMedia', associated_id: ids).order('created_at DESC').first&.created_at || 0 - v2 = ProjectMedia.where(id: ids).order('created_at DESC').first&.created_at || 0 - [v1, v2].max.to_i + v1 = [0] + v2 = [0] + parent = self + if self.is_parent + parent = Relationship.confirmed.where(target_id: self.id).last&.source || self + result = Relationship.select('MAX(pm.created_at) as pm_c, MAX(tr.created_at) as tr_c') + .where(relationship_type: Relationship.confirmed_type, source_id: parent.id) + .joins("INNER JOIN project_medias pm ON pm.id = relationships.target_id") + .joins("LEFT JOIN tipline_requests tr ON tr.associated_id = relationships.target_id AND tr.associated_type = 'ProjectMedia'") + v1.concat(result.map(&:tr_c)) + v2.concat(result.map(&:pm_c)) + end + result = ProjectMedia.select('MAX(project_medias.created_at) as pm_c, MAX(tr.created_at) as tr_c') + .where(id: parent.id) + .joins("INNER JOIN tipline_requests tr ON tr.associated_id = project_medias.id AND tr.associated_type = 'ProjectMedia'") + v1.concat(result.map(&:tr_c)) + v2.concat(result.map(&:pm_c)) + [v1, v2].flatten.map(&:to_i).max end def recalculate_fact_check_id diff --git a/test/models/bot/alegre_test.rb b/test/models/bot/alegre_test.rb index f15e9b5d08..00787c5529 100644 --- a/test/models/bot/alegre_test.rb +++ b/test/models/bot/alegre_test.rb @@ -171,6 +171,7 @@ def teardown end test "should set similarity relationship based on date threshold" do + RequestStore.store[:skip_cached_field_update] = false create_verification_status_stuff p = create_project team: @team pm1 = create_project_media project: p, quote: "This is also a long enough Title so as to allow an actual check of other titles", team: @team diff --git a/test/models/tipline_newsletter_test.rb b/test/models/tipline_newsletter_test.rb index a209216c7c..f31c093085 100644 --- a/test/models/tipline_newsletter_test.rb +++ b/test/models/tipline_newsletter_test.rb @@ -10,7 +10,7 @@ def setup rss_feed_url: 'https://example.com/feed', number_of_articles: 3, send_every: ['monday'], - send_on: Time.parse('2025-01-01'), + send_on: Time.parse('2030-01-01'), timezone: 'UTC', time: Time.parse('10:00'), footer: 'Test', From 819bc4a3d3a834de2be6f5bee5e50e69ef996b76 Mon Sep 17 00:00:00 2001 From: Daniele Valverde <34126648+danielevalverde@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:06:40 -0300 Subject: [PATCH 20/52] Create and associate ProjectMedia to a fact-check for proper indexing (#2172) Fix helper method to proper indexing fact check in Alegre Reference: CV2-5737, CV2-5900 --- app/controllers/test_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/test_controller.rb b/app/controllers/test_controller.rb index 69ec297339..a40f260772 100644 --- a/app/controllers/test_controller.rb +++ b/app/controllers/test_controller.rb @@ -235,12 +235,15 @@ def create_imported_standalone_fact_check url = params[:url] language = params[:language] || 'en' + project_media = ProjectMedia.create!(media: Blank.create!, team: team, user: user) + # Create ClaimDescription claim_description = ClaimDescription.create!( description: description, context: context, user: user, - team: team + team: team, + project_media: project_media ) # Set up FactCheck From b7524affcd58fa4a5b77409ab60c651cb225d617 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 12 Jan 2025 14:19:10 -0300 Subject: [PATCH 21/52] Return early from method if the item doesn't exist anymore. (#2175) If a `ProjectMedia` is deleted, handling task information about it is useless. So, return early in this case. Fixes: CV2-5989. --- app/models/annotations/dynamic.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/annotations/dynamic.rb b/app/models/annotations/dynamic.rb index 596c9c582e..bdb509a3f0 100644 --- a/app/models/annotations/dynamic.rb +++ b/app/models/annotations/dynamic.rb @@ -166,6 +166,7 @@ def handle_annotated_by(op) task = self.annotated if task&.annotated_type == 'ProjectMedia' pm = task.project_media + return if pm.nil? key = "project_media:annotated_by:#{pm.id}" uids = [] if Rails.cache.exist?(key) From 5c4cb6f571948ea6235c841eb9e996ac392cdf6f Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 13 Jan 2025 07:52:28 +0200 Subject: [PATCH 22/52] CV2-5847: Use OCR information for image item (#2176) * CV2-5847: Use OCR information for image item * CV2-5847: filter by published and unpublished articles --- app/models/concerns/smooch_search.rb | 23 ++++++++++++----------- app/models/project_media.rb | 4 +++- test/models/project_media_7_test.rb | 23 ++++++++++++++++++++--- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index f24bd1119f..c82bf71313 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -78,7 +78,7 @@ def go_to_state_and_ask_if_ready_to_submit(uid, language, workflow) self.ask_if_ready_to_submit(uid, workflow, 'ask_if_ready', language) end - def filter_search_results(pms, after, feed_id, team_ids) + def filter_search_results(pms, after, feed_id, team_ids, published_only) return [] if pms.empty? feed_results = [] if feed_id && team_ids @@ -87,12 +87,13 @@ def filter_search_results(pms, after, feed_id, team_ids) feed_results = CheckSearch.new(filters.to_json, nil, team_ids).medias.to_a.map(&:id) end pms.compact_blank.select do |pm| - (feed_id && feed_results.include?(pm.id)) || (!feed_id && pm.updated_at.to_i > after.to_i && is_a_valid_search_result(pm)) + (feed_id && feed_results.include?(pm.id)) || (!feed_id && pm.updated_at.to_i > after.to_i && is_a_valid_search_result(pm, published_only)) end end - def is_a_valid_search_result(pm) - (pm.report_status == 'published' || pm.explainers.count > 0) && [CheckArchivedFlags::FlagCodes::NONE, CheckArchivedFlags::FlagCodes::UNCONFIRMED].include?(pm.archived) + def is_a_valid_search_result(pm, published_only) + published_condition = (published_only ? pm.report_status == 'published' : true) + (published_condition || pm.explainers.count > 0) && [CheckArchivedFlags::FlagCodes::NONE, CheckArchivedFlags::FlagCodes::UNCONFIRMED].include?(pm.archived) end def reject_temporary_results(results) @@ -101,9 +102,9 @@ def reject_temporary_results(results) end end - def parse_search_results_from_alegre(results, limit, after = nil, feed_id = nil, team_ids = nil) + def parse_search_results_from_alegre(results, limit, published_only, after = nil, feed_id = nil, team_ids = nil) pms = reject_temporary_results(results).sort_by{ |a| [a[1][:model] != Bot::Alegre::ELASTICSEARCH_MODEL ? 1 : 0, a[1][:score]] }.to_h.keys.reverse.collect{ |id| Relationship.confirmed_parent(ProjectMedia.find_by_id(id)) } - filter_search_results(pms, after, feed_id, team_ids).uniq(&:id).first(limit) + filter_search_results(pms, after, feed_id, team_ids, published_only).uniq(&:id).first(limit) end def date_filter(team_id) @@ -128,14 +129,14 @@ def get_search_query(uid, last_message) self.bundle_list_of_messages(list, last_message, true) end - def get_search_results(uid, message, team_id, language, limit) + def get_search_results(uid, message, team_id, language, limit, published_only = true) results = [] begin type = message['type'] after = self.date_filter(team_id) query = message['text'] query = CheckS3.rewrite_url(message['mediaUrl']) unless type == 'text' - results = self.search_for_similar_published_fact_checks(type, query, [team_id], limit, after, nil, language).select{ |pm| is_a_valid_search_result(pm) } + results = self.search_for_similar_published_fact_checks(type, query, [team_id], limit, after, nil, language).select{ |pm| is_a_valid_search_result(pm, published_only) } rescue StandardError => e self.handle_search_error(uid, e, language) end @@ -171,7 +172,7 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, lim unless link.nil? Rails.logger.info "[Smooch Bot] Search query (URL): #{link.url}" pms = ProjectMedia.joins(:media).where('medias.url' => link.url, 'project_medias.team_id' => team_ids).to_a - result = self.filter_search_results(pms, after, feed_id, team_ids) + result = self.filter_search_results(pms, after, feed_id, team_ids, published_only) return result unless result.empty? text = [link.pender_data['description'].to_s, text.to_s.gsub(/https?:\/\/[^\s]+/, '').strip].max_by(&:length) end @@ -183,7 +184,7 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, lim results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language, published_only) else alegre_results = Bot::Alegre.get_merged_similar_items(pm, [{ value: self.get_text_similarity_threshold }], Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, text, team_ids) - results = self.parse_search_results_from_alegre(alegre_results, limit, after, feed_id, team_ids) + results = self.parse_search_results_from_alegre(alegre_results, limit, published_only, after, feed_id, team_ids) Rails.logger.info "[Smooch Bot] Text similarity search got #{results.count} results while looking for '#{text}' after date #{after.inspect} for teams #{team_ids}" end else @@ -193,7 +194,7 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, lim media_url = self.save_locally_and_return_url(media_url, type, feed_id) threshold = Bot::Alegre.get_threshold_for_query(type, pm)[0][:value] alegre_results = Bot::Alegre.get_items_with_similar_media_v2(media_url: media_url, threshold: [{ value: threshold }], team_ids: team_ids, type: type) - results = self.parse_search_results_from_alegre(alegre_results, limit, after, feed_id, team_ids) + results = self.parse_search_results_from_alegre(alegre_results, limit, published_only, after, feed_id, team_ids) Rails.logger.info "[Smooch Bot] Media similarity search got #{results.count} results while looking for '#{query}' after date #{after.inspect} for teams #{team_ids}" end results diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 8cfc39e82d..331b65b910 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -467,8 +467,10 @@ def get_similar_articles search_query = case media.type when 'Claim' media.quote - when 'UploadedVideo', 'UploadedAudio', 'UploadedImage' + when 'UploadedVideo', 'UploadedAudio' self.transcription + when 'UploadedImage' + self.extracted_text end search_query ||= self.title results = self.team.search_for_similar_articles(search_query, self) diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index 18dcbedea0..2ae8f149fe 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -160,10 +160,27 @@ def setup } } create_annotation_type_and_fields('Transcription', {}, json_schema) - img = create_uploaded_image - pm_i = create_project_media team: t, media: img + audio = create_uploaded_audio + pm_a = create_project_media team: t, media: audio data = { 'job_status' => 'COMPLETED', 'transcription' => 'Foo Bar'} - a = create_dynamic_annotation annotation_type: 'transcription', annotated: pm_i, set_fields: { text: 'Foo Bar', job_name: '0c481e87f2774b1bd41a0a70d9b70d11', last_response: data }.to_json + create_dynamic_annotation annotation_type: 'transcription', annotated: pm_a, set_fields: { text: 'Foo Bar', job_name: '0c481e87f2774b1bd41a0a70d9b70d11', last_response: data }.to_json + sleep 1 + assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].concat([ex1.id, ex2.id, ex3.id]).sort, pm_a.get_similar_articles.map(&:id).sort + # Verify search query for images + create_extracted_text_annotation_type + pm_i = nil + stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do + Sidekiq::Testing.fake! do + WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ + WebMock.stub_request(:post, 'http://alegre/image/ocr/').with({ body: { url: "some/path" } }).to_return(body: { text: 'Foo Bar' }.to_json) + WebMock.stub_request(:post, 'http://alegre/text/similarity/') + Bot::Alegre.unstub(:media_file_url) + pm_i = create_project_media team: t, media: create_uploaded_image + Bot::Alegre.stubs(:media_file_url).with(pm_i).returns('some/path') + Bot::Alegre.get_extracted_text(pm_i) + Bot::Alegre.unstub(:media_file_url) + end + end sleep 1 assert_equal [pm1.fact_check_id, pm2.fact_check_id, pm3.fact_check_id].concat([ex1.id, ex2.id, ex3.id]).sort, pm_i.get_similar_articles.map(&:id).sort Bot::Smooch.unstub(:search_for_explainers) From 2370b29a3e8e7eb35ba18e2bab0ea29726556503 Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Mon, 13 Jan 2025 08:54:00 +0300 Subject: [PATCH 23/52] Explainers should be indexed using `team_id`, not `team_slug` (#2167) * Explainers should be indexed using `team_id`, not `team_slug` Currently, fact checks are indexed with `team_id`, but explainers are indexed with `team_slug`. Here we are changing the indexing for explainers to use `team_id`. * Add rake task to reindex explainers Add rake task to reindex all explainers. * Applying Scott's comment from PR review --------- Co-authored-by: Caio <117518+caiosba@users.noreply.github.com> --- app/models/explainer.rb | 4 +-- .../20250108070424_reindex_explainers.rake | 26 ++++++++++++++ .../20250108070424_reindex_explainers_test.rb | 34 +++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 lib/tasks/migrate/20250108070424_reindex_explainers.rake create mode 100644 test/lib/tasks/migrate/20250108070424_reindex_explainers_test.rb diff --git a/app/models/explainer.rb b/app/models/explainer.rb index c6f9da7e71..96e81be933 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -61,7 +61,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) base_context = { type: 'explainer', - team: explainer.team.slug, + team_id: explainer.team_id, language: explainer.language, explainer_id: explainer.id } @@ -107,7 +107,7 @@ def self.search_by_similarity(text, language, team_id, limit) models_thresholds = Explainer.get_alegre_models_and_thresholds(team_id) context = { type: 'explainer', - team: Team.find(team_id).slug + team_id: team_id } context[:language] = language unless language.nil? params = { diff --git a/lib/tasks/migrate/20250108070424_reindex_explainers.rake b/lib/tasks/migrate/20250108070424_reindex_explainers.rake new file mode 100644 index 0000000000..8799671bdc --- /dev/null +++ b/lib/tasks/migrate/20250108070424_reindex_explainers.rake @@ -0,0 +1,26 @@ +# bundle exec rake check:migrate:reindex_explainers + +namespace :check do + namespace :migrate do + desc 'Reindex all explainers' + task reindex_explainers: :environment do + cache_key = 'check:migrate:reindex_explainers:last_migrated_id' + last_migrated_id = Rails.cache.read(cache_key).to_i + + puts "[#{Time.now}] Starting reindex of all explainers" + + Explainer.where('id > ?', last_migrated_id).order('id ASC').find_each(batch_size: 100) do |explainer| + begin + Explainer.update_paragraphs_in_alegre(explainer.id, 0, Time.now.to_f) + Rails.cache.write(cache_key, explainer.id) + puts "[#{Time.now}] Successfully reindexed explainer with ID #{explainer.id}" + rescue StandardError => e + Rails.logger.error "[#{Time.now}] Error reindexing explainer with ID #{explainer.id}: #{e.message}" + end + end + + Rails.cache.delete(cache_key) + puts "[#{Time.now}] Successfully reindexed all explainers" + end + end +end diff --git a/test/lib/tasks/migrate/20250108070424_reindex_explainers_test.rb b/test/lib/tasks/migrate/20250108070424_reindex_explainers_test.rb new file mode 100644 index 0000000000..8cd5e1883e --- /dev/null +++ b/test/lib/tasks/migrate/20250108070424_reindex_explainers_test.rb @@ -0,0 +1,34 @@ +require_relative '../../../test_helper' +require 'rake' + +class ReindexExplainersTest < ActiveSupport::TestCase + def setup + Rake.application.rake_require('tasks/migrate/20250108070424_reindex_explainers') + Rake::Task.define_task(:environment) + Rake::Task['check:migrate:reindex_explainers'].reenable + + @explainer = create_explainer + Explainer.stubs(:update_paragraphs_in_alegre).returns(true) + Rails.cache.write('check:migrate:reindex_explainers:last_migrated_id', 0) + end + + def teardown + @explainer.destroy + Rails.cache.delete('check:migrate:reindex_explainers:last_migrated_id') + Explainer.unstub(:update_paragraphs_in_alegre) + end + + test "should reindex explainers via rake task" do + out, err = capture_io do + Rake::Task['check:migrate:reindex_explainers'].invoke + end + + Rake::Task['check:migrate:reindex_explainers'].reenable + + assert err.blank?, "Expected no errors, but got: #{err}" + + assert_match /Starting reindex of all explainers/, out + assert_match /Successfully reindexed explainer with ID #{@explainer.id}/, out + assert_match /Successfully reindexed all explainers/, out + end +end From c0f714cfe79e6c4379c757b62c57418b6fac2748 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:03:33 -0300 Subject: [PATCH 24/52] Adding an accessor `enable_create_blank_media` to the `ClaimDescription` model. (#2177) Adding an accessor `enable_create_blank_media` to the `ClaimDescription` model. Also exposed as an argument for the claim description GraphQL mutations. When set to `true` (default is `false`), the claim description is associated with a blank item when it's not associated to any item. Reference: CV2-5900. --- .../mutations/claim_description_mutations.rb | 1 + app/models/claim_description.rb | 11 +++++++-- lib/relay.idl | 2 ++ lib/sample_data.rb | 3 ++- public/relay.json | 24 +++++++++++++++++++ test/models/claim_description_test.rb | 21 ++++++++++++++++ 6 files changed, 59 insertions(+), 3 deletions(-) diff --git a/app/graph/mutations/claim_description_mutations.rb b/app/graph/mutations/claim_description_mutations.rb index dc8a9ac977..625d2a73ac 100644 --- a/app/graph/mutations/claim_description_mutations.rb +++ b/app/graph/mutations/claim_description_mutations.rb @@ -9,6 +9,7 @@ module SharedCreateAndUpdateFields argument :description, GraphQL::Types::String, required: false argument :context, GraphQL::Types::String, required: false, as: :claim_context argument :project_media_id, GraphQL::Types::Int, required: false, camelize: false + argument :enable_create_blank_media, GraphQL::Types::Boolean, required: false, camelize: false end end diff --git a/app/models/claim_description.rb b/app/models/claim_description.rb index b2d1ece896..35be97c6d0 100644 --- a/app/models/claim_description.rb +++ b/app/models/claim_description.rb @@ -1,20 +1,21 @@ class ClaimDescription < ApplicationRecord - attr_accessor :disable_replace_media + attr_accessor :disable_replace_media, :enable_create_blank_media include Article has_paper_trail on: [:create, :update], ignore: [:updated_at, :created_at], if: proc { |_x| User.current.present? }, versions: { class_name: 'Version' } - before_validation :set_team, on: :create belongs_to :project_media, optional: true belongs_to :team has_one :fact_check, dependent: :destroy accepts_nested_attributes_for :fact_check, reject_if: proc { |attributes| attributes['summary'].blank? } + before_validation :set_team, on: :create validates_presence_of :team validates_uniqueness_of :project_media_id, allow_nil: true validate :cant_apply_article_to_item_if_article_is_in_the_trash + before_create :create_blank_media_if_needed after_commit :update_fact_check, on: [:update] after_update :update_report after_update :reset_item_rating_if_removed @@ -136,4 +137,10 @@ def reset_item_rating_if_removed end end end + + def create_blank_media_if_needed + if self.enable_create_blank_media && self.project_media_id.blank? + self.project_media = ProjectMedia.create!(media: Blank.create!, team: self.team) + end + end end diff --git a/lib/relay.idl b/lib/relay.idl index cf9c3f1168..1301253858 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -1180,6 +1180,7 @@ input CreateClaimDescriptionInput { clientMutationId: String context: String description: String + enable_create_blank_media: Boolean project_media_id: Int } @@ -14187,6 +14188,7 @@ input UpdateClaimDescriptionInput { clientMutationId: String context: String description: String + enable_create_blank_media: Boolean id: ID project_media_id: Int } diff --git a/lib/sample_data.rb b/lib/sample_data.rb index 297716a0c3..ccd7c01650 100644 --- a/lib/sample_data.rb +++ b/lib/sample_data.rb @@ -900,7 +900,8 @@ def create_claim_description(options = {}) description: random_string, context: random_string, user: options[:user] || create_user, - project_media: options.has_key?(:project_media) ? options[:project_media] : create_project_media + project_media: options.has_key?(:project_media) ? options[:project_media] : create_project_media, + enable_create_blank_media: options[:enable_create_blank_media] }.merge(options)) end diff --git a/public/relay.json b/public/relay.json index 19244acc86..b703fa38e7 100644 --- a/public/relay.json +++ b/public/relay.json @@ -5907,6 +5907,18 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "enable_create_blank_media", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", @@ -75357,6 +75369,18 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "enable_create_blank_media", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "clientMutationId", "description": "A unique identifier for the client performing the mutation.", diff --git a/test/models/claim_description_test.rb b/test/models/claim_description_test.rb index 3a8418606d..746d1207f2 100644 --- a/test/models/claim_description_test.rb +++ b/test/models/claim_description_test.rb @@ -209,4 +209,25 @@ def setup cd.save! end end + + test "should create blank media if needed" do + t = create_team + pm = create_project_media team: t + cd = nil + + assert_no_difference 'Blank.count' do + cd = create_claim_description project_media: nil, team: t + end + assert_nil cd.project_media + + assert_no_difference 'Blank.count' do + cd = create_claim_description project_media: pm, team: t, enable_create_blank_media: true + end + assert_equal pm, cd.project_media + + assert_difference 'Blank.count' do + cd = create_claim_description project_media: nil, team: t, enable_create_blank_media: true + end + assert cd.project_media.media.is_a?(Blank) + end end From d655925093546f7af7d98ef91b599da4d561ef0d Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:39:01 -0300 Subject: [PATCH 25/52] Media cluster origin (#2174) Adding new cached fields to `ProjectMedia` items, also exposed in the GraphQL `ProjectMediaType`, for the media origin, which explains how that media ended up in that particular media cluster. The new fields are: - `media_cluster_origin` - `media_cluster_origin_user` - `media_cluster_origin_timestamp` - `media_cluster_relationship` The (current) possible values for the media origin are: - Tipline-submitted - Manually-created - User matched - User merged - Auto merged The goal of this PR is to cover just these cases. Other cases may be handled in the future. Steps: - [x] Media cluster origin library, with all possible values represented by different constants. - [x] Expose it in GraphQL `AboutType`. - [x] Media cluster origin. - [x] Expose it in GraphQL `ProjectMediaType`. - [x] Media cluster origin author. - [x] Expose it in GraphQL `ProjectMediaType`. - [x] Media cluster origin timestamp. - [x] Expose it in GraphQL `ProjectMediaType`. - [x] Media cluster relationship. - [x] Expose it in GraphQL `ProjectMediaType`. - [x] Implement logic as cached fields for the different cases and fields: - [x] Initial / Empty state. - [x] Origin value. - [x] User. - [x] Timestamp. - [x] Tipline submission. - [x] Origin value - [x] User - [x] Timestamp - [x] User submitted. - [x] Origin value - [x] User - [x] Timestamp - [x] User merged. - [x] Origin value - [x] User - [x] Timestamp - [x] User confirmed. - [x] Origin value - [x] User - [x] Timestamp - [x] Auto matched. - [x] Origin value - [x] User - [x] Timestamp - [x] Implement automated tests. References: CV2-5933. --- app/graph/types/about_type.rb | 1 + app/graph/types/project_media_type.rb | 20 ++++++ app/graph/types/query_type.rb | 3 +- app/lib/check_media_cluster_origins.rb | 20 ++++++ .../concerns/project_media_cached_fields.rb | 55 +++++++++++++++ config/initializers/plugins.rb | 2 +- lib/relay.idl | 9 +++ public/relay.json | 70 +++++++++++++++++++ .../controllers/graphql_controller_11_test.rb | 22 ++++++ test/models/project_media_8_test.rb | 53 ++++++++++++++ 10 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 app/lib/check_media_cluster_origins.rb diff --git a/app/graph/types/about_type.rb b/app/graph/types/about_type.rb index 01a3f69eaf..2508c3cba1 100644 --- a/app/graph/types/about_type.rb +++ b/app/graph/types/about_type.rb @@ -29,4 +29,5 @@ class AboutType < BaseObject field :channels, JsonStringType, "List check channels", null: true field :countries, JsonStringType, "List of workspace countries", null: true + field :media_cluster_origins, JsonStringType, "List of media cluster origins", null: true end diff --git a/app/graph/types/project_media_type.rb b/app/graph/types/project_media_type.rb index 986cd4097c..05a09ed7b2 100644 --- a/app/graph/types/project_media_type.rb +++ b/app/graph/types/project_media_type.rb @@ -406,4 +406,24 @@ def relevant_articles def relevant_articles_count object.get_similar_articles.count end + + field :media_cluster_origin, GraphQL::Types::Int, null: true + field :media_cluster_origin_timestamp, GraphQL::Types::Int, null: true + field :media_cluster_origin_user, UserType, null: true + + def media_cluster_origin_user + RecordLoader + .for(User) + .load(object.media_cluster_origin_user_id) + .then do |user| + ability = context[:ability] || Ability.new + user if ability.can?(:read, user) + end + end + + field :media_cluster_relationship, RelationshipType, null: true + + def media_cluster_relationship + Relationship.where(target_id: object.id).last || Relationship.where(source_id: object.id).last + end end diff --git a/app/graph/types/query_type.rb b/app/graph/types/query_type.rb index 8ff93f7a16..62265a522a 100644 --- a/app/graph/types/query_type.rb +++ b/app/graph/types/query_type.rb @@ -50,7 +50,8 @@ def about "#{SizeValidator.config("max_width")}x#{SizeValidator.config("max_height")}", languages_supported: CheckCldr.localized_languages.to_json, terms_last_updated_at: User.terms_last_updated_at, - channels: CheckChannels::ChannelCodes.all_channels + channels: CheckChannels::ChannelCodes.all_channels, + media_cluster_origins: CheckMediaClusterOrigins::OriginCodes.all_origins } ) end diff --git a/app/lib/check_media_cluster_origins.rb b/app/lib/check_media_cluster_origins.rb new file mode 100644 index 0000000000..aed0829d6e --- /dev/null +++ b/app/lib/check_media_cluster_origins.rb @@ -0,0 +1,20 @@ +module CheckMediaClusterOrigins + class OriginCodes + TIPLINE_SUBMITTED = 0 + USER_ADDED = 1 + USER_MERGED = 2 + USER_MATCHED = 3 + AUTO_MATCHED = 4 + ALL = [TIPLINE_SUBMITTED, USER_ADDED, USER_MERGED, USER_MATCHED, AUTO_MATCHED] + + def self.all_origins + { + 'TIPLINE_SUBMITTED' => TIPLINE_SUBMITTED, # First media of a cluster, submitted through a tipline + 'USER_ADDED' => USER_ADDED, # First media of a cluster, uploaded manually using Check Web + 'USER_MERGED' => USER_MERGED, # When a user manually-creates a relationship + 'USER_MATCHED' => USER_MATCHED, # When a user confirms a suggestion + 'AUTO_MATCHED' => AUTO_MATCHED # When a bot creates a relationship + } + end + end +end diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb index 96561d9c13..93acdafee6 100644 --- a/app/models/concerns/project_media_cached_fields.rb +++ b/app/models/concerns/project_media_cached_fields.rb @@ -515,6 +515,18 @@ def title_or_description_update } ] + cached_field :media_cluster_origin, + update_on: [SIMILARITY_EVENT], + recalculate: :recalculate_media_cluster_origin + + cached_field :media_cluster_origin_user_id, + update_on: [SIMILARITY_EVENT], + recalculate: :recalculate_media_cluster_origin_user_id + + cached_field :media_cluster_origin_timestamp, + update_on: [SIMILARITY_EVENT], + recalculate: :recalculate_media_cluster_origin_timestamp + def recalculate_linked_items_count count = Relationship.send('confirmed').where(source_id: self.id).count count += 1 unless self.media.type == 'Blank' @@ -715,6 +727,49 @@ def recalculate_tipline_search_results_count types = ["relevant_search_result_requests", "irrelevant_search_result_requests", "timeout_search_requests"] TiplineRequest.where(associated_type: 'ProjectMedia', associated_id: self.id, smooch_request_type: types).count end + + def recalculate_media_cluster_origin(field = :origin) # Possible values for "field": :origin, :user_id, :timestamp + relationship = Relationship.where(target_id: self.id).last + origin = { origin: nil, user_id: nil, timestamp: nil } + + # Not child of any media cluster + if relationship.nil? + if self.user == BotUser.smooch_user + origin[:origin] = CheckMediaClusterOrigins::OriginCodes::TIPLINE_SUBMITTED + else + origin[:origin] = CheckMediaClusterOrigins::OriginCodes::USER_ADDED + end + origin[:user_id] = self.user_id + origin[:timestamp] = self.created_at.to_i + + # Child of a media cluster + # FIXME: Replace the `elsif`'s below by a single `else` when we start handling all cases, so we don't repeat code + else + if relationship.confirmed_at # A suggestion that was confirmed + origin[:origin] = CheckMediaClusterOrigins::OriginCodes::USER_MATCHED + origin[:user_id] = relationship.confirmed_by + origin[:timestamp] = relationship.confirmed_at.to_i + elsif relationship.user == BotUser.alegre_user + origin[:origin] = CheckMediaClusterOrigins::OriginCodes::AUTO_MATCHED + origin[:user_id] = relationship.user_id + origin[:timestamp] = relationship.created_at.to_i + elsif relationship.user.is_a?(User) + origin[:origin] = CheckMediaClusterOrigins::OriginCodes::USER_MERGED + origin[:user_id] = relationship.user_id + origin[:timestamp] = relationship.created_at.to_i + end + end + + origin[field] + end + end + + def recalculate_media_cluster_origin_user_id + self.recalculate_media_cluster_origin(:user_id) + end + + def recalculate_media_cluster_origin_timestamp + self.recalculate_media_cluster_origin(:timestamp) end DynamicAnnotation::Field.class_eval do diff --git a/config/initializers/plugins.rb b/config/initializers/plugins.rb index 52e80f5d9d..36bf7b1794 100644 --- a/config/initializers/plugins.rb +++ b/config/initializers/plugins.rb @@ -1,2 +1,2 @@ # Load classes on boot, in production, that otherwise wouldn't be auto-loaded by default -CcDeville && Bot::Keep && Workflow::Workflow.workflows && CheckS3 && Bot::Tagger && Bot::Fetch && Bot::Smooch && Bot::Slack && Bot::Alegre && CheckChannels && RssFeed && UrlRewriter && ClusterTeam && ListExport && TeamStatistics && CheckDataPoints +CcDeville && Bot::Keep && Workflow::Workflow.workflows && CheckS3 && Bot::Tagger && Bot::Fetch && Bot::Smooch && Bot::Slack && Bot::Alegre && CheckChannels && RssFeed && UrlRewriter && ClusterTeam && ListExport && TeamStatistics && CheckDataPoints && CheckMediaClusterOrigins diff --git a/lib/relay.idl b/lib/relay.idl index 1301253858..1ded4b8dd1 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -53,6 +53,11 @@ type About implements Node { """ languages_supported: String + """ + List of media cluster origins + """ + media_cluster_origins: JsonStringType + """ Application name """ @@ -11654,6 +11659,10 @@ type ProjectMedia implements Node { who_dunnit: [String] ): VersionConnection media: Media + media_cluster_origin: Int + media_cluster_origin_timestamp: Int + media_cluster_origin_user: User + media_cluster_relationship: Relationship media_id: Int media_slug: String oembed_metadata: String diff --git a/public/relay.json b/public/relay.json index b703fa38e7..8e53385c6c 100644 --- a/public/relay.json +++ b/public/relay.json @@ -166,6 +166,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "media_cluster_origins", + "description": "List of media cluster origins", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "name", "description": "Application name", @@ -61421,6 +61435,62 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "media_cluster_origin", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "media_cluster_origin_timestamp", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "media_cluster_origin_user", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "media_cluster_relationship", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Relationship", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "media_id", "description": null, diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index a41fa4a002..ae8695bf14 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -355,4 +355,26 @@ def teardown actual_titles = data.map { |result| result['title'] } assert_equal expected_titles.sort, actual_titles.sort, "Results should match the search query" end + + test "should get media cluster origin fields for an item" do + Sidekiq::Testing.fake! + Relationship.delete_all + + u = create_user is_admin: true + t = create_team + pm1 = create_project_media team: t, user: u + pm2 = create_project_media team: t + r = create_relationship source: pm1, target: pm2, user: u + + authenticate_with_user(u) + query = "query { project_media(ids: \"#{pm1.id},nil,#{t.id}\") { media_cluster_relationship { dbid }, media_cluster_origin, media_cluster_origin_user { dbid }, media_cluster_origin_timestamp } }" + post :create, params: { query: query } + assert_response :success + + response = JSON.parse(@response.body)['data']['project_media'] + assert_equal r.id, response['media_cluster_relationship']['dbid'] + assert_equal u.id, response['media_cluster_origin_user']['dbid'] + assert_equal pm1.created_at.to_i, response['media_cluster_origin_timestamp'] + assert_equal CheckMediaClusterOrigins::OriginCodes::USER_ADDED, response['media_cluster_origin'] + end end diff --git a/test/models/project_media_8_test.rb b/test/models/project_media_8_test.rb index 4ec1d1d2df..48aae6a727 100644 --- a/test/models/project_media_8_test.rb +++ b/test/models/project_media_8_test.rb @@ -62,4 +62,57 @@ def teardown pm.get_deduplicated_tipline_requests } end + + test "should set media origin information" do + Sidekiq::Testing.fake! + Team.current = User.current = nil + + t = create_team + u1 = create_user + u2 = create_user + u3 = create_user + + # TIPLINE_SUBMITTED + b1 = create_bot_user login: 'smooch', name: 'Smooch', approved: true + b2 = create_bot_user login: 'alegre', name: 'Alegre', approved: true + pm1 = create_project_media team: t, user: b1 + assert_equal CheckMediaClusterOrigins::OriginCodes::TIPLINE_SUBMITTED, pm1.media_cluster_origin(true) + assert_equal pm1.created_at.to_i, pm1.media_cluster_origin_timestamp(true) + assert_equal b1.id, pm1.media_cluster_origin_user_id(true) + + # USER_ADDED + pm2 = create_project_media team: t, user: u1 + assert_equal CheckMediaClusterOrigins::OriginCodes::USER_ADDED, pm2.media_cluster_origin(true) + assert_equal pm2.created_at.to_i, pm2.media_cluster_origin_timestamp(true) + assert_equal u1.id, pm2.media_cluster_origin_user_id(true) + + # USER_MERGED + r1 = create_relationship source: pm1, target: pm2, user: u2 + assert_equal CheckMediaClusterOrigins::OriginCodes::TIPLINE_SUBMITTED, pm1.media_cluster_origin(true) + assert_equal pm1.created_at.to_i, pm1.media_cluster_origin_timestamp(true) + assert_equal b1.id, pm1.media_cluster_origin_user_id(true) + assert_equal CheckMediaClusterOrigins::OriginCodes::USER_MERGED, pm2.media_cluster_origin(true) + assert_equal r1.created_at.to_i, pm2.media_cluster_origin_timestamp(true) + assert_equal u2.id, pm2.media_cluster_origin_user_id(true) + + # USER_MATCHED + pm3 = create_project_media team: t, user: u1 + r2 = create_relationship source: pm1, target: pm3, user: b2, confirmed_at: Time.now, confirmed_by: u3.id + assert_equal CheckMediaClusterOrigins::OriginCodes::TIPLINE_SUBMITTED, pm1.media_cluster_origin(true) + assert_equal pm1.created_at.to_i, pm1.media_cluster_origin_timestamp(true) + assert_equal b1.id, pm1.media_cluster_origin_user_id(true) + assert_equal CheckMediaClusterOrigins::OriginCodes::USER_MATCHED, pm3.media_cluster_origin(true) + assert_equal r2.confirmed_at.to_i, pm3.media_cluster_origin_timestamp(true) + assert_equal u3.id, pm3.media_cluster_origin_user_id(true) + + # AUTO_MATCHED + pm4 = create_project_media team: t, user: u1 + r3 = create_relationship source: pm1, target: pm4, user: b2 + assert_equal CheckMediaClusterOrigins::OriginCodes::TIPLINE_SUBMITTED, pm1.media_cluster_origin(true) + assert_equal pm1.created_at.to_i, pm1.media_cluster_origin_timestamp(true) + assert_equal b1.id, pm1.media_cluster_origin_user_id(true) + assert_equal CheckMediaClusterOrigins::OriginCodes::AUTO_MATCHED, pm4.media_cluster_origin(true) + assert_equal r3.created_at.to_i, pm4.media_cluster_origin_timestamp(true) + assert_equal b2.id, pm4.media_cluster_origin_user_id(true) + end end From 5d42e9733b0aaf9bd6f0c5e603ed8863968857f3 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:33:35 -0300 Subject: [PATCH 26/52] Adding rake task to migrate standalone fact-checks by adding blank media to them. (#2178) Reference: CV2-5900. --- ...blank_media_to_standalone_fact_checks.rake | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/tasks/migrate/20250113231210_add_blank_media_to_standalone_fact_checks.rake diff --git a/lib/tasks/migrate/20250113231210_add_blank_media_to_standalone_fact_checks.rake b/lib/tasks/migrate/20250113231210_add_blank_media_to_standalone_fact_checks.rake new file mode 100644 index 0000000000..2c8f92afd9 --- /dev/null +++ b/lib/tasks/migrate/20250113231210_add_blank_media_to_standalone_fact_checks.rake @@ -0,0 +1,21 @@ +# rake check:migrate:add_blank_media_to_standalone_fact_checks +namespace :check do + namespace :migrate do + task add_blank_media_to_standalone_fact_checks: :environment do + started = Time.now.to_i + query = FactCheck.joins(:claim_description).where(imported: false).where('claim_descriptions.project_media_id' => nil) + n = query.count + i = 0 + query.find_each do |fact_check| + i += 1 + claim = fact_check.claim_description + claim.enable_create_blank_media = true + claim.send(:create_blank_media_if_needed) + claim.save! + puts "[#{Time.now}] [#{i}/#{n}] Added blank media to claim ##{claim.id}" + end + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes. Number of standalone fact-checks without blank media: #{query.count}" + end + end +end From ced29483ebf18a14b26d80ceaaf05384d7a44310 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:46:25 -0300 Subject: [PATCH 27/52] =?UTF-8?q?Bug=20=E2=80=93=20CV2-5913=20=E2=80=93=20?= =?UTF-8?q?Cannot=20update=20a=20new=20record=20(#2173)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had an error (ActiveRecord::ActiveRecordError: cannot update a new record) only that happened in a very specific context: - we have an existing ProjectMedia with a LinkMedia - we make a graphql request with: - the same link in set_original_claim - the same language - the title field is requested as part of the response This happened because of the way we create_link_media and handle_fact_check_for_existing_claim(existing_pm, new_pm). The solution was change handle_fact_check_for_existing_claim to not just return new_pm (since that was returning an object that was not persisted) but to try to save! it. So we either return a persisted object or an error is raised CV2-5913, PR 2173 --- app/models/project_media.rb | 2 +- .../controllers/graphql_controller_12_test.rb | 53 +++++++++++++++++-- test/models/project_media_7_test.rb | 4 +- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 331b65b910..8b8c4bfdda 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -440,7 +440,7 @@ def self.handle_fact_check_for_existing_claim(existing_pm, new_pm) return new_pm end end - new_pm + new_pm.save! end def append_fact_check_from(new_pm) diff --git a/test/controllers/graphql_controller_12_test.rb b/test/controllers/graphql_controller_12_test.rb index 4b0b16a3a5..71ecee5609 100644 --- a/test/controllers/graphql_controller_12_test.rb +++ b/test/controllers/graphql_controller_12_test.rb @@ -676,9 +676,9 @@ def teardown } } ' - post :create, params: { query: query2, team: t.slug } - assert_response :success - assert_equal 'science', JSON.parse(@response.body)['data']['createProjectMedia']['project_media']['tags']['edges'][0]['node']['tag_text'] + post :create, params: { query: query2, team: t.slug } + assert_response :success + assert_equal 'science', JSON.parse(@response.body)['data']['createProjectMedia']['project_media']['tags']['edges'][0]['node']['tag_text'] end test "should not create duplicate tags for the ProjectMedia and FactCheck" do @@ -855,4 +855,51 @@ def teardown assert_not_nil fc_2 assert_not_equal response_pm['dbid'], pm.id end + + test 'should raise exception but not create item when trying to create imported item with fact-check in the same language and existing original media URL' do + create_metadata_stuff + Sidekiq::Testing.fake! + + url = 'http://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response_body = '{"type":"media","data":{"url":"' + url + '","type":"item","title":"Test"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response_body) + + t = create_team + pm = create_project_media team: t, set_original_claim: url + cd = create_claim_description project_media: pm + fc = create_fact_check claim_description: cd + + a = ApiKey.create! + b = create_bot_user api_key_id: a.id + create_team_user team: t, user: b + authenticate_with_token(a) + + query = <<~GRAPHQL + mutation { + createProjectMedia(input: { + media_type: "Blank", + set_status: "undetermined", + set_claim_description: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + set_original_claim: "#{url}", + set_fact_check: { + title: "Fact Check Title", + summary: "Fact Check Summary", + language: "#{fc.language}", + publish_report: false + } + }) { + project_media { + title + } + } + } + GRAPHQL + + assert_nothing_raised do + post :create, params: { query: query, team: t.slug } + end + assert_response 400 + assert_equal 'This item already exists', JSON.parse(@response.body).dig('errors', 0, 'message') + end end diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index 2ae8f149fe..24bbfe1ceb 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -192,6 +192,8 @@ def setup c = create_claim_description project_media: pm1 create_fact_check claim_description: c, language: 'en' pm2 = ProjectMedia.new team: t, set_fact_check: { 'language' => 'en' } - assert_not_nil ProjectMedia.handle_fact_check_for_existing_claim(pm1, pm2) + assert_raise ActiveRecord::RecordInvalid do + ProjectMedia.handle_fact_check_for_existing_claim(pm1, pm2) + end end end From 5d4d244f64d21b00c62b8c56c52023283304b431 Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Thu, 16 Jan 2025 08:43:46 +0300 Subject: [PATCH 28/52] Fix nil:NilClass error when parsing Link smooch messages (#2179) * Fix nil:NilClass error when parsing Link smooch messages This error happens when the URL cannot be parsed. --- app/models/concerns/smooch_messages.rb | 4 ++- test/models/bot/smooch_3_test.rb | 44 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index f23daea9c8..9d728612e1 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -443,7 +443,9 @@ def smooch_relate_items_for_same_message(message, associated, app_id, author, re if text_words > self.min_number_of_words_for_tipline_long_text # Remove link from text link = self.extract_url(message['text']) - message['text'] = message['text'].remove(link.url) + if link && link.respond_to?(:url) + message['text'] = message['text'].remove(link.url) + end self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.confirmed_type) end end diff --git a/test/models/bot/smooch_3_test.rb b/test/models/bot/smooch_3_test.rb index f3cb76cdda..b211b2f692 100644 --- a/test/models/bot/smooch_3_test.rb +++ b/test/models/bot/smooch_3_test.rb @@ -780,4 +780,48 @@ def teardown end Bot::Smooch.stubs(:save_user_information).returns(nil) end + + test "should not throw exception when extract_url fails" do + # Temporarily redefine extract_url to check the caller stack + original_extract_url = Bot::Smooch.method(:extract_url) + + Bot::Smooch.define_singleton_method(:extract_url) do |text| + if caller.any? { |c| c.include?('smooch_relate_items_for_same_message') } + Rails.logger.debug "Returning nil for extract_url from smooch_relate_items_for_same_message" + nil + else + original_extract_url.call(text) # Call the original method for other cases + end + end + + begin + uid = random_string + payload = { + trigger: 'message:appUser', + app: { '_id': @app_id }, + version: 'v1.1', + messages: [ + { + '_id': random_string, + authorId: uid, + type: 'text', + source: { type: "whatsapp" }, + text: "#{@link_url} This is a long message with a link." + } + ], + appUser: { '_id': random_string, 'conversationStarted': true } + }.to_json + + Sidekiq::Testing.fake! do + Bot::Smooch.run(payload) + + assert_no_difference 'Relationship.count' do + Sidekiq::Worker.drain_all + end + end + ensure + # Restore the original extract_url method + Bot::Smooch.define_singleton_method(:extract_url, original_extract_url) + end + end end From 4a09b88707697152ac24f74d71c6e6076b12d621 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:08:08 -0300 Subject: [PATCH 29/52] =?UTF-8?q?Bug=20=E2=80=93=205926=20=E2=80=93=20Upda?= =?UTF-8?q?te=20foreign=20key=20on=20explainer=20(#2181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we have a user sign in with their email, and then sign in using multiauth we need to merge the users. When the user had created an explainer this was causing an issue, which was fixed by adding has_many :explainers to the User model. This fixes it, because of these two lines we run in merge_with: all_associations = User.reflect_on_all_associations(:has_many).select{|a| a.foreign_key == 'user_id'} all_associations.each do |assoc| assoc.class_name.constantize.where(assoc.foreign_key => user.id).update_all(assoc.foreign_key => self.id) end CV2-5926 PR 2181 --- app/models/user.rb | 1 + test/models/user_test.rb | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/app/models/user.rb b/app/models/user.rb index 3620a4e314..3e7a23291a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -29,6 +29,7 @@ class ToSOrPrivacyPolicyReadError < StandardError; end has_many :feed_invitations has_many :tipline_requests has_many :api_keys + has_many :explainers devise :registerable, :recoverable, :rememberable, :trackable, :validatable, :confirmable, diff --git a/test/models/user_test.rb b/test/models/user_test.rb index 112005e931..4689d9e74f 100644 --- a/test/models/user_test.rb +++ b/test/models/user_test.rb @@ -1409,4 +1409,15 @@ def setup assert_match /sawy/, u2.reload.source.name Team.unstub(:current) end + + test "when merging should be able to destroy a user even if they created an explainer" do + u = create_user + create_explainer user: u + + u2 = create_user + + assert_nothing_raised do + u2.merge_with(u) + end + end end From c115a52ab695f2f6062d394930f628b87deef6ab Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:45:23 -0300 Subject: [PATCH 30/52] Make sure that the URL shortening setting returns as "enabled" even when it's not explicitly enabled but there are active RSS newsletters (#2180) Make sure that the URL shortening setting returns as "enabled" even when it's not explicitly enabled but there are active RSS newsletters. Fixes: CV2-5998. --- app/models/team.rb | 4 ++++ lib/url_rewriter.rb | 1 + test/models/team_test.rb | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/app/models/team.rb b/app/models/team.rb index 18d0f45f52..f5519b8eea 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -596,6 +596,10 @@ def search_for_similar_articles(query, pm = nil) items end + def get_shorten_outgoing_urls + self.settings.to_h.with_indifferent_access[:shorten_outgoing_urls] || self.tipline_newsletters.where(content_type: 'rss', enabled: true).exists? + end + # private # # Please add private methods to app/models/concerns/team_private.rb diff --git a/lib/url_rewriter.rb b/lib/url_rewriter.rb index 8955a21be0..9f126378e7 100644 --- a/lib/url_rewriter.rb +++ b/lib/url_rewriter.rb @@ -22,6 +22,7 @@ def self.utmize(url, source) def self.shorten_and_utmize_urls(input_text, source = nil, owner = nil) text = input_text + return text if text.blank? # Encode URLs in Arabic which are not detected by the URL extraction methods text = text.gsub(/https?:\/\/[\S]+/) { |url| url =~ /\p{Arabic}/ ? Addressable::URI.escape(url) : url } if input_text =~ /\p{Arabic}/ entities = Twitter::TwitterText::Extractor.extract_urls_with_indices(text, extract_url_without_protocol: true) diff --git a/test/models/team_test.rb b/test/models/team_test.rb index 6635867cd8..c1b8509df9 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -1300,4 +1300,19 @@ def setup assert_equal 2, t.filtered_fact_checks(trashed: false).count assert_equal 1, t.filtered_fact_checks(trashed: true).count end + + test "should return that URL shortening is enabled if there are active RSS newsletters" do + t = create_team + assert !t.get_shorten_outgoing_urls + t.set_shorten_outgoing_urls = true + t.save! + assert t.get_shorten_outgoing_urls + t.set_shorten_outgoing_urls = false + t.save! + assert !t.get_shorten_outgoing_urls + tn = create_tipline_newsletter team: t, enabled: true, content_type: 'rss', rss_feed_url: random_url + assert t.get_shorten_outgoing_urls + tn.destroy! + assert !t.get_shorten_outgoing_urls + end end From ebd796035ee61b03cfa376fd641dbcb6546d11a8 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 19 Jan 2025 17:54:22 -0300 Subject: [PATCH 31/52] Adding new fields to GraphQL `RelationshipType` (#2182) Exposing new fields in GraphQL RelationshipType. All fields are columns from the relationships table, so, no logic involved in computing them, and all of them are retried in O(1). References: CV2-5934. --- app/graph/types/relationship_type.rb | 12 ++ lib/relay.idl | 8 ++ public/relay.json | 122 ++++++++++++++++++ .../controllers/graphql_controller_11_test.rb | 2 +- 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/app/graph/types/relationship_type.rb b/app/graph/types/relationship_type.rb index 861ed3419a..1be39035fe 100644 --- a/app/graph/types/relationship_type.rb +++ b/app/graph/types/relationship_type.rb @@ -9,7 +9,19 @@ class RelationshipType < BaseObject field :source_id, GraphQL::Types::Int, null: true field :permissions, GraphQL::Types::String, null: true field :relationship_type, GraphQL::Types::String, null: true + field :user_id, GraphQL::Types::Int, null: true + field :confirmed_at, GraphQL::Types::Int, null: true + field :weight, GraphQL::Types::Float, null: true + field :source_field, GraphQL::Types::String, null: true + field :target_field, GraphQL::Types::String, null: true + field :model, GraphQL::Types::String, null: true field :target, ProjectMediaType, null: true field :source, ProjectMediaType, null: true + field :user, UserType, null: true + field :confirmed_by, UserType, null: true + + def confirmed_by + User.find_by_id(object.confirmed_by) + end end diff --git a/lib/relay.idl b/lib/relay.idl index 1ded4b8dd1..d96afad880 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -12131,14 +12131,22 @@ type RejectPayload { A relationship between two items """ type Relationship implements Node { + confirmed_at: Int + confirmed_by: User dbid: Int id: ID! + model: String permissions: String relationship_type: String source: ProjectMedia + source_field: String source_id: Int target: ProjectMedia + target_field: String target_id: Int + user: User + user_id: Int + weight: Float } """ diff --git a/public/relay.json b/public/relay.json index 8e53385c6c..338e710c74 100644 --- a/public/relay.json +++ b/public/relay.json @@ -48310,6 +48310,16 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "SCALAR", + "name": "Float", + "description": "Represents signed double-precision fractional values as specified by [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point).", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, { "kind": "INPUT_OBJECT", "name": "GenerateTwoFactorBackupCodesInput", @@ -64055,6 +64065,34 @@ "name": "Relationship", "description": "A relationship between two items", "fields": [ + { + "name": "confirmed_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "confirmed_by", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "dbid", "description": null, @@ -64087,6 +64125,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "model", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "permissions", "description": null, @@ -64129,6 +64181,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "source_field", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "source_id", "description": null, @@ -64157,6 +64223,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "target_field", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "target_id", "description": null, @@ -64170,6 +64250,48 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "user", + "description": null, + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "User", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "user_id", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "weight", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index ae8695bf14..402f873882 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -367,7 +367,7 @@ def teardown r = create_relationship source: pm1, target: pm2, user: u authenticate_with_user(u) - query = "query { project_media(ids: \"#{pm1.id},nil,#{t.id}\") { media_cluster_relationship { dbid }, media_cluster_origin, media_cluster_origin_user { dbid }, media_cluster_origin_timestamp } }" + query = "query { project_media(ids: \"#{pm1.id},nil,#{t.id}\") { media_cluster_relationship { dbid, user_id, confirmed_at, weight, source_field, target_field, model, user { name }, confirmed_by { name } }, media_cluster_origin, media_cluster_origin_user { dbid }, media_cluster_origin_timestamp } }" post :create, params: { query: query } assert_response :success From 30a0f26a8e0974705df2906bd693f20569c40271 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 20 Jan 2025 18:11:07 +0200 Subject: [PATCH 32/52] CV2-5668: Record user selection/non selection for most relevant result (#2157) * CV2-5668: cache relevant articles * CV2-5668: record user selection/non selection * CV2-5668: fix tests * CV2-5668: log user selection * CV2-5668: fix CC * CV2-5668: apply PR comments * CV2-5668: fix tests --- app/models/claim_description.rb | 7 +- app/models/explainer_item.rb | 7 ++ app/models/project_media.rb | 85 +++++++++++++++- app/models/relevant_results_item.rb | 16 +++ ...113174153_create_relevant_results_items.rb | 19 ++++ db/schema.rb | 23 ++++- lib/sample_data.rb | 23 +++++ test/models/relevant_results_item_test.rb | 97 +++++++++++++++++++ 8 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 app/models/relevant_results_item.rb create mode 100644 db/migrate/20250113174153_create_relevant_results_items.rb create mode 100644 test/models/relevant_results_item_test.rb diff --git a/app/models/claim_description.rb b/app/models/claim_description.rb index 35be97c6d0..a740453215 100644 --- a/app/models/claim_description.rb +++ b/app/models/claim_description.rb @@ -20,7 +20,7 @@ class ClaimDescription < ApplicationRecord after_update :update_report after_update :reset_item_rating_if_removed after_update :replace_media, unless: proc { |cd| cd.disable_replace_media } - after_update :migrate_claim_and_fact_check_logs, if: proc { |cd| cd.saved_change_to_project_media_id? && !cd.project_media_id.nil? } + after_update :migrate_claim_and_fact_check_logs, :log_relevant_article_results, if: proc { |cd| cd.saved_change_to_project_media_id? && !cd.project_media_id.nil? } # To avoid GraphQL conflict with name `context` alias_attribute :claim_context, :context @@ -120,6 +120,11 @@ def migrate_claim_and_fact_check_logs end end + def log_relevant_article_results + fc = self.fact_check + self.project_media.delay.log_relevant_results(fc.class.name, fc.id, User.current&.id, self.class.actor_session_id) + end + def cant_apply_article_to_item_if_article_is_in_the_trash errors.add(:base, I18n.t(:cant_apply_article_to_item_if_article_is_in_the_trash)) if self.project_media && self.fact_check&.trashed end diff --git a/app/models/explainer_item.rb b/app/models/explainer_item.rb index 0fba5d12d9..7359df17de 100644 --- a/app/models/explainer_item.rb +++ b/app/models/explainer_item.rb @@ -9,6 +9,8 @@ class ExplainerItem < ApplicationRecord validate :same_team validate :cant_apply_article_to_item_if_article_is_in_the_trash + after_create :log_relevant_article_results + def version_metadata(_changes) { explainer_title: self.explainer.title }.to_json end @@ -22,4 +24,9 @@ def same_team def cant_apply_article_to_item_if_article_is_in_the_trash errors.add(:base, I18n.t(:cant_apply_article_to_item_if_article_is_in_the_trash)) if self.explainer&.trashed end + + def log_relevant_article_results + ex = self.explainer + self.project_media.delay.log_relevant_results(ex.class.name, ex.id, User.current&.id, self.class.actor_session_id) + end end diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 8b8c4bfdda..f99ac69827 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -475,21 +475,102 @@ def get_similar_articles search_query ||= self.title results = self.team.search_for_similar_articles(search_query, self) fact_check_ids = results.select{|article| article.is_a?(FactCheck)}.map(&:id) + fc_pm = {} + unless fact_check_ids.blank? + # Get ProjectMedia for FactCheck for RelevantResultsItem logs and should use sort_by to keep existing order + FactCheck.select('fact_checks.id AS fc_id, claim_descriptions.project_media_id AS pm_id') + .where(id: fact_check_ids).joins(:claim_description).each do |raw| + fc_pm[raw.fc_id] = raw.pm_id + end + end explainer_ids = results.select{|article| article.is_a?(Explainer)}.map(&:id) - { fact_check: fact_check_ids, explainer: explainer_ids }.to_json + ex_pm = {} + unless explainer_ids.blank? + # Intiate the ex_pm with nil values as some Explainer not assinged to existing items + default_pm = nil + ex_pm = explainer_ids.each_with_object(default_pm).to_h + # Get ProjectMedia for Explainer for RelevantResultsItem logs and should use sort_by to keep existing order + ExplainerItem.where(explainer_id: explainer_ids).find_each do |raw| + ex_pm[raw.explainer_id] = raw.project_media_id + end + end + { + fact_check: fc_pm.sort_by { |k, _v| fact_check_ids.index(k) }.to_h, + explainer: ex_pm.sort_by { |k, _v| explainer_ids.index(k) }.to_h + }.to_json end if results.blank? # This indicates a cache hit, so we should retrieve the items according to the cached values while maintaining the same sort order. items = JSON.parse(items) - items.each do |klass, ids| + items.each do |klass, data| + ids = data.keys results += klass.camelize.constantize.where(id: ids).sort_by { |result| ids.index(result.id) } end end results end + def log_relevant_results(klass, id, author_id, actor_session_id) + actor_session_id = Digest::MD5.hexdigest("#{actor_session_id}-#{Time.now.to_i}") + article = klass.constantize.find_by_id id + return if article.nil? + data = begin JSON.parse(Rails.cache.read("relevant-items-#{self.id}")) rescue {} end + type = klass.underscore + unless data[type].blank? + user_action = data[type].include?(article.id) ? 'relevant_articles' : 'article_search' + tbi = Bot::Alegre.get_alegre_tbi(self.team_id) + similarity_settings = tbi&.settings&.to_h || {} + # Retrieve the user's selection, which can be either FactCheck or Explainer, + # as this type encompasses the user's choice, and then define the shared field based on this type. + # i.e selected_count either 0/1 + items = data[type] + items.keys.each_with_index do |value, index| + selected_count = (value == article.id).to_i + fields = { + article_id: article.id, + article_type: article.class.name, + matched_media_id: items[value], + selected_count: selected_count, + display_rank: index + 1, + } + self.create_relevant_results_item(user_action, similarity_settings, author_id, actor_session_id, fields) + end + # Retrieve the alternative type (the non-selected type) since all items in this category are marked as non-selected items + # i.e selected_count = 0 + other_type = (['fact_check', 'explainer'] - [type]).first + items = data[other_type] + items.keys.each_with_index do |value, index| + fields = { + article_id: value, + article_type: other_type.camelize, + matched_media_id: items[value], + selected_count: 0, + display_rank: index + 1, + } + self.create_relevant_results_item(user_action, similarity_settings, author_id, actor_session_id, fields) + end + end + Rails.cache.delete("relevant-items-#{self.id}") + end + protected + def create_relevant_results_item(user_action, similarity_settings, author_id, actor_session_id, fields) + rr = RelevantResultsItem.new + rr.team_id = self.team_id + rr.user_id = author_id + rr.relevant_results_render_id = actor_session_id + rr.query_media_parent_id = self.id + rr.query_media_ids = [self.id] + rr.user_action = user_action + rr.similarity_settings = similarity_settings + rr.skip_check_ability = true + fields.each do |k, v| + rr.send("#{k}=", v) if rr.respond_to?("#{k}=") + end + rr.save! + end + def add_extra_elasticsearch_data(ms) analysis = self.analysis analysis_title = analysis['title'].blank? ? nil : analysis['title'] diff --git a/app/models/relevant_results_item.rb b/app/models/relevant_results_item.rb new file mode 100644 index 0000000000..30e8e46bbf --- /dev/null +++ b/app/models/relevant_results_item.rb @@ -0,0 +1,16 @@ +class RelevantResultsItem < ApplicationRecord + belongs_to :article, polymorphic: true + belongs_to :user, optional: true + belongs_to :team, optional: true + + before_validation :set_team_and_user, on: :create + validates_presence_of :team, :user, :query_media_parent_id + validates :user_action, included: { values: %w(relevant_articles article_search) } + + private + + def set_team_and_user + self.team_id ||= Team.current&.id + self.user_id ||= User.current&.id + end +end diff --git a/db/migrate/20250113174153_create_relevant_results_items.rb b/db/migrate/20250113174153_create_relevant_results_items.rb new file mode 100644 index 0000000000..f3a9804e87 --- /dev/null +++ b/db/migrate/20250113174153_create_relevant_results_items.rb @@ -0,0 +1,19 @@ +class CreateRelevantResultsItems < ActiveRecord::Migration[6.1] + def change + create_table :relevant_results_items do |t| + t.references :user + t.references :team + t.string :relevant_results_render_id + t.string :user_action + t.integer :query_media_parent_id + t.integer :query_media_ids, array: true, default: [] + t.jsonb :similarity_settings, default: {} + t.integer :matched_media_id + t.integer :selected_count + t.integer :display_rank + t.references :article, polymorphic: true, null: false + t.timestamps + end + add_index :relevant_results_items, [:article_type, :article_id] + end +end diff --git a/db/schema.rb b/db/schema.rb index 636e74117f..13c3bf461b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_11_23_135242) do +ActiveRecord::Schema.define(version: 2025_01_13_174153) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -620,6 +620,27 @@ t.check_constraint "source_id <> target_id", name: "source_target_must_be_different" end + create_table "relevant_results_items", force: :cascade do |t| + t.bigint "user_id" + t.bigint "team_id" + t.string "relevant_results_render_id" + t.string "user_action" + t.integer "query_media_parent_id" + t.integer "query_media_ids", default: [], array: true + t.jsonb "similarity_settings", default: {} + t.integer "matched_media_id" + t.integer "selected_count" + t.integer "display_rank" + t.string "article_type", null: false + t.bigint "article_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["article_type", "article_id"], name: "index_relevant_results_items_on_article" + t.index ["article_type", "article_id"], name: "index_relevant_results_items_on_article_type_and_article_id" + t.index ["team_id"], name: "index_relevant_results_items_on_team_id" + t.index ["user_id"], name: "index_relevant_results_items_on_user_id" + end + create_table "requests", force: :cascade do |t| t.bigint "feed_id", null: false t.string "request_type", null: false diff --git a/lib/sample_data.rb b/lib/sample_data.rb index ccd7c01650..835e334511 100644 --- a/lib/sample_data.rb +++ b/lib/sample_data.rb @@ -925,6 +925,13 @@ def create_explainer(options = {}) }.merge(options)) end + def create_explainer_item(options = {}) + ExplainerItem.create!({ + explainer: options[:explainer] || create_explainer, + project_media: options[:project_media] || create_project_media + }.merge(options)) + end + def create_feed(options = {}) Feed.create!({ name: random_string, @@ -1202,4 +1209,20 @@ def create_feed_invitation(options = {}) state: :invited }.merge(options)) end + + def create_relevant_results_item(options = {}) + options[:team] = create_team unless options.has_key?(:team) + options[:user] = create_user unless options.has_key?(:user) + options[:article] = create_explainer unless options.has_key?(:article) + options[:user_action] ||= 'relevant_articles' + options[:query_media_parent_id] = create_project_media(team: options[:team]).id unless options.has_key?(:query_media_parent_id) + options[:relevant_results_render_id] ||= Digest::MD5.hexdigest("#{RequestStore[:actor_session_id]}-#{Time.now.to_i}") + rr = RelevantResultsItem.new + options.each do |k, v| + rr.send("#{k}=", v) if rr.respond_to?("#{k}=") + end + rr.skip_check_ability = true + rr.save! + rr.reload + end end diff --git a/test/models/relevant_results_item_test.rb b/test/models/relevant_results_item_test.rb new file mode 100644 index 0000000000..c287b41839 --- /dev/null +++ b/test/models/relevant_results_item_test.rb @@ -0,0 +1,97 @@ +require_relative '../test_helper' + +class RelevantResultsItemTest < ActiveSupport::TestCase + test "should create relevant results item" do + assert_difference 'RelevantResultsItem.count' do + create_relevant_results_item + end + end + + test "should set team and user" do + t = create_team + u = create_user + create_team_user team: t, user: u, role: 'admin' + ex = create_explainer + with_current_user_and_team(u, t) do + rr = create_relevant_results_item team: nil, user: nil, article: ex + assert_equal t.id, rr.team_id + assert_equal u.id, rr.user_id + end + end + + test "should validate user action field" do + ex = create_explainer + assert_no_difference 'RelevantResultsItem.count' do + assert_raises ActiveRecord::RecordInvalid do + create_relevant_results_item article: ex, user_action: random_string + end + assert_raises ActiveRecord::RecordInvalid do + create_relevant_results_item article: ex, query_media_parent_id: nil + end + end + end + + test "should record user selection for relevant articles" do + RequestStore.store[:skip_cached_field_update] = false + setup_elasticsearch + t = create_team + u = create_user + create_team_user team: t, user: u, role: 'admin' + pm1 = create_project_media quote: 'Foo Bar', team: t + pm2 = create_project_media quote: 'Foo Bar Test', team: t + pm3 = create_project_media quote: 'Foo Bar Test Testing', team: t + ex1 = create_explainer language: 'en', team: t, title: 'Foo Bar' + ex2 = create_explainer language: 'en', team: t, title: 'Foo Bar Test' + ex3 = create_explainer language: 'en', team: t, title: 'Foo Bar Test Testing' + pm1.explainers << ex1 + pm2.explainers << ex2 + pm3.explainers << ex3 + ex_ids = [ex1.id, ex2.id, ex3.id] + Bot::Smooch.stubs(:search_for_explainers).returns(Explainer.where(id: ex_ids)) + fact_checks = [] + [pm1, pm2, pm3].each do |pm| + cd = create_claim_description description: pm.title, project_media: pm + fc = create_fact_check claim_description: cd, title: pm.title + fact_checks << fc.id + end + [pm1, pm2].each { |pm| publish_report(pm) } + sleep 1 + fact_checks.delete(pm1.fact_check_id) + expected_result = fact_checks.concat([ex2.id, ex3.id]).sort + assert_equal expected_result, pm1.get_similar_articles.map(&:id).sort + with_current_user_and_team(u, t) do + # Assign explainer to item + assert_not_nil Rails.cache.read("relevant-items-#{pm1.id}") + # Select an Explainer + actor_session_id = random_string + RequestStore.stubs(:[]).with(:actor_session_id).returns(actor_session_id) + now = Time.now + Time.stubs(:now).returns(now) + create_explainer_item explainer: ex2, project_media: pm1 + assert_equal 2, RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'Explainer').count + assert_equal 2, RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'FactCheck').count + assert_equal 4, RelevantResultsItem.where(query_media_parent_id: pm1.id, relevant_results_render_id: Digest::MD5.hexdigest("#{actor_session_id}-#{now.to_i}")).count + Time.unstub(:now) + RequestStore.unstub(:[]) + assert_nil Rails.cache.read("relevant-items-#{pm1.id}") + # Select a FactCheck + log_time = Time.now + pm1.get_similar_articles.map(&:id).sort + assert_not_nil Rails.cache.read("relevant-items-#{pm1.id}") + cd = ClaimDescription.where(project_media_id: pm1.id).last + cd.project_media = create_project_media team: t + cd.save! + cd = ClaimDescription.where.not(project_media_id: pm1.id).last + cd.project_media_id = pm1.id + cd.save! + assert_equal 2, RelevantResultsItem.where('created_at > ?', log_time).where(query_media_parent_id: pm1.id, article_type: 'Explainer').count + assert_equal 2, RelevantResultsItem.where('created_at > ?', log_time).where(query_media_parent_id: pm1.id, article_type: 'FactCheck').count + # Verify selected item + fc = cd.fact_check + selected_item = RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'FactCheck', article_id: fc.id).last + assert_equal fc, selected_item.article + assert_nil Rails.cache.read("relevant-items-#{pm1.id}") + end + Bot::Smooch.unstub(:search_for_explainers) + end +end From c529963a238fa3d67c094521c9b43e57b3988486 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Tue, 21 Jan 2025 21:08:39 +0200 Subject: [PATCH 33/52] CV2-5668: fix user_action and selected_count values (#2185) --- app/models/project_media.rb | 6 +++--- test/models/relevant_results_item_test.rb | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/models/project_media.rb b/app/models/project_media.rb index f99ac69827..ffd7936ffe 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -517,7 +517,7 @@ def log_relevant_results(klass, id, author_id, actor_session_id) data = begin JSON.parse(Rails.cache.read("relevant-items-#{self.id}")) rescue {} end type = klass.underscore unless data[type].blank? - user_action = data[type].include?(article.id) ? 'relevant_articles' : 'article_search' + user_action = data[type].keys.map(&:to_i).include?(article.id) ? 'relevant_articles' : 'article_search' tbi = Bot::Alegre.get_alegre_tbi(self.team_id) similarity_settings = tbi&.settings&.to_h || {} # Retrieve the user's selection, which can be either FactCheck or Explainer, @@ -525,9 +525,9 @@ def log_relevant_results(klass, id, author_id, actor_session_id) # i.e selected_count either 0/1 items = data[type] items.keys.each_with_index do |value, index| - selected_count = (value == article.id).to_i + selected_count = (value.to_i == article.id).to_i fields = { - article_id: article.id, + article_id: value, article_type: article.class.name, matched_media_id: items[value], selected_count: selected_count, diff --git a/test/models/relevant_results_item_test.rb b/test/models/relevant_results_item_test.rb index c287b41839..59c6052328 100644 --- a/test/models/relevant_results_item_test.rb +++ b/test/models/relevant_results_item_test.rb @@ -71,6 +71,11 @@ class RelevantResultsItemTest < ActiveSupport::TestCase assert_equal 2, RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'Explainer').count assert_equal 2, RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'FactCheck').count assert_equal 4, RelevantResultsItem.where(query_media_parent_id: pm1.id, relevant_results_render_id: Digest::MD5.hexdigest("#{actor_session_id}-#{now.to_i}")).count + # Verify user action value + assert_equal ['relevant_articles'], RelevantResultsItem.where(query_media_parent_id: pm1.id).map(&:user_action).uniq + # Verified selected_count value + assert_equal [1], RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'Explainer', article_id: ex2.id).map(&:selected_count) + assert_equal [0], RelevantResultsItem.where(query_media_parent_id: pm1.id, article_type: 'Explainer').where.not(article_id: ex2.id).map(&:selected_count).uniq Time.unstub(:now) RequestStore.unstub(:[]) assert_nil Rails.cache.read("relevant-items-#{pm1.id}") From 29488b50e8814c61cb0f0fd4dcfbe52eb8bafc84 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 21 Jan 2025 18:08:54 -0300 Subject: [PATCH 34/52] Log and notify Sentry when relevant articles are not retrieved. (#2186) * Log and notify Sentry when relevant articles are not retrieved. This PR implements two things: * Case 1: Info-level logging when no relevant articles are returned. * Case 2: Notify Sentry when there is an error retrieving relevant articles. In that case: * Catch the error and return an empty list, so it doesn't crash the frontend, which can then retrieve to recent articles. * Log a warning. * Notify Sentry. The Sentry message is static and implements a custom exception class, so similar notifications can be grouped together automatically, but includes all information we need in order to debug. Reference: CV2-5932. --- app/models/team.rb | 13 ++++++++++++- test/models/team_2_test.rb | 7 +++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/team.rb b/app/models/team.rb index f5519b8eea..56e738d620 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -1,4 +1,6 @@ class Team < ApplicationRecord + class RelevantArticlesError < StandardError; end + # These two callbacks must be in the top after_create :create_team_partition before_destroy :delete_created_bots, :remove_is_default_project_flag @@ -563,10 +565,14 @@ def filter_by_keywords(query, filters, type = 'FactCheck') query.where(Arel.sql("#{tsvector} @@ #{tsquery}")) end + def similar_articles_search_limit(pm = nil) + pm.nil? ? CheckConfig.get('most_relevant_team_limit', 3, :integer) : CheckConfig.get('most_relevant_item_limit', 10, :integer) + end + def search_for_similar_articles(query, pm = nil) # query: expected to be text # pm: to request a most relevant to specific item and also include both FactCheck & Explainer - limit = pm.nil? ? CheckConfig.get('most_relevant_team_limit', 3, :integer) : CheckConfig.get('most_relevant_item_limit', 10, :integer) + limit = self.similar_articles_search_limit(pm) threads = [] fc_items = [] ex_items = [] @@ -593,7 +599,12 @@ def search_for_similar_articles(query, pm = nil) items = fc_items # Get Explainers if no fact-check returned or get similar_articles for a ProjectMedia items += ex_items if items.blank? || !pm.nil? + Rails.logger.info("Relevant articles found for team slug #{self.slug}, project media with ID #{pm&.id} and query #{query}: #{items.map(&:graphql_id)}") items + rescue StandardError => e + Rails.logger.warn("Error when trying to retrieve relevant articles for team slug #{self.slug}, project media with ID #{pm&.id} and query #{query}.") + CheckSentry.notify(RelevantArticlesError.new('Error when trying to retrieve relevant articles'), team_slug: self.slug, project_media_id: pm&.id, query: query, exception_message: e.message, exception: e) + [] end def get_shorten_outgoing_urls diff --git a/test/models/team_2_test.rb b/test/models/team_2_test.rb index 82a0f237f5..4c1f55a369 100644 --- a/test/models/team_2_test.rb +++ b/test/models/team_2_test.rb @@ -1583,4 +1583,11 @@ def setup end Bot::Smooch.unstub(:search_for_explainers) end + + test "should notify Sentry if it fails to retrieve relevant articles" do + Bot::Smooch.stubs(:search_for_similar_published_fact_checks_no_cache).raises(StandardError) + CheckSentry.expects(:notify).once + t = create_team + assert_equal [], t.search_for_similar_articles('Test') + end end From 433c998d2ef20696401f7623a4a1cf777a009072 Mon Sep 17 00:00:00 2001 From: Daniele Valverde <34126648+danielevalverde@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:41:02 -0300 Subject: [PATCH 35/52] set user when creating relationship (#2188) Updated the confirmed_relationship method to include user when creating confirmed relationships in the seed script. Updated the suggested_relationship method to include user when creating suggested relationships in the seed script. References: CV2-5785, CV2-5510 --- db/seeds.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/seeds.rb b/db/seeds.rb index 8f671dfc08..c45d8514da 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -529,11 +529,11 @@ def feed_invitation(feed, invited_user) end def confirmed_relationship(parent, children) - [children].flatten.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type) } + [children].flatten.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.confirmed_type, user_id: child.user_id) } end def suggested_relationship(parent, children) - children.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.suggested_type)} + children.each { |child| Relationship.create!(source_id: parent.id, target_id: child.id, relationship_type: Relationship.suggested_type, user_id: child.user_id)} end def teams_project_medias From 0454728a443a84e9999cb61450c4543a6719b697 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 22 Jan 2025 18:04:50 +0200 Subject: [PATCH 36/52] CV2-5935 indexing issues on lists (#2187) * CV2-5935: update related items status after change claim description item * CV2-5935: apply PR comment --- app/models/fact_check.rb | 16 +++++++++++----- test/models/claim_description_test.rb | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/app/models/fact_check.rb b/app/models/fact_check.rb index 947f011fa5..2ee9faaa9d 100644 --- a/app/models/fact_check.rb +++ b/app/models/fact_check.rb @@ -42,11 +42,17 @@ def team def update_item_status pm = self.project_media - s = pm&.last_status_obj - if !s.nil? && s.status != self.rating - s.skip_check_ability = true - s.status = self.rating - s.save! + unless pm.nil? + s = pm.last_status_obj + if !s.nil? && s.status != self.rating + s.skip_check_ability = true + s.status = self.rating + s.save! + end + # update related items status + Relationship.confirmed.where(source_id: pm.id).find_each do |r| + Relationship.delay_for(2.seconds, { queue: 'smooch'}).inherit_status_and_send_report(r.id) + end end end diff --git a/test/models/claim_description_test.rb b/test/models/claim_description_test.rb index 746d1207f2..2cb5305f08 100644 --- a/test/models/claim_description_test.rb +++ b/test/models/claim_description_test.rb @@ -230,4 +230,26 @@ def setup end assert cd.project_media.media.is_a?(Blank) end + + test "should update status for main and related items when set project_media" do + create_verification_status_stuff + RequestStore.store[:skip_cached_field_update] = false + t = create_team + smooch_bot = create_smooch_bot + create_team_bot_installation team_id: t.id, user_id: smooch_bot.id + pm = create_project_media team: t + pm_child = create_project_media team: t + create_relationship source_id: pm.id, target_id: pm_child.id, relationship_type: Relationship.confirmed_type + assert_equal 'undetermined', pm.reload.status + assert_equal 'undetermined', pm_child.reload.status + # Create fact-check with verified status + cd = create_claim_description team_id: t.id, project_media: nil + fc = create_fact_check claim_description: cd, rating: 'verified' + Sidekiq::Testing.inline! do + cd.project_media = pm + cd.save! + assert_equal 'verified', pm.reload.status + assert_equal 'verified', pm_child.reload.status + end + end end From bde17dcb6fff05ce14cf0aac2f6fe8e436acb181 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 27 Jan 2025 09:34:39 +0200 Subject: [PATCH 37/52] CV2-4985: Export dashboard data as CSV (#2190) * CV2-4985: export dashboard as CSV * CV2-4985: add export to articles dashboard * CV2-4985: fix export * CV2-4985: add missing tests --- app/models/concerns/team_private.rb | 67 +++++++++++++++++++++++++++++ app/models/team.rb | 26 +++++++++++ lib/list_export.rb | 6 ++- test/lib/list_export_test.rb | 20 +++++++++ test/models/team_test.rb | 20 +++++++++ 5 files changed, 138 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/team_private.rb b/app/models/concerns/team_private.rb index 61c817076b..abfed1f29b 100644 --- a/app/models/concerns/team_private.rb +++ b/app/models/concerns/team_private.rb @@ -137,4 +137,71 @@ def empty_data_structure data_structure["Org"] = self.name [data_structure] end + + def get_dashboard_export_headers(ts, dashboard_type) + # Get dashboard headers for both types (articles_dashboard & tipline_dashboard) in format { key: value } + # key(string): header label + # Value(Hash): { method_name: 'callback that should apply to the method output'} + # In some cases, the value may be empty ({}) as some methods will populate more than one row. + + # Common header between articles_dashboard and tipline_dashboard + header = { + 'Articles Sent': { number_of_articles_sent: 'to_i' }, + 'Matched Results (Fact-Checks)': { number_of_matched_results_by_article_type: 'values' }, + 'Matched Results (Explainers)': {}, + } + # Hash to include top items as the header label depend on top_items size + top_items = {} + # tipline_dashboard columns + if dashboard_type == 'tipline_dashboard' + header.merge!({ + 'Conversations': { number_of_conversations: 'to_i' }, + 'Messages': { number_of_messages: 'to_i' }, + 'Conversations (Positive)': { number_of_search_results_by_feedback_type: 'values' }, + 'Conversations (Negative)': {}, 'Conversations (No Response)': {}, + 'Avg. Response Time': { average_response_time: 'to_i' }, + 'Users (Total)': { number_of_total_users: 'to_i' }, + 'Users (Unique)': { number_of_unique_users: 'to_i' }, + 'Users (Returning)': { number_of_returning_users: 'to_i' }, + 'Subscribers': { number_of_subscribers: 'to_i' }, + 'Subscribers (New)': { number_of_new_subscribers: 'to_i' }, + 'Newsletters (Sent)': { number_of_newsletters_sent: 'to_i' }, + 'Newsletters (Delivered)': { number_of_newsletters_delivered: 'to_i' }, + 'Media Received (Text)': { number_of_media_received_by_media_type: 'values' }, + 'Media Received (Link)': {}, 'Media Received (Audio)': {}, 'Media Received (Image)': {}, 'Media Received (Video)': {}, + }) + top_items = { top_media_tags: 'Top tag', top_requested_media_clusters: 'Top Requested' } + else + # article_dashboard columns + header.merge!({ + 'Published Fact-Checks': { number_of_published_fact_checks: 'to_i' }, + 'Explainers Created': { number_of_explainers_created: 'to_i' }, + 'Fact-Checks Created': { number_of_fact_checks_created: 'to_i' }, + }) + rates = ts.send('number_of_fact_checks_by_rating').keys + unless rates.blank? + # Get the first element to fill the label and callback methods as other element will calling with empty callbacks + f_rate = rates.delete_at(0) + header.merge!({ "Claim & Fact-Checks (#{f_rate})": { number_of_fact_checks_by_rating: 'values' }}) + rates.each{ |rate| header.merge!({"Claim & Fact-Checks (#{rate})": {}}) } + end + top_items = { top_articles_tags: 'Top Article Tags', top_articles_sent: 'Top Fact-Checks Sent' } + end + unless top_items.blank? + top_callback = proc { |output| output.collect{|item| "#{item[:label]} (#{item[:value]})"} } + # Append Top tags/requested header based on result count + top_items.each do |method, prefix| + col_numbers = ts.send(method).size + if col_numbers > 0 + # Add a first one with method callback + header.merge!({"#{prefix} (1)": { "#{method}": top_callback } }) + (col_numbers - 1).times do |i| + # Append other columns with empty method + header.merge!({"#{prefix} (#{i+2})": {} }) + end + end + end + end + header + end end diff --git a/app/models/team.rb b/app/models/team.rb index 56e738d620..664873ecbb 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -611,6 +611,32 @@ def get_shorten_outgoing_urls self.settings.to_h.with_indifferent_access[:shorten_outgoing_urls] || self.tipline_newsletters.where(content_type: 'rss', enabled: true).exists? end + def get_dashboard_exported_data(filters, dashboard_type) + filters = filters.with_indifferent_access + ts = TeamStatistics.new(self, filters[:period], filters[:language], filters[:platform]) + headers = get_dashboard_export_headers(ts, dashboard_type) + data = [] + # Add header labels + data << headers.keys + header_methods = headers.values.delete_if{|v| v.blank?} + # Merging multiple hashes as single hash + header_methods = Hash[*header_methods.map{|v|v.to_a}.flatten] + raw = [] + header_methods.each do |method, type| + unless type.blank? + output = ts.send(method) if ts.respond_to?(method) + if type.is_a?(Proc) + output = type.call(output) + else + output = output.send(type) + end + raw << output + end + end + data << raw.flatten + data + end + # private # # Please add private methods to app/models/concerns/team_private.rb diff --git a/lib/list_export.rb b/lib/list_export.rb index 533f74771f..69f9245027 100644 --- a/lib/list_export.rb +++ b/lib/list_export.rb @@ -1,5 +1,5 @@ class ListExport - TYPES = [:media, :feed, :fact_check, :explainer] + TYPES = [:media, :feed, :fact_check, :explainer, :articles_dashboard, :tipline_dashboard ] def initialize(type, query, team_id) @type = type @@ -21,6 +21,8 @@ def number_of_rows @team.filtered_fact_checks(@parsed_query).count when :explainer @team.filtered_explainers(@parsed_query).count + when :articles_dashboard, :tipline_dashboard + 1 # Always maintain one row for dashboard data, but use different columns for export. end end @@ -62,6 +64,8 @@ def export_data FactCheck.get_exported_data(@parsed_query, @team) when :explainer Explainer.get_exported_data(@parsed_query, @team) + when :articles_dashboard, :tipline_dashboard + @team.get_dashboard_exported_data(@parsed_query, @type) end end end diff --git a/test/lib/list_export_test.rb b/test/lib/list_export_test.rb index 15551ba12d..3e26dd208c 100644 --- a/test/lib/list_export_test.rb +++ b/test/lib/list_export_test.rb @@ -118,4 +118,24 @@ def teardown assert_equal 2, csv_content.size assert_equal 2, export.number_of_rows end + + test "should export dashboard CSV" do + t = create_team + # tipline_dashboard + export = ListExport.new(:tipline_dashboard, { period: "past_week", platform: "whatsapp", language: "en" }.to_json, t.id) + csv_url = export.generate_csv_and_send_email(create_user) + response = Net::HTTP.get_response(URI(csv_url)) + assert_equal 200, response.code.to_i + csv_content = CSV.parse(response.body, headers: true) + assert_equal 1, csv_content.size + assert_equal 1, export.number_of_rows + # articles_dashboard + export = ListExport.new(:articles_dashboard, { period: "past_week", platform: "whatsapp", language: "en" }.to_json, t.id) + csv_url = export.generate_csv_and_send_email(create_user) + response = Net::HTTP.get_response(URI(csv_url)) + assert_equal 200, response.code.to_i + csv_content = CSV.parse(response.body, headers: true) + assert_equal 1, csv_content.size + assert_equal 1, export.number_of_rows + end end diff --git a/test/models/team_test.rb b/test/models/team_test.rb index c1b8509df9..fecd3615ab 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -1315,4 +1315,24 @@ def setup tn.destroy! assert !t.get_shorten_outgoing_urls end + + test "should get dashboard data" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + team = create_team + pm = create_project_media team: team, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }, disable_es_callbacks: false + pm2 = create_project_media team: team, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }, disable_es_callbacks: false + create_tipline_request team: team.id, associated: pm, disable_es_callbacks: false + create_tipline_request team: team.id, associated: pm2, disable_es_callbacks: false + sleep 1 + filters = { period: "past_week", platform: "whatsapp", language: "en" } + data = team.get_dashboard_exported_data(filters, 'tipline_dashboard') + assert_not_nil data + assert_equal data[0].length, data[1].length + cd = create_claim_description project_media: pm + fc = create_fact_check claim_description: cd, rating: 'in_progress' + data = team.get_dashboard_exported_data(filters, 'articles_dashboard') + assert_not_nil data + assert_equal data[0].length, data[1].length + end end From f8ad533487b8db85bc5fce4beedec9c75cf71ad5 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:22:59 -0300 Subject: [PATCH 38/52] Adding new arguments to `TeamType.bot_preview` GraphQL field. (#2183) We already have the `TeamType.bot_query` GraphQL field implemented, and it takes only one argument (the text query). It currently uses the settings from the workspace and/or Smooch Bot installation. We want to make sure that it can receive additional optional arguments that override the stored settings. Reference: CV2-5919. --- .rubocop.yml | 4 +- app/graph/types/team_type.rb | 31 ++++++-- app/lib/tipline_search_result.rb | 31 ++++++-- app/models/concerns/smooch_search.rb | 33 ++++++--- app/models/explainer.rb | 10 +-- app/models/fact_check.rb | 5 +- app/models/team.rb | 6 +- lib/relay.idl | 2 +- public/relay.json | 72 +++++++++++++++++++ .../controllers/graphql_controller_11_test.rb | 6 +- 10 files changed, 163 insertions(+), 37 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 78aae1b289..57dbd5d8c7 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -243,10 +243,10 @@ Metrics/ModuleLength: Max: 250 Metrics/ParameterLists: - Description: 'Avoid parameter lists longer than 9 parameters.' + Description: 'Avoid parameter lists longer than 10 parameters.' StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#too-many-params' Enabled: true - Max: 9 + Max: 10 Metrics/PerceivedComplexity: Description: >- diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 01e56ba6d0..70e21b8c7d 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -402,12 +402,29 @@ def statistics(period:, language: nil, platform: nil) field :bot_query, [TiplineSearchResultType], null: true do argument :search_text, GraphQL::Types::String, required: true - end - - def bot_query(search_text:) - return nil unless User.current&.is_admin - - results = object.search_for_similar_articles(search_text) - results.map(&:as_tipline_search_result) + argument :threshold, GraphQL::Types::Float, required: false + argument :max_number_of_words, GraphQL::Types::Int, required: false + argument :enable_language_detection, GraphQL::Types::Boolean, required: false + argument :should_restrict_by_language, GraphQL::Types::Boolean, required: false + argument :enable_link_shortening, GraphQL::Types::Boolean, required: false + argument :utm_code, GraphQL::Types::String, required: false + end + + def bot_query(search_text:, threshold: nil, max_number_of_words: nil, enable_language_detection: nil, should_restrict_by_language: nil, enable_link_shortening: nil, utm_code: nil) + return nil unless User.current&.is_admin # Feature flag + + settings = { + threshold: threshold, + max_number_of_words: max_number_of_words, + enable_language_detection: enable_language_detection, + should_restrict_by_language: should_restrict_by_language, + enable_link_shortening: enable_link_shortening, + utm_code: utm_code + }.with_indifferent_access + + language = (enable_language_detection ? Bot::Smooch.get_language({ 'text' => search_text }, object.default_language) : object.default_language) + + results = object.search_for_similar_articles(search_text, nil, language, settings) + results.collect{ |result| result.as_tipline_search_result(settings) }.select{ |result| result.should_send_in_language?(language, should_restrict_by_language) } end end diff --git a/app/lib/tipline_search_result.rb b/app/lib/tipline_search_result.rb index 5983ec01cf..8aad9298b4 100644 --- a/app/lib/tipline_search_result.rb +++ b/app/lib/tipline_search_result.rb @@ -1,7 +1,8 @@ class TiplineSearchResult - attr_accessor :id, :team, :title, :body, :image_url, :language, :url, :type, :format + attr_accessor :id, :team, :image_url, :language, :type, :format, :link_settings + attr_writer :title, :body, :url - def initialize(id:, team:, title:, body:, image_url:, language:, url:, type:, format:) + def initialize(id:, team:, title:, body:, image_url:, language:, url:, type:, format:, link_settings: nil) self.id = id self.team = team self.title = title @@ -11,12 +12,14 @@ def initialize(id:, team:, title:, body:, image_url:, language:, url:, type:, fo self.url = url self.type = type # :explainer or :fact_check self.format = format # :text or :image + self.link_settings = link_settings end - def should_send_in_language?(language) + def should_send_in_language?(language, force_restrict_by_language = nil) return true if self.team.get_languages.to_a.size < 2 tbi = TeamBotInstallation.where(team_id: self.team.id, user: BotUser.alegre_user).last should_send_report_in_different_language = !tbi&.alegre_settings&.dig('single_language_fact_checks_enabled') + should_send_report_in_different_language = !force_restrict_by_language unless force_restrict_by_language.nil? self.language == language || should_send_report_in_different_language end @@ -47,13 +50,29 @@ def text(language = nil, hide_body = false) text << "*#{self.title.strip}*" unless self.title.blank? text << self.body.to_s unless hide_body text << self.url unless self.url.blank? - text = text.collect do |part| - self.team.get_shorten_outgoing_urls ? UrlRewriter.shorten_and_utmize_urls(part, self.team.get_outgoing_urls_utm_code) : part - end unless language.nil? footer = self.footer(language) text << footer if !footer.blank? && self.team_report_setting_value('use_signature', language) end text.join("\n\n") end + + def title + self.formatted_value(@title) + end + + def url + self.formatted_value(@url) + end + + def body + self.formatted_value(@body) + end + + def formatted_value(text) + link_settings = self.link_settings.to_h.with_indifferent_access + enable_link_shortening = link_settings[:enable_link_shortening].nil? ? self.team.get_shorten_outgoing_urls : link_settings[:enable_link_shortening] + utm_code = link_settings[:utm_code].nil? ? self.team.get_outgoing_urls_utm_code : link_settings[:utm_code] + enable_link_shortening ? UrlRewriter.shorten_and_utmize_urls(text, utm_code) : text + end end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index c82bf71313..3e0b1619f5 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -160,9 +160,24 @@ def search_for_similar_published_fact_checks(type, query, team_ids, limit, after end end + def get_setting_value_or_default(setting_name, custom_settings, team_ids = nil) + custom_value = custom_settings.to_h.with_indifferent_access[setting_name] + return custom_value unless custom_value.nil? + case setting_name.to_sym + when :threshold + self.get_text_similarity_threshold + when :max_number_of_words + self.max_number_of_words_for_keyword_search + when :should_restrict_by_language + self.should_restrict_by_language?(team_ids) + else + nil + end + end + # "type" is text, video, audio or image # "query" is either a piece of text of a media URL - def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil, published_only = true) + def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, limit, after = nil, feed_id = nil, language = nil, published_only = true, settings = nil) results = [] pm = nil pm = ProjectMedia.new(team_id: team_ids[0]) if team_ids.size == 1 # We'll use the settings of a team instead of global settings when there is only one team @@ -180,10 +195,10 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, lim text = self.remove_meaningless_phrases(text) words = text.split(/\s+/) Rails.logger.info "[Smooch Bot] Search query (text): #{text}" - if Bot::Alegre.get_number_of_words(text) <= self.max_number_of_words_for_keyword_search - results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language, published_only) + if Bot::Alegre.get_number_of_words(text) <= self.get_setting_value_or_default('max_number_of_words', settings) + results = self.search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id, language, published_only, settings) else - alegre_results = Bot::Alegre.get_merged_similar_items(pm, [{ value: self.get_text_similarity_threshold }], Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, text, team_ids) + alegre_results = Bot::Alegre.get_merged_similar_items(pm, [{ value: self.get_setting_value_or_default('threshold', settings) }], Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, text, team_ids) results = self.parse_search_results_from_alegre(alegre_results, limit, published_only, after, feed_id, team_ids) Rails.logger.info "[Smooch Bot] Text similarity search got #{results.count} results while looking for '#{text}' after date #{after.inspect} for teams #{team_ids}" end @@ -247,11 +262,11 @@ def should_restrict_by_language?(team_ids) !!tbi&.alegre_settings&.dig('single_language_fact_checks_enabled') end - def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id = nil, language = nil, published_only = true) + def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, limit, feed_id = nil, language = nil, published_only = true, settings = nil) types = CheckSearch::MEDIA_TYPES.clone.push('blank') search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content) filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: limit, show: types } - filters.merge!({ fc_language: [language] }) if !language.blank? && should_restrict_by_language?(team_ids) + filters.merge!({ fc_language: [language] }) if !language.blank? && self.get_setting_value_or_default(:should_restrict_by_language, settings, team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed if feed_id.blank? filters.merge!({ report_status: ['published'] }) if published_only @@ -310,16 +325,16 @@ def ask_for_feedback_when_all_search_results_are_received(app_id, language, work end end - def search_for_explainers(uid, query, team_id, limit, language = nil) + def search_for_explainers(uid, query, team_id, limit, language = nil, settings = nil) results = nil begin text = ::Bot::Smooch.extract_claim(query) if Bot::Alegre.get_number_of_words(text) == 1 results = Explainer.where(team_id: team_id).where('description ILIKE ? OR title ILIKE ?', "%#{text}%", "%#{text}%") - results = results.where(language: language) if !language.nil? && should_restrict_by_language?([team_id]) + results = results.where(language: language) if !language.nil? && self.get_setting_value_or_default(:should_restrict_by_language, settings, [team_id]) results = results.order('updated_at DESC') else - results = Explainer.search_by_similarity(text, language, team_id, limit) + results = Explainer.search_by_similarity(text, language, team_id, limit, settings.to_h.with_indifferent_access[:threshold]) end rescue StandardError => e self.handle_search_error(uid, e, language) unless uid.blank? diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 96e81be933..46f7e69772 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -23,7 +23,7 @@ def send_to_alegre # Let's not use the same callbacks from article.rb end - def as_tipline_search_result + def as_tipline_search_result(settings = nil) TiplineSearchResult.new( id: self.id, team: self.team, @@ -33,7 +33,8 @@ def as_tipline_search_result language: self.language, url: self.url, type: :explainer, - format: :text + format: :text, + link_settings: settings ) end @@ -103,8 +104,9 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) end end - def self.search_by_similarity(text, language, team_id, limit) + def self.search_by_similarity(text, language, team_id, limit, custom_threshold = nil) models_thresholds = Explainer.get_alegre_models_and_thresholds(team_id) + models_thresholds.each { |model, _threshold| models_thresholds[model] = custom_threshold } unless custom_threshold.blank? context = { type: 'explainer', team_id: team_id @@ -116,7 +118,7 @@ def self.search_by_similarity(text, language, team_id, limit) per_model_threshold: models_thresholds, context: context } - response = Bot::Alegre.query_sync_with_params(params, "text") + response = Bot::Alegre.query_sync_with_params(params, 'text') results = response['result'].to_a.sort_by{ |result| [result['model'] != Bot::Alegre::ELASTICSEARCH_MODEL ? 1 : 0, result['_score']] }.reverse explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(limit) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) diff --git a/app/models/fact_check.rb b/app/models/fact_check.rb index 2ee9faaa9d..7ca6aebd67 100644 --- a/app/models/fact_check.rb +++ b/app/models/fact_check.rb @@ -69,7 +69,7 @@ def clean_fact_check_tags self.tags = clean_tags(self.tags) end - def as_tipline_search_result + def as_tipline_search_result(settings = nil) TiplineSearchResult.new( id: self.id, team: self.team, @@ -79,7 +79,8 @@ def as_tipline_search_result url: self.url, image_url: nil, type: :fact_check, - format: :text + format: :text, + link_settings: settings ) end diff --git a/app/models/team.rb b/app/models/team.rb index 664873ecbb..e202256195 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -569,7 +569,7 @@ def similar_articles_search_limit(pm = nil) pm.nil? ? CheckConfig.get('most_relevant_team_limit', 3, :integer) : CheckConfig.get('most_relevant_item_limit', 10, :integer) end - def search_for_similar_articles(query, pm = nil) + def search_for_similar_articles(query, pm = nil, language = nil, settings = nil) # query: expected to be text # pm: to request a most relevant to specific item and also include both FactCheck & Explainer limit = self.similar_articles_search_limit(pm) @@ -577,7 +577,7 @@ def search_for_similar_articles(query, pm = nil) fc_items = [] ex_items = [] threads << Thread.new { - result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit, nil, nil, nil, false).map(&:id) + result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit, nil, nil, language, false, settings).map(&:id) unless result_ids.blank? fc_items = FactCheck.joins(claim_description: :project_media).where('project_medias.id': result_ids) if pm.nil? @@ -591,7 +591,7 @@ def search_for_similar_articles(query, pm = nil) end } threads << Thread.new { - ex_items = Bot::Smooch.search_for_explainers(nil, query, self.id, limit).distinct + ex_items = Bot::Smooch.search_for_explainers(nil, query, self.id, limit, language, settings).distinct # Exclude the ones already applied to a target item ex_items = ex_items.where.not(id: pm.explainer_ids) unless pm&.explainer_ids.blank? } diff --git a/lib/relay.idl b/lib/relay.idl index d96afad880..203a4014fa 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13216,7 +13216,7 @@ type Team implements Node { articles_count(article_type: String, imported: Boolean, language: [String], publisher_ids: [Int], rating: [String], report_status: [String], standalone: Boolean, tags: [String], target_id: Int, text: String, trashed: Boolean = false, updated_at: String, user_ids: [Int]): Int available_newsletter_header_types: JsonStringType avatar: String - bot_query(searchText: String!): [TiplineSearchResult!] + bot_query(enableLanguageDetection: Boolean, enableLinkShortening: Boolean, maxNumberOfWords: Int, searchText: String!, shouldRestrictByLanguage: Boolean, threshold: Float, utmCode: String): [TiplineSearchResult!] check_search_spam: CheckSearch check_search_trash: CheckSearch check_search_unconfirmed: CheckSearch diff --git a/public/relay.json b/public/relay.json index 338e710c74..6476abeec0 100644 --- a/public/relay.json +++ b/public/relay.json @@ -69658,6 +69658,78 @@ "defaultValue": null, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "threshold", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "maxNumberOfWords", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "enableLanguageDetection", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "shouldRestrictByLanguage", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "enableLinkShortening", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "utmCode", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null } ], "type": { diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 402f873882..c0ad9877a6 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -302,7 +302,7 @@ def teardown assert_equal 2, JSON.parse(@response.body).dig('data', 'team', 'tipline_requests', 'edges').size end - test "super admin user should receive the 3 matching FactChecks or Explainers based on search_text" do + test "super admin user should receive 3 matching fact-checks or explainers based on search text and arguments sent to bot query" do t = create_team # Create a super admin user super_admin = create_user(is_admin: true) @@ -333,11 +333,11 @@ def teardown pm3.explainers << ex5 pm3.explainers << ex6 - # Perform the GraphQL query with searchText "123" + # Perform the GraphQL query with searchText "Foo" query = <<~GRAPHQL query { team(slug: "#{t.slug}") { - bot_query(searchText: "Foo") { + bot_query(searchText: "Foo", threshold: 0.75, maxNumberOfWords: 2, enableLanguageDetection: false, shouldRestrictByLanguage: true, enableLinkShortening: true, utmCode: "test") { title type } From 385e4e083c51abe300b9420cfae544e9109bc28b Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:34:44 -0300 Subject: [PATCH 39/52] Update CODEOWNERS (#2193) Updating CODEOWNERS file. - Global owners: @caiosba, @vasconsaurus, @jayjay-w, @melsawy - Specific owners: - `.github/`: @dmou - `production/`: @dmou - `Dockerfile`: @dmou - `docker*`: @dmou No ticket to reference this change. --- CODEOWNERS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 31ad382dcc..9a2ef9c66b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,5 @@ -* @caiosba @melsawy @DGaffney @jayjay-w +* @caiosba @vasconsaurus @jayjay-w @melsawy +/.github/ @dmou +/production/ @dmou +Dockerfile @dmou +/docker* @dmou From bee006379a835bc6d93a916b2471e71a2c7f34f2 Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:15:26 +0300 Subject: [PATCH 40/52] Fix username enumeration vulnerability (#2184) * Add tests for username enumeration vulnerability When creating a user account using an email that has already been taken, or when logging in with an email that does not exist in the database, the error messages should be generic and should not indicate that the email already exists, or that the email does not exist (in the case of logins) Here we are adding the tests for this behaviour. --- .../api/v1/registrations_controller.rb | 24 ++++++++++--- app/controllers/application_controller.rb | 7 ++-- config/locales/ar.yml | 2 +- config/locales/en.yml | 8 +++-- config/locales/es.yml | 2 +- config/locales/fr.yml | 2 +- config/locales/id.yml | 2 +- config/locales/mk.yml | 2 +- config/locales/pt.yml | 2 +- doc/api.md | 2 +- .../registrations_controller_test.rb | 35 +++++++++++++++---- 11 files changed, 65 insertions(+), 23 deletions(-) diff --git a/app/controllers/api/v1/registrations_controller.rb b/app/controllers/api/v1/registrations_controller.rb index 0616e599be..25447b8a64 100644 --- a/app/controllers/api/v1/registrations_controller.rb +++ b/app/controllers/api/v1/registrations_controller.rb @@ -14,6 +14,11 @@ def create begin duplicate_user = User.get_duplicate_user(resource.email, [])[:user] user = resource + error = [ + { + message: I18n.t(:email_exists) + } + ] if !duplicate_user.nil? && duplicate_user.invited_to_sign_up? duplicate_user.accept_invitation_or_confirm duplicate_user.password = resource.password @@ -25,13 +30,24 @@ def create resource.last_accepted_terms_at = Time.now resource.save! end + User.current = user sign_up(resource_name, user) - render_success 'user', user + render_success user, 'user', 401, error rescue ActiveRecord::RecordInvalid => e - clean_up_passwords resource - set_minimum_password_length - render_error e.message.gsub(/^Validation failed: Email /, ''), 'INVALID_VALUE' + # Check if the error is specifically related to the email being taken + if resource.errors.details[:email].any? { |email_error| email_error[:error] == :taken } && resource.errors.details.except(:email).empty? + # Treat as successful sign-up if only the email is taken + duplicate_user = User.get_duplicate_user(resource.email, [])[:user] + User.current = duplicate_user if duplicate_user + sign_up(resource_name, duplicate_user) + render_success nil, 'user', 401, error + else + # For other errors, show the error message in the form + clean_up_passwords resource + set_minimum_password_length + render_error e.message.gsub("Email #{I18n.t(:email_exists)}
", '').strip, 'INVALID_VALUE', 401 + end end end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index ec5847b147..c0af24e419 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -32,11 +32,12 @@ def add_info_to_trace private - def render_success(type = 'success', object = nil) + def render_success(type = 'success', object = nil, status = 200, errors = nil) json = { type: type } json[:data] = object unless object.nil? - logger.info message: json, status: 200 - render json: json, status: 200 + json[:errors] = errors unless errors.nil? + logger.info message: json, status: status + render json: json, status: status end def render_error(message, code, status = 400) diff --git a/config/locales/ar.yml b/config/locales/ar.yml index 9ea74cf245..582f628aeb 100644 --- a/config/locales/ar.yml +++ b/config/locales/ar.yml @@ -294,7 +294,7 @@ ar: account_exists: هذا الحساب موجود بالفعل media_exists: هذا العنصر موجود بالفعل source_exists: هذا المصدر موجود بالفعل - email_exists: ' مستخدم بالفعل' + email_exists: 'يرجى التحقق من بريدك الإلكتروني. إذا كان لا يوجد حساب مرتبط بهذا البريد الإلكتروني، فستتلقى رسالة تأكيد. إذا لم تستلم رسالة التأكيد، حاول إعادة تعيين كلمة المرور أو التواصل مع دعمنا.' banned_user: عذراً ، لقد تم حظرك من %{app_name}. يرجى التواصل مع فريق الدعم إن كان هناك خطأ. devise: mailer: diff --git a/config/locales/en.yml b/config/locales/en.yml index 5994644484..04b11bb4fa 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -162,6 +162,10 @@ en: attributes: slug: slug_format: accepts only letters, numbers and hyphens + user: + attributes: + email: + taken: Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support. messages: record_invalid: "%{errors}" improbable_phone: is an invalid number @@ -308,7 +312,7 @@ en: account_exists: This account already exists media_exists: This item already exists source_exists: This source already exists - email_exists: has already been taken + email_exists: Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support. error_password_not_strong: "Complexity requirement not met. Length should be 8-70 characters and include at least: 1 uppercase, 1 lowercase, 1 digit and 1 special character" banned_user: Sorry, your account has been banned from %{app_name}. Please contact the support team if you think this is an error. @@ -335,7 +339,7 @@ en: ignore: If you don't want to accept the invitation, please ignore this email. app_team: "%{app} Workspace" failure: - unconfirmed: Please check your email to verify your account. + unconfirmed: Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support. user_invitation: invited: "%{email} was already invited to this workspace." member: "%{email} is already a member." diff --git a/config/locales/es.yml b/config/locales/es.yml index 2dd82b5429..4fdaf38341 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -282,7 +282,7 @@ es: account_exists: Esta cuenta ya existe media_exists: Este ítem ya existe source_exists: Esta fuente ya existe - email_exists: ya está en uso + email_exists: Por favor, revise su correo electrónico. Si no existe una cuenta con ese correo, debería haber recibido un correo de confirmación. Si no recibe un correo de confirmación, intente restablecer su contraseña o póngase en contacto con nuestro soporte. banned_user: Lo sentimos, tu cuenta ha sido suspendida de %{app_name}. Por favor comunícate con nuestro equipo de soporte técnico si crees que ha sido un error. devise: mailer: diff --git a/config/locales/fr.yml b/config/locales/fr.yml index b59859d321..197641db50 100644 --- a/config/locales/fr.yml +++ b/config/locales/fr.yml @@ -282,7 +282,7 @@ fr: account_exists: Ce compte existe déjà media_exists: Cet élément existe déjà source_exists: Cette source existe déjà - email_exists: est déjà utilisée + email_exists: Veuillez vérifier votre courriel. Si aucun compte n'existe avec ce courriel, vous devriez avoir reçu un courriel de confirmation. Si vous ne recevez pas de courriel de confirmation, essayez de réinitialiser votre mot de passe ou contactez notre support. banned_user: Désolé, votre compte a été banni de %{app_name}. Contactez l’équipe d’assistance si vous pensez que c’est une erreur. devise: mailer: diff --git a/config/locales/id.yml b/config/locales/id.yml index 095d42a128..92909adb21 100644 --- a/config/locales/id.yml +++ b/config/locales/id.yml @@ -274,7 +274,7 @@ id: account_exists: Akun ini sudah ada media_exists: Perihal ini sudah ada source_exists: Sumber ini sudah ada - email_exists: sudah diambil + email_exists: Silakan periksa email Anda. Jika tidak ada akun dengan email tersebut, Anda seharusnya menerima email konfirmasi. Jika Anda tidak menerima email konfirmasi, coba atur ulang kata sandi Anda atau hubungi dukungan kami. banned_user: 'Maaf, akun Anda telah dilarang dari %{app_name}. Mohon hubungi tim dukungan jika Anda merasa ini adalah sebuah kesalahan. ' devise: mailer: diff --git a/config/locales/mk.yml b/config/locales/mk.yml index 9060075df5..f68d7c40c6 100644 --- a/config/locales/mk.yml +++ b/config/locales/mk.yml @@ -278,7 +278,7 @@ mk: account_exists: Оваа сметка веќе постои media_exists: Оваа статија веќе постои source_exists: Овој извор веќе постои - email_exists: веќе е во употреба + email_exists: Ве молиме проверете ја вашата е-пошта. Ако не постои сметка со таа е-пошта, треба да добиете потврден е-маил. Ако не добиете потврден е-маил, обидете се да ја ресетирате вашата лозинка или контактирајте ја нашата поддршка. banned_user: За жал, Вашата сметка има забрана за %{app_name}. Стапете во контакт со тимот за поддршка доколку сметате дека станува збор за грешка. devise: mailer: diff --git a/config/locales/pt.yml b/config/locales/pt.yml index 7899661656..46ba446f46 100644 --- a/config/locales/pt.yml +++ b/config/locales/pt.yml @@ -282,7 +282,7 @@ pt: account_exists: Esta conta já existe media_exists: Esse item já existe source_exists: Essa fonte já existe - email_exists: já está em uso + email_exists: Por favor, verifique seu e-mail. Se não existir uma conta com esse e-mail, você deve ter recebido um e-mail de confirmação. Se você não receber um e-mail de confirmação, tente redefinir sua senha ou entre em contato com nosso suporte. banned_user: Desculpe, sua conta foi banida do %{app_name}. Por favor, entre em contato com a equipe de suporte se você achar que isso é um erro. devise: mailer: diff --git a/doc/api.md b/doc/api.md index 11cf1aa2a2..87f7fc9196 100644 --- a/doc/api.md +++ b/doc/api.md @@ -365,7 +365,7 @@ Use this method in order to create a new user account { "errors": [ { - "message": "Please check your email to verify your account.", + "message": "Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support.", "code": 1, "data": { } diff --git a/test/controllers/registrations_controller_test.rb b/test/controllers/registrations_controller_test.rb index df4bd0500e..bdaa3835b4 100644 --- a/test/controllers/registrations_controller_test.rb +++ b/test/controllers/registrations_controller_test.rb @@ -36,7 +36,7 @@ def teardown p1 = random_complex_password assert_no_difference 'User.count' do post :create, params: { api_user: { password: p1, password_confirmation: p1, email: email, login: 'test', name: 'Test' } } - assert_response :success + assert_response 401 end end @@ -45,7 +45,7 @@ def teardown User.any_instance.stubs(:confirmation_required?).returns(false) assert_difference 'User.count' do post :create, params: { api_user: { password: p1, password_confirmation: p1, email: 't@test.com', login: 'test', name: 'Test' } } - assert_response :success + assert_response 401 end User.any_instance.unstub(:confirmation_required?) end @@ -54,7 +54,7 @@ def teardown p1 = random_complex_password assert_no_difference 'User.count' do post :create, params: { api_user: { password_confirmation: p1, email: 't@test.com', login: 'test', name: 'Test' } } - assert_response 400 + assert_response 401 end end @@ -62,7 +62,7 @@ def teardown p1 = '1234' assert_no_difference 'User.count' do post :create, params: { api_user: { password: p1, password_confirmation: p1, email: 't@test.com', login: 'test', name: 'Test' } } - assert_response 400 + assert_response 401 end end @@ -70,7 +70,7 @@ def teardown p1 = random_complex_password assert_no_difference 'User.count' do post :create, params: { api_user: { password: random_complex_password, password_confirmation: random_complex_password, email: 't@test.com', login: 'test', name: 'Test' } } - assert_response 400 + assert_response 401 end end @@ -78,7 +78,7 @@ def teardown p1 = random_complex_password assert_no_difference 'User.count' do post :create, params: { api_user: { password: p1, password_confirmation: p1, email: '', login: 'test', name: 'Test' } } - assert_response 400 + assert_response 401 end end @@ -94,7 +94,7 @@ def teardown p1 = random_complex_password assert_no_difference 'User.count' do post :create, params: { api_user: { password: p1, password_confirmation: p1, email: 't@test.com', login: 'test', name: '' } } - assert_response 400 + assert_response 401 end end @@ -143,4 +143,25 @@ def teardown end assert_response 401 end + + test "should return generic response in case of error when registering using an existing email" do + existing_user = create_user(email: 'existing@test.com') + p1 = random_complex_password + + assert_no_difference 'User.count' do + post :create, params: { api_user: { password: p1, password_confirmation: p1, email: existing_user.email, login: 'test', name: 'Test' } } + assert_response 401 + assert_equal 'Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support.', response.parsed_body.dig("errors", 0, "message") + end + end + + test "should return generic response when registering with non-existing email" do + p1 = random_complex_password + + assert_difference 'User.count', 1 do + post :create, params: { api_user: { password: p1, password_confirmation: p1, email: 'non_existing@test.com', login: 'test', name: 'Test' } } + assert_response 401 + assert_equal 'Please check your email. If an account with that email doesn’t exist, you should have received a confirmation email. If you don’t receive a confirmation e-mail, try to reset your password or get in touch with our support.', JSON.parse(response.parsed_body).dig("errors", 0, "message") + end + end end From 79846285043c5534e7107cb87e54fd7aafd8ce00 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 29 Jan 2025 12:02:20 +0200 Subject: [PATCH 41/52] CV2-5503: check status object (#2192) --- app/models/project_media.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/project_media.rb b/app/models/project_media.rb index ffd7936ffe..4364dcc54b 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -352,14 +352,16 @@ def self.apply_replace_by(old_pm_id, new_pm_id, options_json) def replace_merge_assignments(assignments_ids) unless assignments_ids.blank? status = self.last_status_obj - assignments_uids = status.assignments.map(&:user_id) - Assignment.where(id: assignments_ids).find_each do |as| - if assignments_uids.include?(as.user_id) - as.skip_check_ability = true - as.delete - else - as.update_columns(assigned_id: status.id) - as.send(:increase_assignments_count) + unless status.nil? + assignments_uids = status.assignments.map(&:user_id) + Assignment.where(id: assignments_ids).find_each do |as| + if assignments_uids.include?(as.user_id) + as.skip_check_ability = true + as.delete + else + as.update_columns(assigned_id: status.id) + as.send(:increase_assignments_count) + end end end end From 75dd80267b56cf902249b2b2cd8b2d2dc0983512 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Wed, 29 Jan 2025 12:05:47 +0200 Subject: [PATCH 42/52] CV2-4985: Fix type condition (#2194) * CV2-4985: fix type condition * CV2-4985: apply PR comments --- app/models/concerns/team_private.rb | 2 +- test/models/team_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/concerns/team_private.rb b/app/models/concerns/team_private.rb index abfed1f29b..688edb0801 100644 --- a/app/models/concerns/team_private.rb +++ b/app/models/concerns/team_private.rb @@ -153,7 +153,7 @@ def get_dashboard_export_headers(ts, dashboard_type) # Hash to include top items as the header label depend on top_items size top_items = {} # tipline_dashboard columns - if dashboard_type == 'tipline_dashboard' + if dashboard_type.to_sym == :tipline_dashboard header.merge!({ 'Conversations': { number_of_conversations: 'to_i' }, 'Messages': { number_of_messages: 'to_i' }, diff --git a/test/models/team_test.rb b/test/models/team_test.rb index fecd3615ab..4134f078cd 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -1326,12 +1326,12 @@ def setup create_tipline_request team: team.id, associated: pm2, disable_es_callbacks: false sleep 1 filters = { period: "past_week", platform: "whatsapp", language: "en" } - data = team.get_dashboard_exported_data(filters, 'tipline_dashboard') + data = team.get_dashboard_exported_data(filters, :tipline_dashboard) assert_not_nil data assert_equal data[0].length, data[1].length cd = create_claim_description project_media: pm fc = create_fact_check claim_description: cd, rating: 'in_progress' - data = team.get_dashboard_exported_data(filters, 'articles_dashboard') + data = team.get_dashboard_exported_data(filters, :articles_dashboard) assert_not_nil data assert_equal data[0].length, data[1].length end From 0f15720e06447ab4f3046bf6da54ff0a1f2736ca Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 30 Jan 2025 09:52:44 -0300 Subject: [PATCH 43/52] Account for null value. (#2197) Don't crash if the data returned is not in the expected structure. Fixes: CV2-6053. --- app/models/concerns/alegre_v2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 9bc2a70ab4..8cc080bccc 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -358,7 +358,7 @@ def parse_similarity_results(project_media, field, results, relationship_type) context: result["context"], model: result["model"], source_field: get_target_field(project_media, field), - target_field: get_target_field(project_media, result["field"] || result["context"]["field"]), + target_field: get_target_field(project_media, result["field"] || result.dig("context", "field")), relationship_type: relationship_type } ] From bcd39dd2fff153cd74e2ade410ac0d42a9066943 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Thu, 30 Jan 2025 09:54:25 -0300 Subject: [PATCH 44/52] Adding `social_media` value to media type search filter. (#2196) This new value `social_media` is basically a shortcut for all the possible social media types. Reference: CV2-4980. --- lib/check_search.rb | 8 ++++++++ test/lib/check_search_test.rb | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/lib/check_search.rb b/lib/check_search.rb index 509570c82e..cedbe5c675 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -22,6 +22,7 @@ def initialize(options, file = nil, team_id = Team.current&.id) @options['esoffset'] ||= 0 adjust_es_window_size + adjust_show_filter adjust_channel_filter adjust_numeric_range_filter adjust_archived_filter @@ -429,6 +430,13 @@ def adjust_channel_filter end end + def adjust_show_filter + if @options['show'].is_a?(Array) && @options['show'].include?('social_media') + @options['show'].concat(%w[twitter youtube tiktok instagram facebook telegram]).delete('social_media') + @options['show'].uniq! + end + end + def adjust_numeric_range_filter @options['range_numeric'] = {} [:linked_items_count, :suggestions_count, :demand, :positive_tipline_search_results_count, :negative_tipline_search_results_count, :tags_as_sentence].each do |field| diff --git a/test/lib/check_search_test.rb b/test/lib/check_search_test.rb index 8a59c5bd24..b23b93be2f 100644 --- a/test/lib/check_search_test.rb +++ b/test/lib/check_search_test.rb @@ -26,4 +26,9 @@ def teardown search = CheckSearch.new({ users: [1, nil] }.to_json, nil, @team.id) assert_not_nil search.send(:doc_conditions) end + + test "should adjust social media filter" do + search = CheckSearch.new({ show: ['social_media', 'images'] }.to_json, nil, @team.id) + assert_equal ['images', 'twitter', 'youtube', 'tiktok', 'instagram', 'facebook', 'telegram'].sort, search.instance_variable_get('@options')['show'].sort + end end From 6eef2388cd6b9e25ffcb2c0fc976cb05b9cf7fa2 Mon Sep 17 00:00:00 2001 From: Manu Vasconcelos <87862340+vasconsaurus@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:38:05 -0300 Subject: [PATCH 45/52] =?UTF-8?q?feature=20=E2=80=93=206021=20=E2=80=93=20?= =?UTF-8?q?Store=20original=20claim=20in=20Media=20object=20(#2189)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: To debug items created using the API (e.g., through the Zapier integration), improve performance, and avoid duplications, it is important to store the original_claim and original_claim_hash in the Media object created when this attribute is present. Context: When a ProjectMedia is created (e.g., via a GraphQL mutation) and the original_claim attribute (URL or text) is provided, a new Media instance is created from that original claim and associated with the ProjectMedia instance. Now we store the reference between the original claim and the created media. Why original_claim_hash: Since we’re handling text as well, it can be tricky (e.g., bad performance, or not feasible at all) to guarantee uniqueness for long text values (it’s a PostgreSQL limitation for unique index). We can solve it by adding an additional column `original_claim_hash`, and use it for the uniqueness validation. When creating a `Media` instance based on the `original_claim`: - We check if a `Media` instance with the same `original_claim_hash` already exists. If it does, we return that instance instead of creating a new one. - Otherwise, we save the `original_claim` and its hash. References: CV2-6021 PR: 2189 --- app/models/concerns/project_media_creators.rb | 54 +------- app/models/media.rb | 60 ++++++++- ...50124155814_add_original_claim_to_media.rb | 7 ++ db/schema.rb | 7 +- test/models/media_test.rb | 119 ++++++++++++++++++ test/models/project_media_7_test.rb | 78 +++++++++++- 6 files changed, 265 insertions(+), 60 deletions(-) create mode 100644 db/migrate/20250124155814_add_original_claim_to_media.rb diff --git a/app/models/concerns/project_media_creators.rb b/app/models/concerns/project_media_creators.rb index dca9f1566b..60b10b1d77 100644 --- a/app/models/concerns/project_media_creators.rb +++ b/app/models/concerns/project_media_creators.rb @@ -54,59 +54,7 @@ def create_annotation def create_original_claim claim = self.set_original_claim.strip - if claim.match?(/\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/) - uri = URI.parse(claim) - content_type = fetch_content_type(uri) - ext = File.extname(uri.path) - - case content_type - when /^image\// - self.media = create_media_from_url('UploadedImage', claim, ext) - when /^video\// - self.media = create_media_from_url('UploadedVideo', claim, ext) - when /^audio\// - self.media = create_media_from_url('UploadedAudio', claim, ext) - else - self.media = create_link_media(claim) - end - else - self.media = create_claim_media(claim) - end - end - - def fetch_content_type(uri) - response = Net::HTTP.get_response(uri) - response['content-type'] - end - - def create_media_from_url(type, url, ext) - klass = type.constantize - file = download_file(url, ext) - m = klass.new - m.file = file - m.save! - m - end - - def download_file(url, ext) - raise "Invalid URL when creating media from original claim attribute" unless url =~ /\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/ - - file = Tempfile.new(['download', ext]) - file.binmode - file.write(URI(url).open.read) - file.rewind - file - end - - def create_claim_media(text) - Claim.create!(quote: text) - end - - def create_link_media(url) - team = self.team || Team.current - pender_key = team.get_pender_key if team - url_from_pender = Link.normalized(url, pender_key) - Link.find_by(url: url_from_pender) || Link.create!(url: url, pender_key: pender_key) + self.media = Media.find_or_create_from_original_claim(claim, self.team) end def set_quote_metadata diff --git a/app/models/media.rb b/app/models/media.rb index 76faf15a97..61bd26c2b7 100644 --- a/app/models/media.rb +++ b/app/models/media.rb @@ -11,7 +11,7 @@ class Media < ApplicationRecord has_many :requests, dependent: :destroy has_annotations - before_validation :set_type, :set_url_nil_if_empty, :set_user, on: :create + before_validation :set_type, :set_url_nil_if_empty, :set_user, :set_original_claim_hash, on: :create after_create :set_uuid @@ -20,6 +20,7 @@ def self.types end validates_inclusion_of :type, in: Media.types + validates_uniqueness_of :original_claim_hash, allow_nil: true def class_name 'Media' @@ -76,6 +77,27 @@ def domain '' end + def self.find_or_create_from_original_claim(claim, project_media_team) + if claim.match?(/\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/) + uri = URI.parse(claim) + content_type = Net::HTTP.get_response(uri)['content-type'] + ext = File.extname(uri.path) + + case content_type + when /^image\// + find_or_create_uploaded_file_media_from_original_claim('UploadedImage', claim, ext) + when /^video\// + find_or_create_uploaded_file_media_from_original_claim('UploadedVideo', claim, ext) + when /^audio\// + find_or_create_uploaded_file_media_from_original_claim('UploadedAudio', claim, ext) + else + find_or_create_link_media_from_original_claim(claim, project_media_team) + end + else + find_or_create_claim_media_from_original_claim(claim) + end + end + private def set_url_nil_if_empty @@ -103,4 +125,40 @@ def set_type def set_uuid self.update_column(:uuid, self.id) end + + def set_original_claim_hash + self.original_claim_hash = Digest::MD5.hexdigest(original_claim) unless self.original_claim.blank? + end + + def self.find_or_create_uploaded_file_media_from_original_claim(media_type, url, ext) + klass = media_type.constantize + existing_media = klass.find_by(original_claim_hash: Digest::MD5.hexdigest(url)) + + if existing_media + existing_media + else + file = download_file(url, ext) + klass.create!(file: file, original_claim: url) + end + end + + def self.download_file(url, ext) + raise "Invalid URL when creating media from original claim attribute" unless url =~ /\A#{URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/ + + file = Tempfile.new(['download', ext]) + file.binmode + file.write(URI(url).open.read) + file.rewind + file + end + + def self.find_or_create_claim_media_from_original_claim(text) + Claim.find_by(original_claim_hash: Digest::MD5.hexdigest(text)) || Claim.create!(quote: text, original_claim: text) + end + + def self.find_or_create_link_media_from_original_claim(url, project_media_team) + pender_key = project_media_team.get_pender_key if project_media_team + url_from_pender = Link.normalized(url, pender_key) + Link.find_by(url: url_from_pender) || Link.find_by(original_claim_hash: Digest::MD5.hexdigest(url)) || Link.create!(url: url, pender_key: pender_key, original_claim: url) + end end diff --git a/db/migrate/20250124155814_add_original_claim_to_media.rb b/db/migrate/20250124155814_add_original_claim_to_media.rb new file mode 100644 index 0000000000..2c1bdec40a --- /dev/null +++ b/db/migrate/20250124155814_add_original_claim_to_media.rb @@ -0,0 +1,7 @@ +class AddOriginalClaimToMedia < ActiveRecord::Migration[6.1] + def change + add_column :medias, :original_claim, :text, null: true + add_column :medias, :original_claim_hash, :string, null: true + add_index :medias, :original_claim_hash, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 13c3bf461b..9909e84e86 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2025_01_13_174153) do +ActiveRecord::Schema.define(version: 2025_01_24_155814) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -457,6 +457,9 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "uuid", default: 0, null: false + t.text "original_claim" + t.string "original_claim_hash" + t.index ["original_claim_hash"], name: "index_medias_on_original_claim_hash", unique: true t.index ["url"], name: "index_medias_on_url", unique: true end @@ -1005,6 +1008,6 @@ add_foreign_key "requests", "feeds" create_trigger :enforce_relationships, sql_definition: <<-SQL - CREATE TRIGGER enforce_relationships BEFORE INSERT ON public.relationships FOR EACH ROW EXECUTE PROCEDURE validate_relationships() + CREATE TRIGGER enforce_relationships BEFORE INSERT ON public.relationships FOR EACH ROW EXECUTE FUNCTION validate_relationships() SQL end diff --git a/test/models/media_test.rb b/test/models/media_test.rb index 711f8f74ca..8c877c1a72 100644 --- a/test/models/media_test.rb +++ b/test/models/media_test.rb @@ -627,4 +627,123 @@ def setup create_project_media media: c2 assert_equal c1.id, c2.uuid end + + test "Claim Media: should save the original_claim and original_claim_hash when created from original claim" do + claim = 'This is a claim.' + claim_media = Media.find_or_create_claim_media_from_original_claim(claim) + + assert_not_nil claim_media.original_claim_hash + assert_not_nil claim_media.original_claim + assert_equal claim, claim_media.original_claim + end + + test "Claim Media: should not save original_claim and original_claim_hash when not created from original claim" do + claim_media = Claim.create!(quote: 'This is a claim.') + assert_nil claim_media.original_claim_hash + assert_nil claim_media.original_claim + end + + test "Claim Media: should not create duplicate media if media with original_claim_hash exists" do + assert_difference 'Claim.count', 1 do + 2.times { Media.find_or_create_claim_media_from_original_claim('This is a claim.') } + end + end + + test "Link Media: should save the original_claim and original_claim_hash when created from original claim" do + team = create_team + + # Mock Pender response for Link + link_url = 'https://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + link_response = { + type: 'media', + data: { + url: link_url, + type: 'item' + } + }.to_json + WebMock.stub_request(:get, pender_url).with(query: { url: link_url }).to_return(body: link_response) + + link_media = Media.find_or_create_link_media_from_original_claim(link_url, team) + + assert_not_nil link_media.original_claim_hash + assert_not_nil link_media.original_claim + assert_equal link_url, link_media.original_claim + end + + test "Link Media: should not save original_claim and original_claim_hash when not created from original claim" do + # Mock Pender response for Link + link_url = 'https://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + link_response = { + type: 'media', + data: { + url: link_url, + type: 'item' + } + }.to_json + WebMock.stub_request(:get, pender_url).with(query: { url: link_url }).to_return(body: link_response) + + link_media = Link.create!(url: link_url) + + assert_nil link_media.original_claim_hash + assert_nil link_media.original_claim + end + + test "Link Media: should not create duplicate media if media with original_claim_hash exists" do + team = create_team + + # Mock Pender response for Link + link_url = 'https://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + link_response = { + type: 'media', + data: { + url: link_url, + type: 'item' + } + }.to_json + WebMock.stub_request(:get, pender_url).with(query: { url: link_url }).to_return(body: link_response) + + assert_difference 'Link.count', 1 do + 2.times { Media.find_or_create_link_media_from_original_claim(link_url, team) } + end + end + + test "Uploaded Media: should save the original_claim and original_claim_hash when created from original claim" do + Tempfile.create(['test_audio', '.mp3']) do |file| + file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.mp3'))) + file.rewind + audio_url = "http://example.com/#{file.path.split('/').last}" + ext = File.extname(URI.parse(audio_url).path) + WebMock.stub_request(:get, audio_url).to_return(body: file.read, headers: { 'Content-Type' => 'audio/mp3' }) + + uploaded_media = Media.find_or_create_uploaded_file_media_from_original_claim('UploadedAudio', audio_url, ext) + + assert_not_nil uploaded_media.original_claim_hash + assert_not_nil uploaded_media.original_claim + assert_equal audio_url, uploaded_media.original_claim + end + end + + test "Uploaded Media: should not save original_claim and original_claim_hash when not created from original claim" do + uploaded_media = create_uploaded_audio + + assert_nil uploaded_media.original_claim_hash + assert_nil uploaded_media.original_claim + end + + test "Uploaded Media: should not create duplicate media if media with original_claim_hash exists" do + Tempfile.create(['test_audio', '.mp3']) do |file| + file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.mp3'))) + file.rewind + audio_url = "http://example.com/#{file.path.split('/').last}" + ext = File.extname(URI.parse(audio_url).path) + WebMock.stub_request(:get, audio_url).to_return(body: file.read, headers: { 'Content-Type' => 'audio/mp3' }) + + assert_difference 'UploadedAudio.count', 1 do + 2.times { Media.find_or_create_uploaded_file_media_from_original_claim('UploadedAudio', audio_url, ext) } + end + end + end end diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index 24bbfe1ceb..424e1e9ddf 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -91,7 +91,7 @@ def setup end end - test "should create duplicate media from original claim URL as UploadedImage" do + test "should not create duplicate media from original claim URL as UploadedImage" do Tempfile.create(['test_image', '.jpg']) do |file| file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.png'))) file.rewind @@ -101,17 +101,17 @@ def setup t = create_team create_project team: t - assert_difference 'ProjectMedia.count', 2 do + assert_raise RuntimeError do 2.times { create_project_media(team: t, set_original_claim: image_url) } end end end - test "should create duplicate media from original claim URL as Claim" do + test "should not create duplicate media from original claim URL as Claim" do t = create_team create_project team: t - assert_difference 'ProjectMedia.count', 2 do + assert_raise RuntimeError do 2.times { create_project_media(team: t, set_original_claim: 'This is a claim.') } end end @@ -196,4 +196,74 @@ def setup ProjectMedia.handle_fact_check_for_existing_claim(pm1, pm2) end end + + test "should save the original_claim url and original_claim_hash when Link Media is created from original_claim" do + # Mock Pender response for Link + link_url = 'https://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + link_response = { + type: 'media', + data: { + url: link_url, + type: 'item' + } + }.to_json + WebMock.stub_request(:get, pender_url).with(query: { url: link_url }).to_return(body: link_response) + + pm_link = create_project_media(set_original_claim: link_url) + media = pm_link.media + + assert_not_nil media.original_claim + assert_not_nil media.original_claim_hash + assert_equal link_url, media.original_claim + end + + test "should save the original_claim text and original_claim_hash when Claim media is created from original_claim" do + claim = 'This is a claim.' + pm_claim = create_project_media(set_original_claim: claim) + media = pm_claim.media + + assert_not_nil media.original_claim + assert_not_nil media.original_claim_hash + assert_equal claim, media.original_claim + end + + test "should save the original_claim url and original_claim_hash when UploadedAudio Media is created from original_claim" do + Tempfile.create(['test_audio', '.mp3']) do |file| + file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.mp3'))) + file.rewind + audio_url = "http://example.com/#{file.path.split('/').last}" + WebMock.stub_request(:get, audio_url).to_return(body: file.read, headers: { 'Content-Type' => 'audio/mp3' }) + + pm_audio = create_project_media(set_original_claim: audio_url) + media = pm_audio.media + + assert_not_nil media.original_claim + assert_not_nil media.original_claim_hash + assert_equal audio_url, media.original_claim + end + end + + test "should check if the original_claim text exists and return that instance when trying to create claim media" do + text = 'This is a claim.' + + claim = Claim.create!(quote: text, original_claim: text) + pm = create_project_media(set_original_claim: text) + + assert_equal claim.id, pm.media.id + end + + test "should check if the original_claim url exists and return that instance when trying to create UploadedAudio media" do + Tempfile.create(['test_audio', '.mp3']) do |file| + file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.mp3'))) + file.rewind + audio_url = "http://example.com/#{file.path.split('/').last}" + WebMock.stub_request(:get, audio_url).to_return(body: file.read, headers: { 'Content-Type' => 'audio/mp3' }) + + audio = UploadedAudio.create!(file: file, original_claim: audio_url) + pm = create_project_media(set_original_claim: audio_url) + + assert_equal audio.id, pm.media.id + end + end end From 5ec4090bd5d9f972d0c4228b46e1465131329c22 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:59:43 -0300 Subject: [PATCH 46/52] Notify Sentry when a request to Alegre fails. (#2198) Notify for every retry. Fixes: CV2-6070. --- app/models/concerns/alegre_v2.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 8cc080bccc..f6a89ed8de 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -124,11 +124,12 @@ def request(method, path, params, retries=3) Rails.logger.info("[Alegre Bot] Alegre response: #{parsed_response.inspect}") parsed_response rescue StandardError => e + Rails.logger.error("[Alegre Bot] Alegre error: (#{method}, #{path}, #{params.inspect}, #{retries}), #{e.inspect} #{e.message}") + CheckSentry.notify(e, bot: 'alegre', method: method, path: path, params: params, retries: retries) if retries > 0 sleep 1 self.request(method, path, params, retries - 1) end - Rails.logger.error("[Alegre Bot] Alegre error: (#{method}, #{path}, #{params.inspect}, #{retries}), #{e.inspect} #{e.message}") { 'type' => 'error', 'data' => { 'message' => e.message } } end end From 866d888285476517cf22bc575f05e871360da626 Mon Sep 17 00:00:00 2001 From: Caio Almeida <117518+caiosba@users.noreply.github.com> Date: Sun, 2 Feb 2025 18:10:03 -0300 Subject: [PATCH 47/52] Fixing how published fact-checks are filtered for bot preview query. (#2199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, both published and unpublished fact-checks were returned by the Smooch Bot query and then re-filtered in PostgreSQL. The problem with this approach was that if the query returned three results—two published and one unpublished—only the two published fact-checks would remain after re-filtering. This change also makes the code behave more similarly to the real tipline search. Fixes: CV2-5572. --- app/models/team.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/models/team.rb b/app/models/team.rb index e202256195..55809ee42b 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -577,14 +577,10 @@ def search_for_similar_articles(query, pm = nil, language = nil, settings = nil) fc_items = [] ex_items = [] threads << Thread.new { - result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit, nil, nil, language, false, settings).map(&:id) + result_ids = Bot::Smooch.search_for_similar_published_fact_checks_no_cache('text', query, [self.id], limit, nil, nil, language, pm.nil?, settings).map(&:id) unless result_ids.blank? fc_items = FactCheck.joins(claim_description: :project_media).where('project_medias.id': result_ids) - if pm.nil? - # This means we obtain relevant items for the Bot preview, so we should limit FactChecks to published articles; - # otherwise, relevant articles for ProjectMedia should include all FactChecks. - fc_items = fc_items.where(report_status: 'published') - elsif !pm.fact_check_id.nil? + if !pm&.fact_check_id.nil? # Exclude the ones already applied to a target item if exists. fc_items = fc_items.where.not('fact_checks.id' => pm.fact_check_id) unless pm&.fact_check_id.nil? end From 31039ea1c8706f2d08d00d15e75864dd9ef4c860 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 3 Feb 2025 12:54:25 +0200 Subject: [PATCH 48/52] CV2-5915: Delete search filters that are not needed anymore (#2191) * CV2-5915: remove project_id and comments from ES docs * CV2-5915: fix tests --- app/lib/check_elastic_search.rb | 1 - app/models/annotations/comment.rb | 22 ---------- app/models/project_media.rb | 2 +- app/repositories/media_search.rb | 10 ----- app/workers/project_media_cache_worker.rb | 2 +- lib/check_search.rb | 13 +----- test/controllers/elastic_search_2_test.rb | 26 ----------- test/controllers/elastic_search_4_test.rb | 5 --- test/controllers/elastic_search_5_test.rb | 53 +---------------------- test/controllers/elastic_search_7_test.rb | 3 -- test/controllers/elastic_search_test.rb | 12 ----- test/models/project_media_5_test.rb | 19 +------- test/models/project_media_6_test.rb | 4 -- 13 files changed, 8 insertions(+), 164 deletions(-) diff --git a/app/lib/check_elastic_search.rb b/app/lib/check_elastic_search.rb index 300c094777..77bbda06cb 100644 --- a/app/lib/check_elastic_search.rb +++ b/app/lib/check_elastic_search.rb @@ -16,7 +16,6 @@ def create_elasticsearch_doc_bg(options) # TODO: Sawy remove annotation_type field ms.attributes[:annotation_type] = 'mediasearch' ms.attributes[:team_id] = self.team_id - ms.attributes[:project_id] = self.project_id ms.attributes[:annotated_type] = self.class.name ms.attributes[:annotated_id] = self.id ms.attributes[:parent_id] = self.id diff --git a/app/models/annotations/comment.rb b/app/models/annotations/comment.rb index 59a30366f8..e1da04cf3a 100644 --- a/app/models/annotations/comment.rb +++ b/app/models/annotations/comment.rb @@ -8,9 +8,6 @@ class Comment < ApplicationRecord before_save :extract_check_entities, unless: proc { |p| p.is_being_copied } after_commit :send_slack_notification, on: [:create, :update] - after_commit :add_elasticsearch_comment, on: :create - after_commit :update_elasticsearch_comment, on: :update - after_commit :destroy_elasticsearch_comment, on: :destroy notifies_pusher on: :destroy, event: 'media_updated', @@ -86,23 +83,4 @@ def extract_check_entities end self.entities = ids end - - def add_elasticsearch_comment - add_update_elasticsearch_comment('create') - end - - def update_elasticsearch_comment - add_update_elasticsearch_comment('update') - end - - def add_update_elasticsearch_comment(op) - # add item/task notes - if self.annotated_type == 'ProjectMedia' - add_update_nested_obj({ op: op, nested_key: 'comments', keys: ['text'], pm_id: self.annotated_id }) - end - end - - def destroy_elasticsearch_comment - destroy_es_items('comments', 'destroy_doc_nested', self.annotated_id) if self.annotated_type == 'ProjectMedia' - end end diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 4364dcc54b..4ddd6cb6f4 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -597,7 +597,7 @@ def add_extra_elasticsearch_data(ms) # set fields with integer value including cached fields fields_i = [ 'archived', 'sources_count', 'linked_items_count', 'share_count','last_seen', 'demand', 'user_id', - 'read', 'suggestions_count','related_count', 'reaction_count', 'comment_count', 'media_published_at', + 'read', 'suggestions_count','related_count', 'reaction_count', 'media_published_at', 'unmatched', 'fact_check_published_on' ] fields_i.each{ |f| ms.attributes[f] = self.send(f).to_i } diff --git a/app/repositories/media_search.rb b/app/repositories/media_search.rb index 44748659cc..ea534bc737 100644 --- a/app/repositories/media_search.rb +++ b/app/repositories/media_search.rb @@ -4,7 +4,6 @@ class MediaSearch mapping do indexes :team_id, { type: 'integer' } - indexes :project_id, { type: 'integer' } indexes :annotated_type, { type: 'text' } indexes :annotated_id, { type: 'integer' } indexes :parent_id, { type: 'integer' } @@ -21,13 +20,6 @@ class MediaSearch indexes :created_at, { type: 'date' } indexes :updated_at, { type: 'date' } indexes :language, { type: 'text', analyzer: 'keyword' } - indexes :comments, { - type: 'nested', - properties: { - id: { type: 'text'}, - text: { type: 'text', analyzer: 'check'} - } - } indexes :tags, { type: 'nested', properties: { @@ -83,8 +75,6 @@ class MediaSearch indexes :reaction_count, { type: 'long' } - indexes :comment_count, { type: 'long' } - indexes :related_count, { type: 'long' } indexes :suggestions_count, { type: 'long' } diff --git a/app/workers/project_media_cache_worker.rb b/app/workers/project_media_cache_worker.rb index 053ef6c62b..abaecd41bf 100644 --- a/app/workers/project_media_cache_worker.rb +++ b/app/workers/project_media_cache_worker.rb @@ -6,7 +6,7 @@ class ProjectMediaCacheWorker PROJECT_MEDIA_CACHED_FIELDS = [ 'linked_items_count', 'suggestions_count', 'is_suggested', 'is_confirmed', 'related_count', 'requests_count', 'demand', 'last_seen', 'description', 'title', 'status', 'share_count', - 'reaction_count', 'comment_count', 'report_status', 'tags_as_sentence', 'sources_as_sentence', + 'reaction_count', 'report_status', 'tags_as_sentence', 'sources_as_sentence', 'media_published_at', 'published_by', 'type_of_media', 'added_as_similar_by_name', 'confirmed_as_similar_by_name', 'folder', 'show_warning_cover', 'picture', 'team_name', 'creator_name' diff --git a/lib/check_search.rb b/lib/check_search.rb index cedbe5c675..c14d388ebd 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -43,7 +43,7 @@ def initialize(options, file = nil, team_id = Team.current&.id) 'recent_activity' => 'updated_at', 'recent_added' => 'created_at', 'demand' => 'demand', 'related' => 'linked_items_count', 'last_seen' => 'last_seen', 'share_count' => 'share_count', 'report_status' => 'report_status', 'tags_as_sentence' => 'tags_as_sentence', - 'media_published_at' => 'media_published_at', 'reaction_count' => 'reaction_count', 'comment_count' => 'comment_count', + 'media_published_at' => 'media_published_at', 'reaction_count' => 'reaction_count', 'related_count' => 'related_count', 'suggestions_count' => 'suggestions_count', 'status_index' => 'status_index', 'type_of_media' => 'type_of_media', 'title' => 'title_index', 'creator_name' => 'creator_name', 'cluster_size' => 'cluster_size', 'cluster_first_item_at' => 'cluster_first_item_at', @@ -167,7 +167,7 @@ def should_hit_elasticsearch? status_blank = false unless @options[field].blank? end filters_blank = true - ['tags', 'keyword', 'rules', 'language', 'fc_language', 'request_language', 'report_language', 'team_tasks', 'assigned_to', 'report_status', 'range_numeric', + ['tags', 'keyword', 'language', 'fc_language', 'request_language', 'report_language', 'team_tasks', 'assigned_to', 'report_status', 'range_numeric', 'has_claim', 'cluster_teams', 'published_by', 'annotated_by', 'channels', 'cluster_published_reports' ].each do |filter| filters_blank = false unless @options[filter].blank? @@ -471,15 +471,6 @@ def keyword_conditions keyword_c = [] field_conditions = build_keyword_conditions_media_fields keyword_c.concat field_conditions - # Search in comments - keyword_c << { - nested: { - path: "comments", - query: { - simple_query_string: { query: @options["keyword"], fields: ["comments.text"], default_operator: "AND" } - } - } - } if should_include_keyword_field?('comments') # Search in requests [['request_username', 'username'], ['request_identifier', 'identifier'], ['request_content', 'content']].each do |pair| keyword_c << { diff --git a/test/controllers/elastic_search_2_test.rb b/test/controllers/elastic_search_2_test.rb index 963da34f48..627af80656 100644 --- a/test/controllers/elastic_search_2_test.rb +++ b/test/controllers/elastic_search_2_test.rb @@ -5,32 +5,6 @@ def setup super setup_elasticsearch end - - test "should update elasticsearch after move media to other projects" do - t = create_team - u = create_user - create_team_user team: t, user: u, role: 'admin' - p = create_project team: t - p2 = create_project team: t - m = create_valid_media - User.stubs(:current).returns(u) - pm = create_project_media project: p, media: m, disable_es_callbacks: false - create_comment annotated: pm - create_tag annotated: pm - sleep 1 - id = get_es_id(pm) - ms = $repository.find(id) - assert_equal ms['project_id'].to_i, p.id - assert_equal ms['team_id'].to_i, t.id - pm = ProjectMedia.find pm.id - pm.project_id = p2.id - pm.save! - # confirm annotations log - sleep 1 - ms = $repository.find(id) - assert_equal ms['project_id'].to_i, p2.id - assert_equal ms['team_id'].to_i, t.id - end test "should destroy elasticseach project media" do t = create_team diff --git a/test/controllers/elastic_search_4_test.rb b/test/controllers/elastic_search_4_test.rb index 0cb5351b99..1ed8c5bc08 100644 --- a/test/controllers/elastic_search_4_test.rb +++ b/test/controllers/elastic_search_4_test.rb @@ -24,11 +24,6 @@ def setup # keyword & tags & status result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], verification_status: ['verified']}.to_json) assert_equal [pm.id], result.medias.map(&:id) - # search keyword in comments - create_comment text: 'add_comment', annotated: pm, disable_es_callbacks: false - sleep 1 - result = CheckSearch.new({keyword: 'add_comment'}.to_json) - assert_equal [pm.id], result.medias.map(&:id) end test "should sort results by recent activities and recent added" do diff --git a/test/controllers/elastic_search_5_test.rb b/test/controllers/elastic_search_5_test.rb index e37507c3e5..7f3c975ca5 100644 --- a/test/controllers/elastic_search_5_test.rb +++ b/test/controllers/elastic_search_5_test.rb @@ -74,9 +74,6 @@ def setup Sidekiq::Testing.inline! do pm = create_project_media project: p, media: m, disable_es_callbacks: false c = create_comment annotated: pm, disable_es_callbacks: false - sleep 1 - result = $repository.find(get_es_id(pm)) - assert_equal 1, result['comments'].count id = pm.id m.destroy assert_equal 0, ProjectMedia.where(media_id: id).count @@ -108,28 +105,6 @@ def setup end end - test "should create update destroy elasticsearch comment" do - t = create_team - p = create_project team: t - m = create_valid_media - s = create_source - pm = create_project_media project: p, media: m, disable_es_callbacks: false - c = create_comment annotated: pm, text: 'test', disable_es_callbacks: false - sleep 1 - result = $repository.find(get_es_id(pm)) - assert_equal [c.id], result['comments'].collect{|i| i["id"]} - # update es comment - c.text = 'test-mod'; c.save! - sleep 1 - result = $repository.find(get_es_id(pm)) - assert_equal ['test-mod'], result['comments'].collect{|i| i["text"]} - # destroy es comment - c.destroy - sleep 1 - result = $repository.find(get_es_id(pm)) - assert_empty result['comments'] - end - test "should create update destroy elasticsearch tag" do t = create_team p = create_project team: t @@ -169,9 +144,8 @@ def setup test "should create parent if not exists" do t = create_team - p = create_project team: t - pm = create_project_media project: p - c = create_comment annotated: pm, disable_es_callbacks: false + pm = create_project_media team: t + t = create_tag annotated: pm, tag: 'sports', disable_es_callbacks: false sleep 1 result = $repository.find(get_es_id(pm)) assert_not_nil result @@ -223,28 +197,5 @@ def setup end end - test "should update elasticsearch after move project to other team" do - u = create_user - t = create_team - t2 = create_team - u.is_admin = true; u.save! - p = create_project team: t - m = create_valid_media - User.stubs(:current).returns(u) - Sidekiq::Testing.inline! do - pm = create_project_media project: p, media: m, disable_es_callbacks: false - pm2 = create_project_media project: p, quote: 'Claim', disable_es_callbacks: false - sleep 2 - results = $repository.search(query: { match: { team_id: t.id } }).results - assert_equal [pm.id, pm2.id], results.collect{|i| i['annotated_id']}.sort - p.team_id = t2.id; p.save! - sleep 2 - results = $repository.search(query: { match: { team_id: t.id } }).results - assert_equal [], results.collect{|i| i['annotated_id']} - results = $repository.search(query: { match: { team_id: t2.id } }).results - assert_equal [pm.id, pm2.id], results.collect{|i| i['annotated_id']}.sort - end - end - # Please add new tests to test/controllers/elastic_search_7_test.rb end diff --git a/test/controllers/elastic_search_7_test.rb b/test/controllers/elastic_search_7_test.rb index 128556df40..6788b0963f 100644 --- a/test/controllers/elastic_search_7_test.rb +++ b/test/controllers/elastic_search_7_test.rb @@ -209,9 +209,6 @@ def setup assert_equal [pm2.id, pm3.id], result.medias.map(&:id).sort create_comment annotated: pm, text: 'item notepm', disable_es_callbacks: false create_comment annotated: pm2, text: 'item comment', disable_es_callbacks: false - sleep 2 - result = CheckSearch.new({keyword: 'item', keyword_fields: {fields: ['comments']}}.to_json, nil, t.id) - assert_equal [pm.id, pm2.id], result.medias.map(&:id).sort end test "should search by media url" do diff --git a/test/controllers/elastic_search_test.rb b/test/controllers/elastic_search_test.rb index 16d7f4e6b1..b9823e1732 100644 --- a/test/controllers/elastic_search_test.rb +++ b/test/controllers/elastic_search_test.rb @@ -29,18 +29,6 @@ def setup ids << id["node"]["dbid"] end assert_equal [pm2.id], ids - create_comment text: 'title_a', annotated: pm1, disable_es_callbacks: false - sleep 2 - Team.stubs(:current).returns(@team) - query = 'query Search { search(query: "{\"keyword\":\"title_a\",\"sort\":\"recent_activity\"}") { medias(first: 10) { edges { node { dbid } } } } }' - post :create, params: { query: query } - Team.unstub(:current) - assert_response :success - ids = [] - JSON.parse(@response.body)['data']['search']['medias']['edges'].each do |id| - ids << id["node"]["dbid"] - end - assert_equal [pm1.id, pm2.id], ids.sort end test "should search media with multiple projects" do diff --git a/test/models/project_media_5_test.rb b/test/models/project_media_5_test.rb index 8ef306782b..4cf8291e2c 100644 --- a/test/models/project_media_5_test.rb +++ b/test/models/project_media_5_test.rb @@ -757,7 +757,6 @@ def setup end test "should restore and confirm item if not super admin" do - setup_elasticsearch t = create_team p = create_project team: t p3 = create_project team: t @@ -765,42 +764,28 @@ def setup create_team_user user: u, team: t, role: 'admin', is_admin: false Sidekiq::Testing.inline! do # test restore - pm = create_project_media project: p, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::TRASHED - sleep 1 - result = $repository.find(get_es_id(pm))['project_id'] - assert_equal p.id, result + pm = create_project_media project: p, archived: CheckArchivedFlags::FlagCodes::TRASHED assert_equal CheckArchivedFlags::FlagCodes::TRASHED, pm.archived with_current_user_and_team(u, t) do pm.archived = CheckArchivedFlags::FlagCodes::NONE - pm.disable_es_callbacks = false pm.project_id = p3.id pm.save! end pm = pm.reload assert_equal CheckArchivedFlags::FlagCodes::NONE, pm.archived assert_equal p3.id, pm.project_id - sleep 1 - result = $repository.find(get_es_id(pm))['project_id'] - assert_equal p3.id, result # test confirm - pm = create_project_media project: p, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::UNCONFIRMED - sleep 1 + pm = create_project_media project: p, archived: CheckArchivedFlags::FlagCodes::UNCONFIRMED assert_equal p.id, pm.project_id - result = $repository.find(get_es_id(pm))['project_id'] - assert_equal p.id, result assert_equal CheckArchivedFlags::FlagCodes::UNCONFIRMED, pm.archived with_current_user_and_team(u, t) do pm.archived = CheckArchivedFlags::FlagCodes::NONE - pm.disable_es_callbacks = false pm.project_id = p3.id pm.save! end pm = pm.reload assert_equal CheckArchivedFlags::FlagCodes::NONE, pm.archived assert_equal p3.id, pm.project_id - sleep 1 - result = $repository.find(get_es_id(pm))['project_id'] - assert_equal p3.id, result end end diff --git a/test/models/project_media_6_test.rb b/test/models/project_media_6_test.rb index b5f46bd20d..bdf2181f1f 100644 --- a/test/models/project_media_6_test.rb +++ b/test/models/project_media_6_test.rb @@ -96,10 +96,8 @@ def setup assert_equal CheckArchivedFlags::FlagCodes::TRASHED, result['archived'] result = $repository.find(get_es_id(pm1_s)) assert_equal CheckArchivedFlags::FlagCodes::NONE, result['archived'] - assert_equal p.id, result['project_id'] result = $repository.find(get_es_id(pm2_s)) assert_equal CheckArchivedFlags::FlagCodes::NONE, result['archived'] - assert_equal p.id, result['project_id'] end test "should detach similar items when spam parent item" do @@ -136,10 +134,8 @@ def setup assert_equal CheckArchivedFlags::FlagCodes::SPAM, result['archived'] result = $repository.find(get_es_id(pm1_s)) assert_equal CheckArchivedFlags::FlagCodes::NONE, result['archived'] - assert_equal p.id, result['project_id'] result = $repository.find(get_es_id(pm2_s)) assert_equal CheckArchivedFlags::FlagCodes::NONE, result['archived'] - assert_equal p.id, result['project_id'] end test "should complete media if there are pending tasks" do From eeef6b235821cdb3a082eafc7b9d85e8593ddbf5 Mon Sep 17 00:00:00 2001 From: Mohamed El-Sawy Date: Mon, 3 Feb 2025 23:35:49 +0200 Subject: [PATCH 49/52] Fix N+1 query for BotUser query (#2200) --- app/models/bot/smooch.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/bot/smooch.rb b/app/models/bot/smooch.rb index 6a17a24741..5446c3d2bc 100644 --- a/app/models/bot/smooch.rb +++ b/app/models/bot/smooch.rb @@ -75,7 +75,7 @@ def self.inherit_status_and_send_report(rid) # A relationship created by the Smooch Bot or Alegre Bot is related to search results (unless it's a suggestion that was confirmed), so the user has already received the report as a search result... no need to send another report # Only send a report for (1) Confirmed matches created manually OR (2) Suggestions accepted - created_by_bot = [BotUser.smooch_user&.id, BotUser.alegre_user&.id].include?(relationship.user_id) + created_by_bot = BotUser.where(login: ['alegre', 'smooch'], id: relationship.user_id).exists? ::Bot::Smooch.send_report_from_parent_to_child(parent.id, target.id) if !created_by_bot || relationship.confirmed_by end end From 9f4dfe551744d0450cefc5532e8faea5fba9c59d Mon Sep 17 00:00:00 2001 From: Alexandre Amoedo Amorim Date: Wed, 5 Feb 2025 09:10:00 -0300 Subject: [PATCH 50/52] Update L10n (#2203) --- config/locales/ar.yml | 44 +++++++++++++++++++++++++++++++++------ config/locales/es.yml | 44 ++++++++++++++++++++++++++++++++++----- config/locales/fr.yml | 48 +++++++++++++++++++++++++++++++++++-------- config/locales/id.yml | 46 ++++++++++++++++++++++++++++++++++------- 4 files changed, 156 insertions(+), 26 deletions(-) diff --git a/config/locales/ar.yml b/config/locales/ar.yml index 582f628aeb..2af162225f 100644 --- a/config/locales/ar.yml +++ b/config/locales/ar.yml @@ -73,6 +73,7 @@ ar: description: 'النتيجة: هذا مصدر ذو هوية مزيفة على الانترنت' errors: messages: + extension_whitelist_error: الامتداد غير مدعوم invalid_password: كلمة السّر غير صالحة invalid_qrcode: رمز التحقق غير صالح extension_white_list_error: 'لا يمكن أن تكون من نوع %{extension}، الأنواع المسموح بها : %{allowed_types}' @@ -93,9 +94,10 @@ ar: invalid_project_media_channel_value: عذرا، لا ندعم هذه الغرفة invalid_project_media_channel_update: عذراً، لقد تعذر عليك تحديث الغرفة invalid_project_media_archived_value: عذرا، إن القيم المؤرشفة غير مدعومة - invalid_fact_check_language_value: عذرا، قيمة اللغة غير مدعومة + invalid_article_language_value: عذرا، قيمة اللغة غير مدعومة fact_check_empty_title_and_summary: عذرا، يجب ملء العنوان أو الملخص invalid_feed_saved_search_value: يجب أن ينتمي إلى مساحة العمل و التي تشكل جزءا من هذا الموجز + platform_allowed_values_error: '‫لا يمكن أن تكون من نوع %{type}، الأنواع المسموح بها : %{allowed_types}' activerecord: models: link: رابط @@ -146,6 +148,10 @@ ar: attributes: slug: slug_format: 'يقبل الأحرف والأرقام و''-'' فقط ' + user: + attributes: + email: + taken: يرجى التحقق من بريدك الإلكتروني. إذا كان لا يوجد حساب مرتبط بهذا البريد الإلكتروني، فستتلقى رسالة تأكيد. إذا لم تستلم رسالة التأكيد، حاول إعادة تعيين كلمة السر أو تواصل مع دعمنا. messages: record_invalid: "%{errors}" improbable_phone: رقم غير صالح @@ -294,7 +300,8 @@ ar: account_exists: هذا الحساب موجود بالفعل media_exists: هذا العنصر موجود بالفعل source_exists: هذا المصدر موجود بالفعل - email_exists: 'يرجى التحقق من بريدك الإلكتروني. إذا كان لا يوجد حساب مرتبط بهذا البريد الإلكتروني، فستتلقى رسالة تأكيد. إذا لم تستلم رسالة التأكيد، حاول إعادة تعيين كلمة المرور أو التواصل مع دعمنا.' + email_exists: يرجى التحقق من بريدك الإلكتروني. إذا كان لا يوجد حساب مرتبط بهذا البريد الإلكتروني، فستتلقى رسالة تأكيد. إذا لم تستلم رسالة التأكيد، حاول إعادة تعيين كلمة السر أو تواصل مع دعمنا. + error_password_not_strong: "لم يتم استيفاء مَطلَب التعقيد. يجب أن يتراوح الطول بين 8 و 70 حرفا وأن يتضمن على الأقل: حرفا كبيرا واحدا، وحرفا صغيرا واحدا، ورقما واحدا، وحرفا خاصا واحدا." banned_user: عذراً ، لقد تم حظرك من %{app_name}. يرجى التواصل مع فريق الدعم إن كان هناك خطأ. devise: mailer: @@ -318,7 +325,7 @@ ar: ignore: لرفض الدعوة ، يُمكنكم تجاهل هذه الرسالة. app_team: "مساحة العمل %{app}" failure: - unconfirmed: يرجى مراجعة بريدك الالكتروني لتأكيد حسابك. + unconfirmed: يرجى التحقق من بريدك الإلكتروني. إذا كان لا يوجد حساب مرتبط بهذا البريد الإلكتروني، فستتلقى رسالة تأكيد. إذا لم تستلم رسالة التأكيد، حاول إعادة تعيين كلمة السر أو تواصل مع دعمنا. user_invitation: invited: "تمت دعوة %{email} لمساحة العمل هذه من قبل." member: "%{email} عضو بالفعل." @@ -432,6 +439,24 @@ ar: approved_text: تمت الموافقة على طلب انضمامكم إلى مساحة عمل %{team} على%{app_name}. يمكنكم الآن الذهاب إلى %{url} والبدء في المساهمة! rejected_title: تم رفض الطلب rejected_text: عذراً ، لم تتم الموافقة على طلبك للإنضمام إلى مساحة العمل %{team}على %{app_name} . + feed_invitation: + subject: "‫قام %{user} بدعوة منظمتك للمساهمة في %{feed}" + hello: '‫مرحبا %{email} ' + body: "قام %{name}، %{email} بدعوة منظمتك للمساهمة في موجز Check الذي تمت مشاركته" + view_button: عرض الدعوة + updated_terms: + subject: شروط استخدام Check المُحدَّثة + hello: ‫مرحباً %{name} + title: شروط الاستخدام المُحدَّثة + body: لقد قمنا بتحديث شروط الاستعمال لمستخدمي Check. أرسلنا هذا الإشعار إلى جميع الأشخاص الذين لديهم حساب Check. يُعد استمرار استخدام Check بعد التحديث بمثابة الموافقة على شروط الاستخدام المُحدَّثة. + term_button: شروط الاستخدام + more_info: ‫هذا إشعار قانوني إلزامي يتم إرساله مرة واحدة إلى جميع مستخدمي Check، بما في ذلك الذين ألغوا الاشتراك في البلاغات الاختيارية. + export_list: + hello: مرحباً %{name} + subject: تصدير بيانات Check + body: طلب تصدير بيانات Check الخاص بك متاح للتنزيل. + button_label: تنزيل التصدير + footer: ستنتهي صلاحية رابط التنزيل بتاريخ %{date}. mail_security: device_subject: 'تنبيه: تسجيل دخول جديد على %{app_name} عبر %{browser} على %{platform}' ip_subject: 'تنبيه: هناك تسجيل دخول جديد أو غير اعتيادي على %{app_name}' @@ -576,9 +601,9 @@ ar: smooch_requests_desc: اﻷكثر طلباً bot_request_url_invalid: رابط البوت غير صالح smooch_facebook_success: | - تم بنجاح! - أصبح خطك الساخن (Tipline) متصلا اﻵن بمِرْسال فيسبوك Facebook Messenger. - سيقوم خط الساخن بمعالجة أي رسالة مُستلَمة من صفحة فيسبوك هذه. + لقد تم بنجاح! + أصبح خطك الساخن متصلا اﻵن بالحساب. + سيقوم الخط الساخن بمعالجة أي رسالة مُستلَمة من هذا الحساب. يرجى إعادة تحميل صفحة إعدادات الخط الساخن لمشاهدة التغييرات الجديدة. smooch_twitter_success: | تم بنجاح! @@ -586,6 +611,7 @@ ar: سيقوم الخط الساخن بمعالجة أي رسالة مباشرة جرى استلامها انطلاقا من حساب تويتر هذا. يرجى إعادة تحميل صفحة إعدادات الخط الساخن لمشاهدة التغييرات الجديدة. must_select_exactly_one_facebook_page: يُرجى تحديد صفحة واحدة من فيسبوك لكي يتم وصلها بالخط الساخن (Tipline). + invalid_facebook_authdata: لقد تعذر الاستيثاق من حسابك على فيسبوك، يرجى التواصل مع فريق الدعم. invalid_task_answer: تنسيق إجابة المهمة غير صالح team_rule_name: اسم فريد يصف ما تفعله القاعدة team_rule_project_ids: تطبيق على المجلد @@ -737,6 +763,12 @@ ar: send_every_must_be_a_list_of_days_of_the_week: يجب أن تكون قائمة أيام الأسبوع. send_on_must_be_in_the_future: لا يمكن أن تكون في الماضي. cant_delete_default_folder: لا يمكن حذف المجلد الافتراضي + explainer_and_item_must_be_from_the_same_team: يجب أن يكون التوضيح والعنصر من نفس مساحة العمل. + cant_apply_article_to_item_if_article_is_in_the_trash: هذا المقال موجود في سلة المهملات، لذا لا يمكن إضافته إلى هذا التجمُّع من المواد الإعلامية. يرجى استعادته أولا من سلة المهملات. + shared_feed_imported_media_already_exist: |- + لا توجد مواد إعلامية صالحة للاستيراد نحو مساحة عملك. + إن المواد الإعلامية التي تم اختيارها للاستيراد موجودة بالفعل في مساحة عملك, داخل العناصر التالية: + %{urls} info: messages: sent_to_trash_by_rule: 'أُرسِل "%{item_title}" إلى المهملات بواسطة قاعدة آلية.' diff --git a/config/locales/es.yml b/config/locales/es.yml index 4fdaf38341..0ded619e04 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -73,6 +73,7 @@ es: description: 'Conclusión: la fuente es un farsante  ' errors: messages: + extension_whitelist_error: extensión no es admitida invalid_password: Contraseña incorrecta invalid_qrcode: Código de validación incorrecto extension_white_list_error: 'no puede ser de tipo %{extension}, tipos permitidos: %{allowed_types}' @@ -93,9 +94,10 @@ es: invalid_project_media_channel_value: Lo sentimos, el valor del canal no es compatible invalid_project_media_channel_update: Lo sentimos, no pudiste actualizar el valor del canal invalid_project_media_archived_value: Lo sentimos, el valor archivado no es compatible - invalid_fact_check_language_value: Lo sentimos, valor de idioma no admitido + invalid_article_language_value: Lo sentimos, valor de idioma no admitido fact_check_empty_title_and_summary: Lo sentimos, debes llenar el título o resumen invalid_feed_saved_search_value: debe pertenecer a un área de trabajo que sea parte de este feed + platform_allowed_values_error: 'no puede ser del tipo %{type}, tipos permitidos: %{allowed_types}' activerecord: models: link: Enlace @@ -146,6 +148,10 @@ es: attributes: slug: slug_format: acepta solo letras, números y guiones + user: + attributes: + email: + taken: Por favor, revisa tu correo electrónico. Si no existe una cuenta con ese correo, deberías haber recibido un correo de confirmación. Si no recibes un correo de confirmación, intenta restablecer tu contraseña o contáctate con nuestro soporte. messages: record_invalid: "%{errors}" improbable_phone: 'es un número inválido ' @@ -282,7 +288,8 @@ es: account_exists: Esta cuenta ya existe media_exists: Este ítem ya existe source_exists: Esta fuente ya existe - email_exists: Por favor, revise su correo electrónico. Si no existe una cuenta con ese correo, debería haber recibido un correo de confirmación. Si no recibe un correo de confirmación, intente restablecer su contraseña o póngase en contacto con nuestro soporte. + email_exists: Por favor, revisa tu correo electrónico. Si no existe una cuenta con ese correo, deberías haber recibido un correo de confirmación. Si no recibes un correo de confirmación, intenta restablecer tu contraseña o contáctate con nuestro soporte. + error_password_not_strong: "No cumple con el requisito de complejidad. La longitud debe ser 8-70 caracteres e incluir al menos: 1 mayúscula, 1 minúscula, 1 dígito y 1 carácter especial" banned_user: Lo sentimos, tu cuenta ha sido suspendida de %{app_name}. Por favor comunícate con nuestro equipo de soporte técnico si crees que ha sido un error. devise: mailer: @@ -306,7 +313,7 @@ es: ignore: Si no desea aceptar la invitación, ignore este correo electrónico. app_team: "%{app} Área de trabajo" failure: - unconfirmed: Por favor consulta tu correo electrónico para verificar tu cuenta. + unconfirmed: Por favor, revisa tu correo electrónico. Si no existe una cuenta con ese correo, deberías haber recibido un correo de confirmación. Si no recibes un correo de confirmación, intenta restablecer tu contraseña o contáctate con nuestro soporte. user_invitation: invited: "%{email} : ya ha sido invitado(a) a esta área de trabajo." member: "%{email} : ya es miembro." @@ -420,6 +427,24 @@ es: approved_text: Tu solicitud para unirte a área de trabajo %{team} en %{app_name}fue aprobada. Puedes ir a %{url} y empezar a contribuir. rejected_title: Solicitud denegada rejected_text: Lo sentimos, pero tu solicitud para unirte a área de trabajo %{team} en %{app_name} no fue aprobada. + feed_invitation: + subject: "%{user} invitó a tu organización a contribuir a %{feed}" + hello: Hola %{email} + body: "%{name},%{email} ha invitado a tu organización a contribuir a un Feed Compartido de Check" + view_button: Ver invitación + updated_terms: + subject: Términos del Servicio de Check Actualizados + hello: Hola %{name} + title: Términos del Servicio Actualizados + body: Hemos hecho actualizaciones a nuestros Términos del Servicio para usuaria(o)s de Check. Enviamos esta notificación a toda(o)s las personas que tengan una cuenta de Check. El uso contínuo de Check después de la actualización constituye la aceptación de nuestros Términos de Servicio actualizados. + term_button: Términos del Servicio + more_info: Este es un aviso legal requerido que se envía una vez a toda(o)s la(o)s usuaria(o)s de Check, incluso a quienes se han dado de baja a través de anuncios opcionales. + export_list: + hello: Hola %{name} + subject: Exportación de Datos de Check + body: Tu exportación de datos de Check solicitada está disponible para descargar. + button_label: Descargar Exportación + footer: Este enlace de descarga expirará en %{date}. mail_security: device_subject: 'Alerta de seguridad: Nuevo inicio de sesión de %{app_name} dede %{browser} en %{platform}' ip_subject: 'Alerta de seguridad: Nuevo o inusual inicio de sesión en %{app_name} ' @@ -564,14 +589,16 @@ es: bot_request_url_invalid: El URL del bot es inválido smooch_facebook_success: | ¡Éxito! - Tu Tipline está ahora connectada a Facebook Messenger. - Cualquier mensaje recibido por esta Página de Facebook va a ser manejado por la Tipline. + Tu tipline ahora está conectada al perfil. + Cualquier mensaje recibido por este perfil será manejado por la tipline. + Por favor vuelve a cargar la página de configuración de tu tipline para ver los nuevos cambios. smooch_twitter_success: | ¡Éxito! Tu Tipline está ahora connectada a Twitter. Cualquier mensaje recibido por este perfil de Twitter va a ser manejado por la Tipline. Por favor vuelve a cargar la página de configuración de tu Tipline para ver cambios nuevos must_select_exactly_one_facebook_page: Por favor selecciona exactamente la página de Facebook que quieres integrar con la Tipline. + invalid_facebook_authdata: No fue posible autenticar tu cuenta de Facebook, por favor contacta a nuestro equipo de soporte. invalid_task_answer: Formato de respuesta de tarea inválido team_rule_name: Un nombre único que identifica lo que hace esta regla team_rule_project_ids: Aplicar a carpeta @@ -720,6 +747,13 @@ es: Para dejar de recibir este boletín, escribe "%{unsubscribe}". send_every_must_be_a_list_of_days_of_the_week: debe ser una lista de días de la semana. send_on_must_be_in_the_future: no puede estar en pasado. + cant_delete_default_folder: La carpeta por defecto no puede ser eliminada + explainer_and_item_must_be_from_the_same_team: Explicativo e ítem deben estar en la misma área de trabajo. + cant_apply_article_to_item_if_article_is_in_the_trash: Este artículo está en la papelera por lo que no puede ser añadido a este conjunto de medios. Por favor primero restáuralo de la papelera. + shared_feed_imported_media_already_exist: |- + Ningún medio elegible para ser importado a tu área de trabajo. + El medio seleccionado para importar ya existe en tu área de trabajo en los siguientes ítems: + %{urls} info: messages: sent_to_trash_by_rule: '%{item_title}" ha sido enviado a la papelera por una regla de automatización.' diff --git a/config/locales/fr.yml b/config/locales/fr.yml index 197641db50..fa8c64fea8 100644 --- a/config/locales/fr.yml +++ b/config/locales/fr.yml @@ -73,6 +73,7 @@ fr: description: 'Conclusion : la source est une désinformation populaire planifiée' errors: messages: + extension_whitelist_error: l’extension n’est pas prise en charge invalid_password: Le mot de passe est invalide invalid_qrcode: Le code de validation est invalide extension_white_list_error: 'ne peut pas être de type %{extension}, les types autorisés sont :%{allowed_types}' @@ -93,9 +94,10 @@ fr: invalid_project_media_channel_value: Désolé, la valeur de canal n’est pas prise en charge invalid_project_media_channel_update: Désolé, il ne vous a pas été possible de mettre à jour la valeur du canal invalid_project_media_archived_value: Désolé, la valeur archivée n’est pas prise en charge - invalid_fact_check_language_value: Désolé, la valeur de langue n’est pas prise en charge - fact_check_empty_title_and_summary: Désolé, vous devez remplir le titre ou le résumé + invalid_article_language_value: Désolé, la valeur de langue n’est pas prise en charge# + fact_check_empty_title_and_summary: Désolé, vous devez saisir le titre ou le résumé invalid_feed_saved_search_value: doit appartenir à un espace de travail qui fait partie de ce fil + platform_allowed_values_error: 'ne peut pas être de type %{type}, les types autorisés sont :%{allowed_types}' activerecord: models: link: Lien @@ -146,6 +148,10 @@ fr: attributes: slug: slug_format: n’accepte que les lettres, les chiffres et les tirets + user: + attributes: + email: + taken: Vérifiez vos courriels. Si aucun compte n'existe pour cette adresse de courriel, vous devriez avoir reçu un courriel de confirmation. Si vous ne le recevez pas, essayez de réinitialiser votre mot de passe ou contactez notre équipe d’assistance. messages: record_invalid: "%{errors}" improbable_phone: est un numéro invalide @@ -282,7 +288,8 @@ fr: account_exists: Ce compte existe déjà media_exists: Cet élément existe déjà source_exists: Cette source existe déjà - email_exists: Veuillez vérifier votre courriel. Si aucun compte n'existe avec ce courriel, vous devriez avoir reçu un courriel de confirmation. Si vous ne recevez pas de courriel de confirmation, essayez de réinitialiser votre mot de passe ou contactez notre support. + email_exists: Vérifiez vos courriels. Si aucun compte n'existe pour cette adresse de courriel, vous devriez avoir reçu un courriel de confirmation. Si vous ne le recevez pas, essayez de réinitialiser votre mot de passe ou contactez notre équipe d’assistance. + error_password_not_strong: "L’exigence de complexité n’est pas satisfaite. La longueur doit être de 8 à 70 caractères et comprendre au moins : 1 majuscule, 1 minuscule, 1 chiffre et 1 caractère spécial." banned_user: Désolé, votre compte a été banni de %{app_name}. Contactez l’équipe d’assistance si vous pensez que c’est une erreur. devise: mailer: @@ -306,7 +313,7 @@ fr: ignore: Si vous ne souhaitez pas accepter l’invitation, ignorez ce courriel. app_team: "Espace de travail %{app}" failure: - unconfirmed: Consultez vos courriels afin de confirmer votre compte. + unconfirmed: Vérifiez vos courriels. Si aucun compte n'existe pour cette adresse de courriel, vous devriez avoir reçu un courriel de confirmation. Si vous ne le recevez pas, essayez de réinitialiser votre mot de passe ou contactez notre équipe d’assistance. user_invitation: invited: "%{email} a déjà été invité dans cet espace de travail." member: "%{email} est déjà membre." @@ -419,6 +426,24 @@ fr: approved_text: Votre demande à vous joindre à l’espace de travail %{team} sur %{app_name} a été approuvée. Vous pouvez maintenant accéder à %{url} et commencer à contribuer. rejected_title: Demande refusée rejected_text: Désolé, votre demande de vous joindre à l’espace de travail %{team} sur %{app_name} n’a pas été approuvée. + feed_invitation: + subject: "%{user} a invité votre organisme à contribuer à %{feed}" + hello: Bonjour, %{email} + body: "%{name}, %{email} a invité votre organisme à contribuer à un fil Check partagé" + view_button: Afficher l’invitation + updated_terms: + subject: Mise à jour des Conditions générales d’utilisation de Check + hello: Bonjour, %{name} + title: Mise à jour des Conditions générales d’utilisation + body: Nous avons mis à jour nos Conditions générales d’utilisation pour les utilisateurs de Check. Nous envoyons cette notification à toutes les personnes inscrites à Check. Le fait de continuer à utiliser Check après la mise à jour implique l’acceptation de nos nouvelles Conditions générales d’utilisation. + term_button: Conditions générales d’utilisation + more_info: Il s’agit d’un avis légal unique envoyé à toutes les personnes qui utilisent Check, même à celles qui se sont désabonnées de nos annonces. + export_list: + hello: Bonjour, %{name} + subject: Exportation des données de Check + body: L’exportation des données de Check que vous avez demandée est prête à être téléchargée. + button_label: Télécharger l’exportation + footer: Ce lien de téléchargement expirera le %{date}. mail_security: device_subject: 'Alerte de sécurité : Nouvelle connexion à %{app_name} de %{browser} sur %{platform}' ip_subject: 'Alerte de sécurité : Connexion à %{app_name} nouvelle ou inhabituelle' @@ -564,15 +589,16 @@ fr: bot_request_url_invalid: L’URL du robot est invalide smooch_facebook_success: | C’est réussi ! - Votre ligne info Tipline est désormais connectée à Facebook Messenger. - Tout message reçu par cette page Facebook sera traité par la ligne info. - Afin de voir les nouveaux changements, rechargez la page des paramètres du service Tipline. + Votre ligne info Tipline est désormais connectée au profil. + Tout message reçu par ce profil sera traité par le service Tipline. + Rechargez la page des paramètres du service Tipline pour voir les nouveaux changements. smooch_twitter_success: | C’est réussi ! Votre ligne info Tipline est désormais connectée à Twitter. Tout message direct reçu par ce profil Twitter sera traité par le service Tipline. Rechargez la page des paramètres du service Tipline pour voir les nouveaux changements. must_select_exactly_one_facebook_page: Sélectionnez la page précise et unique de Facebook que vous voulez intégrer au service Tipline. + invalid_facebook_authdata: Il n’a pas été possible d’authentifier votre compte Facebook. Prenez contact avec notre équipe d’assistance. invalid_task_answer: Le format de réponse de la tâche est invalide team_rule_name: Un nom unique qui décrit ce que cette règle fait team_rule_project_ids: Appliquer au dossier @@ -688,7 +714,7 @@ fr: list_column_tags_as_sentence: Étiquettes list_column_media_published_at: Média publié list_column_published_by: Rapport publié par - list_column_fact_check_published_on: Vérification des faits publiés le + list_column_fact_check_published_on: Vérification des faits publiée le list_column_related_count: Associés list_column_suggestions_count: Suggestions list_column_folder: Dossier @@ -722,6 +748,12 @@ fr: send_every_must_be_a_list_of_days_of_the_week: doit être une liste de jours de la semaine. send_on_must_be_in_the_future: ne peut pas être dans le passé. cant_delete_default_folder: Le dossier par défaut ne peut pas être supprimé + explainer_and_item_must_be_from_the_same_team: L’article explicatif et l’élément doivent provenir du même espace de travail. + cant_apply_article_to_item_if_article_is_in_the_trash: Cet article se trouve dans la corbeille et ne peut donc pas être ajouté à ce groupe de médias. Restaurez-le d’abord de la corbeille. + shared_feed_imported_media_already_exist: |- + Aucun média ne peut être importé dans votre espace de travail. + Les médias sélectionnés pour l’importation existent déjà dans votre espace de travail dans les éléments suivants : + %{urls} info: messages: sent_to_trash_by_rule: '« %{item_title} » a été mis dans la corbeille par une règle d’automatisation.' diff --git a/config/locales/id.yml b/config/locales/id.yml index 92909adb21..6055d71152 100644 --- a/config/locales/id.yml +++ b/config/locales/id.yml @@ -73,6 +73,7 @@ id: description: 'Kesimpulan: sumber ini adalah identitas palsu' errors: messages: + extension_whitelist_error: ekstensi tidak didukung invalid_password: Kata sandi salah invalid_qrcode: Kode validasi salah extension_white_list_error: 'tidak dapat memiliki tipe format %{extension}, tipe-tipe yang diijinkan: %{allowed_types}' @@ -93,9 +94,10 @@ id: invalid_project_media_channel_value: Maaf, nilai saluran tidak didukung invalid_project_media_channel_update: Maaf, Anda tidak dapat memperbarui nilai saluran invalid_project_media_archived_value: Maaf, nilai yang diarsipkan tidak didukung - invalid_fact_check_language_value: Maaf, bahasa tidak didukung + invalid_article_language_value: Maaf, bahasa tidak didukung fact_check_empty_title_and_summary: Maaf, Anda harus mengisi judul atau ringkasan invalid_feed_saved_search_value: harus menjadi milik ruang kerja yang merupakan bagian dari umpan ini + platform_allowed_values_error: 'tidak dapat memiliki tipe format %{type}, tipe-tipe yang diijinkan: %{allowed_types}' activerecord: models: link: Tautan @@ -146,6 +148,10 @@ id: attributes: slug: slug_format: hanya menerima huruf, angka dan tanda hubung + user: + attributes: + email: + taken: Silakan periksa surel Anda. Jika tidak ada akun dengan surel tersebut, Anda seharusnya menerima surel konfirmasi. Jika Anda tidak menerima surel konfirmasi, coba atur ulang kata sandi Anda atau hubungi dukungan kami. messages: record_invalid: "%{errors}" improbable_phone: salah nomor @@ -274,7 +280,8 @@ id: account_exists: Akun ini sudah ada media_exists: Perihal ini sudah ada source_exists: Sumber ini sudah ada - email_exists: Silakan periksa email Anda. Jika tidak ada akun dengan email tersebut, Anda seharusnya menerima email konfirmasi. Jika Anda tidak menerima email konfirmasi, coba atur ulang kata sandi Anda atau hubungi dukungan kami. + email_exists: Silakan periksa surel Anda. Jika tidak ada akun dengan surel tersebut, Anda seharusnya menerima surel konfirmasi. Jika Anda tidak menerima surel konfirmasi, coba atur ulang kata sandi Anda atau hubungi dukungan kami. + error_password_not_strong: "Persyaratan kerumitan belum terpenuhi. Panjang harus 8-70 karakter dan mencakup setidaknya: 1 huruf besar, 1 huruf kecil, 1 angka, dan 1 karakter khusus." banned_user: 'Maaf, akun Anda telah dilarang dari %{app_name}. Mohon hubungi tim dukungan jika Anda merasa ini adalah sebuah kesalahan. ' devise: mailer: @@ -298,7 +305,7 @@ id: ignore: Jika Anda tidak ingin menerima undangan tersebut, abaikan surel ini. app_team: "%{app} Ruang kerja" failure: - unconfirmed: Mohon periksa surel Anda untuk memverifikasi akun Anda + unconfirmed: Silakan periksa surel Anda. Jika tidak ada akun dengan surel tersebut, Anda seharusnya menerima surel konfirmasi. Jika Anda tidak menerima surel konfirmasi, coba atur ulang kata sandi Anda atau hubungi dukungan kami. user_invitation: invited: "%{email} sudah diundang ke ruang kerja ini." member: "%{email} sudah menjadi anggota." @@ -413,6 +420,24 @@ id: approved_text: Permintaan Anda untuk bergabung ke ruang kerja %{team} pada %{app_name} telah disetujui. Sekarang Anda dapat pergi ke %{url} dan mulai berkontribusi. rejected_title: Permintaan Ditolak rejected_text: Maaf, permintaan Anda untuk bergabung ke ruang kerja %{team}pada %{app_name} tidak disetujui. + feed_invitation: + subject: "%{user} telah mengundang organisasi Anda untuk berkontribusi ke %{feed}" + hello: Halo %{email} + body: "%{name}, %{email} telah mengundang organisasi Anda untuk berkontribusi ke Umpan Bersama Check " + view_button: Lihat undangan + updated_terms: + subject: Persyaratan Layanan Check yang Diperbarui + hello: Halo %{name} + title: Persyaratan Layanan yang Diperbarui + body: Kami telah melakukan pembaruan pada Persyaratan Layanan kami untuk para pengguna Check. Kami mengirimkan notifikasi ini kepada semua orang yang memiliki akun Check. Penggunaan Check secara terus-menerus setelah pembaruan ini merupakan bentuk penerimaan terhadap Persyaratan Layanan kami yang telah diperbarui. + term_button: Persyaratan Layanan + more_info: Ini adalah pemberitahuan hukum wajib satu-kali yang dikirimkan kepada semua pengguna Check, bahkan ke mereka yang telah berhenti berlangganan melalui pengumuman opsional. + export_list: + hello: Halo %{name} + subject: Ekspor Data Check + body: Ekspor data Check yang Anda minta telah tersedia untuk diunduh. + button_label: Unduh Ekspor + footer: Tautan unduhan ini akan kedaluwarsa pada %{date}. mail_security: device_subject: 'Peringatan keamanan: Percobaan masuk ke %{app_name} dari %{browser} pada %{platform}' ip_subject: 'Peringatan keamanan: Percobaan masuk baru atau tidak biasa dari %{app_name}' @@ -558,15 +583,16 @@ id: bot_request_url_invalid: URL bot salah smooch_facebook_success: | Sukses! - Tipline Anda sekarang terhubung dengan Facebook Messenger. - Pesan yang diterima oleh Halaman Facebook ini akan ditangani oleh tipline. - Mohon muat ulang halaman pengaturan tipline Anda untuk melihat pengubahan baru. + Tipline Anda sekarang terhubung dengan profil. + Semua pesan yang diterima oleh profil ini akan ditangani oleh tipline. + Silakan muat ulang halaman pengaturan tipline Anda untuk melihat perubahan yang baru. smooch_twitter_success: | Sukses! Tipline Anda sekarang terhubung dengan Twitter. Semua pesan yang diterima oleh profil Twitter ini akan ditangani oleh tipline. - Mohon muat ulang halaman pengaturan tipline Anda untuk melihat pengubahan baru. + Silakan muat ulang halaman pengaturan tipline Anda untuk melihat perubahan yang baru. must_select_exactly_one_facebook_page: Silakan pilih salah satu halaman Facebook yang Anda ingin integrasikan dengan Tipline. + invalid_facebook_authdata: Tidak memungkinkan untuk mengautentikasi akun Facebook Anda, silakan hubungi tim dukungan kami. invalid_task_answer: Format jawaban dari tugas salah team_rule_name: Nama unik yang dapat mengidentifikasi apa yang aturan ini lakukan team_rule_project_ids: Terapkan pada folder @@ -718,6 +744,12 @@ id: send_every_must_be_a_list_of_days_of_the_week: harus berupa daftar hari dalam seminggu. send_on_must_be_in_the_future: tidak bisa ke masa lalu. cant_delete_default_folder: Folder bawaan tidak dapat dihapus + explainer_and_item_must_be_from_the_same_team: Penjelasan dan perihal harus berasal dari ruang kerja yang sama. + cant_apply_article_to_item_if_article_is_in_the_trash: Artikel ini ada di tong sampah sehingga ia tidak dapat ditambahkan ke klaster media ini. Silakan kembalikan terlebih dahulu dari tong sampah. + shared_feed_imported_media_already_exist: |- + Tidak ada media yang memenuhi syarat untuk diimpor ke ruang kerja Anda.. + Media yang dipilih untuk diimpor sudah ada di ruang kerja Anda dalam perihal-perihal berikut: + %{urls} info: messages: sent_to_trash_by_rule: '"%{item_title}" telah dibuang ke tong sampah oleh aturan otomasi.' From b3a26f01973e510aef8bfe154f6eb138428256ca Mon Sep 17 00:00:00 2001 From: Jay Joshua <7008757+jayjay-w@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:37:07 +0300 Subject: [PATCH 51/52] Use `Sidekiq.fake` instead of `Sidekiq.inline` by default (#2195) * Use `Sidekiq.fake` instead of `Sidekiq.inline` by default Use `Sidekiq.fake` instead of `Sidekiq.inline` by default for faster tests * Fix broken tests * Fixing test --------- Co-authored-by: Caio <117518+caiosba@users.noreply.github.com> --- test/contract/alegre_contract_test.rb | 1 + .../controllers/graphql_controller_10_test.rb | 424 ++++++------ test/controllers/graphql_controller_2_test.rb | 98 +-- test/controllers/graphql_controller_3_test.rb | 602 +++++++++--------- test/controllers/graphql_controller_test.rb | 8 +- .../project_medias_controller_test.rb | 1 + test/models/team_test.rb | 1 + test/test_helper.rb | 1 + 8 files changed, 587 insertions(+), 549 deletions(-) diff --git a/test/contract/alegre_contract_test.rb b/test/contract/alegre_contract_test.rb index 1b66514a06..406edf0d03 100644 --- a/test/contract/alegre_contract_test.rb +++ b/test/contract/alegre_contract_test.rb @@ -4,6 +4,7 @@ class Bot::AlegreContractTest < ActiveSupport::TestCase include Pact::Consumer::Minitest def setup + Sidekiq::Testing.fake! # Set up annotation types that we need that were previously provided by migrations create_metadata_stuff create_extracted_text_annotation_type diff --git a/test/controllers/graphql_controller_10_test.rb b/test/controllers/graphql_controller_10_test.rb index fb61cf884e..689b355c5d 100644 --- a/test/controllers/graphql_controller_10_test.rb +++ b/test/controllers/graphql_controller_10_test.rb @@ -181,31 +181,33 @@ def setup end test "should search for dynamic annotations" do - u = create_user - authenticate_with_user(u) - t = create_team slug: 'team' - create_team_user user: u, team: t - p = create_project team: t - - pm1 = create_project_media disable_es_callbacks: false, project: p - create_dynamic_annotation annotation_type: 'language', annotated: pm1, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false - pm2 = create_project_media disable_es_callbacks: false, project: p - create_dynamic_annotation annotation_type: 'language', annotated: pm2, set_fields: { language: 'pt' }.to_json, disable_es_callbacks: false - - sleep 2 - query = 'query CheckSearch { search(query: "{\"language\":[\"en\"]}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: 'team' } - assert_response :success - pmids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - assert_equal 1, pmids.size - assert_equal pm1.id, pmids[0] - - query = 'query CheckSearch { search(query: "{\"language\":[\"pt\"]}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: 'team' } - assert_response :success - pmids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - assert_equal 1, pmids.size - assert_equal pm2.id, pmids[0] + Sidekiq::Testing.inline! do + u = create_user + authenticate_with_user(u) + t = create_team slug: 'team' + create_team_user user: u, team: t + p = create_project team: t + + pm1 = create_project_media disable_es_callbacks: false, project: p + create_dynamic_annotation annotation_type: 'language', annotated: pm1, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + pm2 = create_project_media disable_es_callbacks: false, project: p + create_dynamic_annotation annotation_type: 'language', annotated: pm2, set_fields: { language: 'pt' }.to_json, disable_es_callbacks: false + + sleep 2 + query = 'query CheckSearch { search(query: "{\"language\":[\"en\"]}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: 'team' } + assert_response :success + pmids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + assert_equal 1, pmids.size + assert_equal pm1.id, pmids[0] + + query = 'query CheckSearch { search(query: "{\"language\":[\"pt\"]}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: 'team' } + assert_response :success + pmids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + assert_equal 1, pmids.size + assert_equal pm2.id, pmids[0] + end end test "should not remove logo when update team" do @@ -291,18 +293,20 @@ def setup end test "should empty trash" do - u = create_user - team = create_team - create_team_user team: team, user: u, role: 'admin' - p = create_project team: team - create_project_media archived: CheckArchivedFlags::FlagCodes::TRASHED, project: p - assert_equal 1, team.reload.trash_count - id = team.graphql_id - authenticate_with_user(u) - query = 'mutation { updateTeam(input: { clientMutationId: "1", id: "' + id + '", empty_trash: 1 }) { public_team { trash_count } } }' - post :create, params: { query: query, team: team.slug } - assert_response :success - assert_equal 0, JSON.parse(@response.body)['data']['updateTeam']['public_team']['trash_count'] + Sidekiq::Testing.inline! do + u = create_user + team = create_team + create_team_user team: team, user: u, role: 'admin' + p = create_project team: team + create_project_media archived: CheckArchivedFlags::FlagCodes::TRASHED, project: p + assert_equal 1, team.reload.trash_count + id = team.graphql_id + authenticate_with_user(u) + query = 'mutation { updateTeam(input: { clientMutationId: "1", id: "' + id + '", empty_trash: 1 }) { public_team { trash_count } } }' + post :create, params: { query: query, team: team.slug } + assert_response :success + assert_equal 0, JSON.parse(@response.body)['data']['updateTeam']['public_team']['trash_count'] + end end test "should provide empty fallback only if deleted status has no items" do @@ -364,155 +368,161 @@ def setup end test "should filter by link published date" do - RequestStore.store[:skip_cached_field_update] = false - u = create_user - t = create_team - create_team_user user: u, team: t, role: 'admin' - - WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] - url = 'http://test.com' - pender_url = CheckConfig.get('pender_url_private') + '/api/medias' - response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Sat Oct 31 15:11:49 +0000 2020"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response) - pm1 = create_project_media team: t, url: url, disable_es_callbacks: false ; sleep 1 - - WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] - url = 'http://test.com/2' - pender_url = CheckConfig.get('pender_url_private') + '/api/medias' - response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Fri Oct 23 14:41:05 +0000 2020"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response) - pm2 = create_project_media team: t, url: url, disable_es_callbacks: false ; sleep 1 - - authenticate_with_user(u) - - query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-30\",\"end_time\":\"2020-11-01\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } - - query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-20\",\"end_time\":\"2020-10-24\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } - - WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] - url = 'http://test.com' - pender_url = CheckConfig.get('pender_url_private') + '/api/medias' - response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Fri Oct 23 14:41:05 +0000 2020"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url, refresh: '1' } }).to_return(body: response) - pm1 = ProjectMedia.find(pm1.id) ; pm1.disable_es_callbacks = false ; pm1.refresh_media = true ; pm1.save! ; sleep 1 - - WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] - url = 'http://test.com/2' - pender_url = CheckConfig.get('pender_url_private') + '/api/medias' - response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Sat Oct 31 15:11:49 +0000 2020"}}' - WebMock.stub_request(:get, pender_url).with({ query: { url: url, refresh: '1' } }).to_return(body: response) - pm2 = ProjectMedia.find(pm2.id) ; pm2.disable_es_callbacks = false ; pm2.refresh_media = true ; pm2.save! ; sleep 1 - - query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-30\",\"end_time\":\"2020-11-01\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } - - query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-20\",\"end_time\":\"2020-10-24\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } + Sidekiq::Testing.inline! do + RequestStore.store[:skip_cached_field_update] = false + u = create_user + t = create_team + create_team_user user: u, team: t, role: 'admin' + + WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] + url = 'http://test.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Sat Oct 31 15:11:49 +0000 2020"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response) + pm1 = create_project_media team: t, url: url, disable_es_callbacks: false ; sleep 1 + + WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] + url = 'http://test.com/2' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Fri Oct 23 14:41:05 +0000 2020"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response) + pm2 = create_project_media team: t, url: url, disable_es_callbacks: false ; sleep 1 + + authenticate_with_user(u) + + query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-30\",\"end_time\":\"2020-11-01\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } + + query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-20\",\"end_time\":\"2020-10-24\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } + + WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] + url = 'http://test.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Fri Oct 23 14:41:05 +0000 2020"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url, refresh: '1' } }).to_return(body: response) + pm1 = ProjectMedia.find(pm1.id) ; pm1.disable_es_callbacks = false ; pm1.refresh_media = true ; pm1.save! ; sleep 1 + + WebMock.disable_net_connect! allow: [CheckConfig.get('elasticsearch_host').to_s + ':' + CheckConfig.get('elasticsearch_port').to_s, CheckConfig.get('storage_endpoint')] + url = 'http://test.com/2' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response = '{"type":"media","data":{"url":"' + url + '","type":"item","published_at":"Sat Oct 31 15:11:49 +0000 2020"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url, refresh: '1' } }).to_return(body: response) + pm2 = ProjectMedia.find(pm2.id) ; pm2.disable_es_callbacks = false ; pm2.refresh_media = true ; pm2.save! ; sleep 1 + + query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-30\",\"end_time\":\"2020-11-01\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } + + query = 'query CheckSearch { search(query: "{\"range\":{\"media_published_at\":{\"start_time\":\"2020-10-20\",\"end_time\":\"2020-10-24\"}}}") { id,medias(first:20){edges{node{dbid}}}}}'; + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm.dig('node', 'dbid') } + end end test "should search for tags using operator" do - u = create_user - t = create_team - create_team_user user: u, team: t, role: 'admin' - authenticate_with_user(u) - pm1 = create_project_media team: t, disable_es_callbacks: false - create_tag annotated: pm1, tag: 'test tag 1', disable_es_callbacks: false - pm2 = create_project_media team: t, disable_es_callbacks: false - create_tag annotated: pm2, tag: 'test tag 2', disable_es_callbacks: false - pm3 = create_project_media team: t, disable_es_callbacks: false - create_tag annotated: pm3, tag: 'test tag 1', disable_es_callbacks: false - create_tag annotated: pm3, tag: 'test tag 2', disable_es_callbacks: false - pm4 = create_project_media team: t, disable_es_callbacks: false - create_tag annotated: pm4, tag: 'test tag 3', disable_es_callbacks: false - pm5 = create_project_media team: t, disable_es_callbacks: false - sleep 2 - - query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"]}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 3, JSON.parse(@response.body)['data']['search']['medias']['edges'].size - - query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"],\"tags_operator\":\"or\"}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 3, JSON.parse(@response.body)['data']['search']['medias']['edges'].size - - query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"],\"tags_operator\":\"and\"}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal [pm3.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] } + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user user: u, team: t, role: 'admin' + authenticate_with_user(u) + pm1 = create_project_media team: t, disable_es_callbacks: false + create_tag annotated: pm1, tag: 'test tag 1', disable_es_callbacks: false + pm2 = create_project_media team: t, disable_es_callbacks: false + create_tag annotated: pm2, tag: 'test tag 2', disable_es_callbacks: false + pm3 = create_project_media team: t, disable_es_callbacks: false + create_tag annotated: pm3, tag: 'test tag 1', disable_es_callbacks: false + create_tag annotated: pm3, tag: 'test tag 2', disable_es_callbacks: false + pm4 = create_project_media team: t, disable_es_callbacks: false + create_tag annotated: pm4, tag: 'test tag 3', disable_es_callbacks: false + pm5 = create_project_media team: t, disable_es_callbacks: false + sleep 2 + + query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"]}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 3, JSON.parse(@response.body)['data']['search']['medias']['edges'].size + + query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"],\"tags_operator\":\"or\"}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 3, JSON.parse(@response.body)['data']['search']['medias']['edges'].size + + query = 'query CheckSearch { search(query: "{\"tags\":[\"test tag 1\",\"test tag 2\"],\"tags_operator\":\"and\"}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal [pm3.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |e| e['node']['dbid'] } + end end test "should search by user assigned to item" do - u = create_user is_admin: true - t = create_team - create_team_user user: u, team: t, role: 'admin' - authenticate_with_user(u) - - u1 = create_user - create_team_user user: u1, team: t - pm1 = create_project_media team: t, disable_es_callbacks: false - a1 = Assignment.create! user: u1, assigned: pm1.last_status_obj, disable_es_callbacks: false - u3 = create_user - create_team_user user: u3, team: t - Assignment.create! user: u3, assigned: pm1.last_status_obj, disable_es_callbacks: false - - u2 = create_user - create_team_user user: u2, team: t - pm2 = create_project_media team: t, disable_es_callbacks: false - a2 = Assignment.create! user: u2, assigned: pm2.last_status_obj, disable_es_callbacks: false - u4 = create_user - create_team_user user: u4, team: t - Assignment.create! user: u4, assigned: pm2.last_status_obj, disable_es_callbacks: false - - u5 = create_user - sleep 2 - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u1.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u2.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[\"ANY_VALUE\"]}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - ids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - assert_equal [pm1.id, pm2.id], ids.sort - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[\"NO_VALUE\"]}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - ids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - assert_empty ids - - a1.destroy! - a2.destroy! - sleep 2 - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u1.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } - - query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u2.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + Sidekiq::Testing.inline! do + u = create_user is_admin: true + t = create_team + create_team_user user: u, team: t, role: 'admin' + authenticate_with_user(u) + + u1 = create_user + create_team_user user: u1, team: t + pm1 = create_project_media team: t, disable_es_callbacks: false + a1 = Assignment.create! user: u1, assigned: pm1.last_status_obj, disable_es_callbacks: false + u3 = create_user + create_team_user user: u3, team: t + Assignment.create! user: u3, assigned: pm1.last_status_obj, disable_es_callbacks: false + + u2 = create_user + create_team_user user: u2, team: t + pm2 = create_project_media team: t, disable_es_callbacks: false + a2 = Assignment.create! user: u2, assigned: pm2.last_status_obj, disable_es_callbacks: false + u4 = create_user + create_team_user user: u4, team: t + Assignment.create! user: u4, assigned: pm2.last_status_obj, disable_es_callbacks: false + + u5 = create_user + sleep 2 + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u1.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u2.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[\"ANY_VALUE\"]}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + ids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + assert_equal [pm1.id, pm2.id], ids.sort + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[\"NO_VALUE\"]}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + ids = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + assert_empty ids + + a1.destroy! + a2.destroy! + sleep 2 + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u1.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + + query = 'query CheckSearch { search(query: "{\"assigned_to\":[' + u2.id.to_s + ',' + u5.id.to_s + ']}") { id,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] } + end end test "should not access GraphQL mutation if not authenticated" do @@ -521,32 +531,34 @@ def setup end test "should get project media assignments" do - u = create_user - u2 = create_user - t = create_team - create_team_user user: u, team: t, status: 'member' - create_team_user user: u2, team: t, status: 'member' - pm1 = create_project_media team: t - pm2 = create_project_media team: t - pm3 = create_project_media team: t - pm4 = create_project_media team: t - s1 = create_status status: 'in_progress', annotated: pm1 - s2 = create_status status: 'in_progress', annotated: pm2 - s3 = create_status status: 'in_progress', annotated: pm3 - s4 = create_status status: 'verified', annotated: pm4 - t1 = create_task annotated: pm1 - t2 = create_task annotated: pm3 - s1.assign_user(u.id) - s2.assign_user(u.id) - s3.assign_user(u.id) - s4.assign_user(u2.id) - authenticate_with_user(u) - post :create, params: { query: "query { me { assignments(first: 10) { edges { node { dbid, assignments(first: 10, user_id: #{u.id}, annotation_type: \"task\") { edges { node { dbid } } } } } } } }" } - data = JSON.parse(@response.body)['data']['me'] - assert_equal [pm3.id, pm2.id, pm1.id], data['assignments']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [t2.id], data['assignments']['edges'][0]['node']['assignments']['edges'].collect{ |x| x['node']['dbid'].to_i } - assert_equal [], data['assignments']['edges'][1]['node']['assignments']['edges'] - assert_equal [t1.id], data['assignments']['edges'][2]['node']['assignments']['edges'].collect{ |x| x['node']['dbid'].to_i } + Sidekiq::Testing.inline! do + u = create_user + u2 = create_user + t = create_team + create_team_user user: u, team: t, status: 'member' + create_team_user user: u2, team: t, status: 'member' + pm1 = create_project_media team: t + pm2 = create_project_media team: t + pm3 = create_project_media team: t + pm4 = create_project_media team: t + s1 = create_status status: 'in_progress', annotated: pm1 + s2 = create_status status: 'in_progress', annotated: pm2 + s3 = create_status status: 'in_progress', annotated: pm3 + s4 = create_status status: 'verified', annotated: pm4 + t1 = create_task annotated: pm1 + t2 = create_task annotated: pm3 + s1.assign_user(u.id) + s2.assign_user(u.id) + s3.assign_user(u.id) + s4.assign_user(u2.id) + authenticate_with_user(u) + post :create, params: { query: "query { me { assignments(first: 10) { edges { node { dbid, assignments(first: 10, user_id: #{u.id}, annotation_type: \"task\") { edges { node { dbid } } } } } } } }" } + data = JSON.parse(@response.body)['data']['me'] + assert_equal [pm3.id, pm2.id, pm1.id], data['assignments']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [t2.id], data['assignments']['edges'][0]['node']['assignments']['edges'].collect{ |x| x['node']['dbid'].to_i } + assert_equal [], data['assignments']['edges'][1]['node']['assignments']['edges'] + assert_equal [t1.id], data['assignments']['edges'][2]['node']['assignments']['edges'].collect{ |x| x['node']['dbid'].to_i } + end end test "should not get private team by slug" do diff --git a/test/controllers/graphql_controller_2_test.rb b/test/controllers/graphql_controller_2_test.rb index 27df89c0d3..bc37484208 100644 --- a/test/controllers/graphql_controller_2_test.rb +++ b/test/controllers/graphql_controller_2_test.rb @@ -352,56 +352,60 @@ def setup end test "should get number of results without similar items" do - u = create_user - authenticate_with_user(u) - t = create_team slug: 'team' - create_team_user user: u, team: t - pm1 = create_project_media team: t, quote: 'Test 1 Bar', disable_es_callbacks: false - pm2 = create_project_media team: t, quote: 'Test 2 Foo', disable_es_callbacks: false - create_relationship source_id: pm1.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type, disable_es_callbacks: false - pm3 = create_project_media team: t, quote: 'Test 3 Bar', disable_es_callbacks: false - pm4 = create_project_media team: t, quote: 'Test 4 Foo', disable_es_callbacks: false - create_relationship source_id: pm3.id, target_id: pm4.id, relationship_type: Relationship.confirmed_type, disable_es_callbacks: false - sleep 1 - - query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\"}") { number_of_results } }' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] - - query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\",\"show_similar\":false}") { number_of_results } }' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] - - query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\",\"show_similar\":true}") { number_of_results } }' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 4, JSON.parse(@response.body)['data']['search']['number_of_results'] - - query = 'query CheckSearch { search(query: "{\"keyword\":\"Foo\",\"show_similar\":true}") { number_of_results } }' - post :create, params: { query: query, team: 'team' } - assert_response :success - assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] + Sidekiq::Testing.inline! do + u = create_user + authenticate_with_user(u) + t = create_team slug: 'team' + create_team_user user: u, team: t + pm1 = create_project_media team: t, quote: 'Test 1 Bar', disable_es_callbacks: false + pm2 = create_project_media team: t, quote: 'Test 2 Foo', disable_es_callbacks: false + create_relationship source_id: pm1.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type, disable_es_callbacks: false + pm3 = create_project_media team: t, quote: 'Test 3 Bar', disable_es_callbacks: false + pm4 = create_project_media team: t, quote: 'Test 4 Foo', disable_es_callbacks: false + create_relationship source_id: pm3.id, target_id: pm4.id, relationship_type: Relationship.confirmed_type, disable_es_callbacks: false + sleep 1 + + query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\"}") { number_of_results } }' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] + + query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\",\"show_similar\":false}") { number_of_results } }' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] + + query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\",\"show_similar\":true}") { number_of_results } }' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 4, JSON.parse(@response.body)['data']['search']['number_of_results'] + + query = 'query CheckSearch { search(query: "{\"keyword\":\"Foo\",\"show_similar\":true}") { number_of_results } }' + post :create, params: { query: query, team: 'team' } + assert_response :success + assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results'] + end end test "should replace blank project media by another" do - u = create_user - t = create_team - create_team_user team: t, user: u, role: 'admin' - old = create_project_media team: t, media: Blank.create! - r = publish_report(old) - new = create_project_media team: t - authenticate_with_user(u) - - query = 'mutation { replaceProjectMedia(input: { clientMutationId: "1", project_media_to_be_replaced_id: "' + old.graphql_id + '", new_project_media_id: "' + new.graphql_id + '" }) { old_project_media_deleted_id, new_project_media { dbid } } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - data = JSON.parse(@response.body)['data']['replaceProjectMedia'] - assert_equal old.graphql_id, data['old_project_media_deleted_id'] - assert_equal new.id, data['new_project_media']['dbid'] - assert_nil ProjectMedia.find_by_id(old.id) - assert_equal r, new.get_dynamic_annotation('report_design') + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user team: t, user: u, role: 'admin' + old = create_project_media team: t, media: Blank.create! + r = publish_report(old) + new = create_project_media team: t + authenticate_with_user(u) + + query = 'mutation { replaceProjectMedia(input: { clientMutationId: "1", project_media_to_be_replaced_id: "' + old.graphql_id + '", new_project_media_id: "' + new.graphql_id + '" }) { old_project_media_deleted_id, new_project_media { dbid } } }' + post :create, params: { query: query, team: t.slug } + assert_response :success + data = JSON.parse(@response.body)['data']['replaceProjectMedia'] + assert_equal old.graphql_id, data['old_project_media_deleted_id'] + assert_equal new.id, data['new_project_media']['dbid'] + assert_nil ProjectMedia.find_by_id(old.id) + assert_equal r, new.get_dynamic_annotation('report_design') + end end test "should set and get Slack settings for team" do diff --git a/test/controllers/graphql_controller_3_test.rb b/test/controllers/graphql_controller_3_test.rb index c8183e8eb4..57606e9f1d 100644 --- a/test/controllers/graphql_controller_3_test.rb +++ b/test/controllers/graphql_controller_3_test.rb @@ -15,272 +15,280 @@ def setup end test "should filter and sort inside ElasticSearch" do - u = create_user is_admin: true - authenticate_with_user(u) - t1 = create_team - pm1a = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 - pm1b = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 - pm1b.disable_es_callbacks = false ; pm1b.updated_at = Time.now ; pm1b.save! ; sleep 1 - pm1a.disable_es_callbacks = false ; pm1a.updated_at = Time.now ; pm1a.save! ; sleep 1 - pm1c = create_project_media team: t1, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::TRASHED ; sleep 1 - t2 = create_team - pm2 = [] - 6.times do - pm2 << create_project_media(team: t2, disable_es_callbacks: false) - end - sleep 2 - - # Default sort criteria and order: recent added, descending - query = 'query CheckSearch { search(query: "{}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1b.id, pm1a.id], results - - # Another sort criteria and default order: recent activity, descending - query = 'query CheckSearch { search(query: "{\"sort\":\"recent_activity\"}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1a.id, pm1b.id], results - - # Default sorting criteria and custom order: recent added, ascending - query = 'query CheckSearch { search(query: "{\"sort_type\":\"asc\"}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1a.id, pm1b.id], results - - # Another search criteria and another order: recent activity, ascending - query = 'query CheckSearch { search(query: "{\"sort\":\"recent_activity\",\"sort_type\":\"asc\"}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1b.id, pm1a.id], results - - # Get archived items - query = 'query CheckSearch { search(query: "{\"archived\":1}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1c.id], results - - # Relationships - pm1e = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 - pm1f = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 1' ; sleep 1 - pm1g = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 2' ; sleep 1 - pm1h = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 3' ; sleep 1 - create_relationship source_id: pm1e.id, target_id: pm1f.id, disable_es_callbacks: false ; sleep 1 - create_relationship source_id: pm1e.id, target_id: pm1g.id, disable_es_callbacks: false ; sleep 1 - create_relationship source_id: pm1e.id, target_id: pm1h.id, disable_es_callbacks: false ; sleep 1 - query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"show_similar\":true}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t1.slug } - assert_response :success - response = JSON.parse(@response.body)['data']['search'] - assert_equal 3, response['number_of_results'] - results = response['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1f.id, pm1g.id, pm1h.id].sort, results.sort - - # Paginate, page 1 - query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":0}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t2.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2[5].id, pm2[4].id], results - - # Paginate, page 2 - query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":2}") {medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t2.slug } - assert_response :success - results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2[3].id, pm2[2].id], results - - # Paginate, page 3 - query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":4}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t2.slug } - assert_response :success - response = JSON.parse(@response.body)['data']['search'] - assert_equal 6, response['number_of_results'] - results = response['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2[1].id, pm2[0].id], results - end - - test "should filter by date range" do - u = create_user - t = create_team - create_team_user user: u, team: t, role: 'admin' - p = create_project team: t - - Time.stubs(:now).returns(Time.new(2019, 05, 18, 13, 00)) - pm1 = create_project_media project: p, quote: 'Test A', disable_es_callbacks: false - pm1.update_attribute(:updated_at, Time.new(2019, 05, 19)) - sleep 1 - - Time.stubs(:now).returns(Time.new(2019, 05, 20, 13, 00)) - pm2 = create_project_media project: p, quote: 'Test B', disable_es_callbacks: false - pm2.update_attribute(:updated_at, Time.new(2019, 05, 21, 12, 00)) - sleep 1 - - Time.stubs(:now).returns(Time.new(2019, 05, 22, 13, 00)) - pm3 = create_project_media project: p, quote: 'Test C', disable_es_callbacks: false - pm3.update_attribute(:updated_at, Time.new(2019, 05, 23)) - sleep 1 - - Time.unstub(:now) - authenticate_with_user(u) - queries = [] - - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"start_time\":\"2019-05-19\",\"end_time\":\"2019-05-24\"},\"updated_at\":{\"start_time\":\"2019-05-20\",\"end_time\":\"2019-05-21\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"start_time\":\"2019-05-19\",\"end_time\":\"2019-05-24\"},\"updated_at\":{\"start_time\":\"2019-05-20\",\"end_time\":\"2019-05-21\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - - queries.each do |query| - post :create, params: { query: query, team: t.slug } + Sidekiq::Testing.inline! do + u = create_user is_admin: true + authenticate_with_user(u) + t1 = create_team + pm1a = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 + pm1b = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 + pm1b.disable_es_callbacks = false ; pm1b.updated_at = Time.now ; pm1b.save! ; sleep 1 + pm1a.disable_es_callbacks = false ; pm1a.updated_at = Time.now ; pm1a.save! ; sleep 1 + pm1c = create_project_media team: t1, disable_es_callbacks: false, archived: CheckArchivedFlags::FlagCodes::TRASHED ; sleep 1 + t2 = create_team + pm2 = [] + 6.times do + pm2 << create_project_media(team: t2, disable_es_callbacks: false) + end + sleep 2 + + # Default sort criteria and order: recent added, descending + query = 'query CheckSearch { search(query: "{}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2.id], results - end - end - - - test "should filter by date range with less_than and more_than options" do - u = create_user - t = create_team - create_team_user user: u, team: t, role: 'admin' - p = create_project team: t - - Time.stubs(:now).returns(Time.new - 5.week) - pm1 = create_project_media project: p, quote: 'Test A', disable_es_callbacks: false - Time.stubs(:now).returns(Time.new - 3.week) - pm2 = create_project_media project: p, quote: 'Test B', disable_es_callbacks: false - sleep 1 + assert_equal [pm1b.id, pm1a.id], results - Time.unstub(:now) - authenticate_with_user(u) + # Another sort criteria and default order: recent activity, descending + query = 'query CheckSearch { search(query: "{\"sort\":\"recent_activity\"}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm1a.id, pm1b.id], results - queries = [] - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - queries.each do |query| - post :create, params: { query: query, team: t.slug } + # Default sorting criteria and custom order: recent added, ascending + query = 'query CheckSearch { search(query: "{\"sort_type\":\"asc\"}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2.id], results - end - # Filter by more_than - queries = [] - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"more_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"more_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - queries.each do |query| - post :create, params: { query: query, team: t.slug } + assert_equal [pm1a.id, pm1b.id], results + + # Another search criteria and another order: recent activity, ascending + query = 'query CheckSearch { search(query: "{\"sort\":\"recent_activity\",\"sort_type\":\"asc\"}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1.id], results - end - # query with period_type = w - queries = [] - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"4\",\"period_type\":\"w\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"4\",\"period_type\":\"w\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - queries.each do |query| - post :create, params: { query: query, team: t.slug } + assert_equal [pm1b.id, pm1a.id], results + + # Get archived items + query = 'query CheckSearch { search(query: "{\"archived\":1}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm2.id], results - end - # query with period_type = y - queries = [] - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"y\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"y\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - queries.each do |query| - post :create, params: { query: query, team: t.slug } + assert_equal [pm1c.id], results + + # Relationships + pm1e = create_project_media team: t1, disable_es_callbacks: false ; sleep 1 + pm1f = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 1' ; sleep 1 + pm1g = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 2' ; sleep 1 + pm1h = create_project_media team: t1, disable_es_callbacks: false, media: nil, quote: 'Test 3' ; sleep 1 + create_relationship source_id: pm1e.id, target_id: pm1f.id, disable_es_callbacks: false ; sleep 1 + create_relationship source_id: pm1e.id, target_id: pm1g.id, disable_es_callbacks: false ; sleep 1 + create_relationship source_id: pm1e.id, target_id: pm1h.id, disable_es_callbacks: false ; sleep 1 + query = 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"show_similar\":true}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t1.slug } + assert_response :success + response = JSON.parse(@response.body)['data']['search'] + assert_equal 3, response['number_of_results'] + results = response['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm1f.id, pm1g.id, pm1h.id].sort, results.sort + + # Paginate, page 1 + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":0}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t2.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_equal [pm1.id, pm2.id], results.sort - end - # query with period_type = d - queries = [] - # query on ES - queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"7\",\"period_type\":\"d\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - # query on PG - queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"7\",\"period_type\":\"d\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' - queries.each do |query| - post :create, params: { query: query, team: t.slug } + assert_equal [pm2[5].id, pm2[4].id], results + + # Paginate, page 2 + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":2}") {medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t2.slug } assert_response :success results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - assert_empty results + assert_equal [pm2[3].id, pm2[2].id], results + + # Paginate, page 3 + query = 'query CheckSearch { search(query: "{\"eslimit\":2,\"esoffset\":4}") {number_of_results,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t2.slug } + assert_response :success + response = JSON.parse(@response.body)['data']['search'] + assert_equal 6, response['number_of_results'] + results = response['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm2[1].id, pm2[0].id], results + end + end + + test "should filter by date range" do + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user user: u, team: t, role: 'admin' + p = create_project team: t + + Time.stubs(:now).returns(Time.new(2019, 05, 18, 13, 00)) + pm1 = create_project_media project: p, quote: 'Test A', disable_es_callbacks: false + pm1.update_attribute(:updated_at, Time.new(2019, 05, 19)) + sleep 1 + + Time.stubs(:now).returns(Time.new(2019, 05, 20, 13, 00)) + pm2 = create_project_media project: p, quote: 'Test B', disable_es_callbacks: false + pm2.update_attribute(:updated_at, Time.new(2019, 05, 21, 12, 00)) + sleep 1 + + Time.stubs(:now).returns(Time.new(2019, 05, 22, 13, 00)) + pm3 = create_project_media project: p, quote: 'Test C', disable_es_callbacks: false + pm3.update_attribute(:updated_at, Time.new(2019, 05, 23)) + sleep 1 + + Time.unstub(:now) + authenticate_with_user(u) + queries = [] + + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"start_time\":\"2019-05-19\",\"end_time\":\"2019-05-24\"},\"updated_at\":{\"start_time\":\"2019-05-20\",\"end_time\":\"2019-05-21\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"start_time\":\"2019-05-19\",\"end_time\":\"2019-05-24\"},\"updated_at\":{\"start_time\":\"2019-05-20\",\"end_time\":\"2019-05-21\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm2.id], results + end + end + end + + + test "should filter by date range with less_than and more_than options" do + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user user: u, team: t, role: 'admin' + p = create_project team: t + + Time.stubs(:now).returns(Time.new - 5.week) + pm1 = create_project_media project: p, quote: 'Test A', disable_es_callbacks: false + Time.stubs(:now).returns(Time.new - 3.week) + pm2 = create_project_media project: p, quote: 'Test B', disable_es_callbacks: false + sleep 1 + + Time.unstub(:now) + authenticate_with_user(u) + + queries = [] + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm2.id], results + end + # Filter by more_than + queries = [] + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"more_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"more_than\",\"period\":\"1\",\"period_type\":\"m\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm1.id], results + end + # query with period_type = w + queries = [] + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"4\",\"period_type\":\"w\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"4\",\"period_type\":\"w\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm2.id], results + end + # query with period_type = y + queries = [] + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"y\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"1\",\"period_type\":\"y\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_equal [pm1.id, pm2.id], results.sort + end + # query with period_type = d + queries = [] + # query on ES + queries << 'query CheckSearch { search(query: "{\"keyword\":\"Test\", \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"7\",\"period_type\":\"d\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + # query on PG + queries << 'query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + '], \"range\": {\"created_at\":{\"condition\":\"less_than\",\"period\":\"7\",\"period_type\":\"d\"},\"timezone\":\"America/Bahia\"}}") { id,medias(first:20){edges{node{dbid}}}}}' + queries.each do |query| + post :create, params: { query: query, team: t.slug } + assert_response :success + results = JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_empty results + end end end test "should retrieve information for grid" do - RequestStore.store[:skip_cached_field_update] = false - u = create_user - authenticate_with_user(u) - t = create_team slug: 'team' - create_team_user user: u, team: t - p = create_project team: t - m = create_uploaded_image - pm = create_project_media project: p, user: create_user, media: m, disable_es_callbacks: false - info = { title: random_string, content: random_string }; pm.analysis = info; pm.save! - create_tipline_request team_id: t.id, associated: pm, smooch_data: {} - pm2 = create_project_media project: p - r = create_relationship source_id: pm.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type - create_tipline_request team_id: t.id, associated: pm2, smooch_data: {} - create_claim_description project_media: pm, description: 'Test' - - sleep 10 - - query = ' - query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + ']}") { - id - number_of_results - medias(first: 1) { - edges { - node { - id - dbid - picture - title - description - virality - demand - linked_items_count - type - status - first_seen: created_at - last_seen + Sidekiq::Testing.inline! do + RequestStore.store[:skip_cached_field_update] = false + u = create_user + authenticate_with_user(u) + t = create_team slug: 'team' + create_team_user user: u, team: t + p = create_project team: t + m = create_uploaded_image + pm = create_project_media project: p, user: create_user, media: m, disable_es_callbacks: false + info = { title: random_string, content: random_string }; pm.analysis = info; pm.save! + create_tipline_request team_id: t.id, associated: pm, smooch_data: {} + pm2 = create_project_media project: p + r = create_relationship source_id: pm.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type + create_tipline_request team_id: t.id, associated: pm2, smooch_data: {} + create_claim_description project_media: pm, description: 'Test' + + sleep 10 + + query = ' + query CheckSearch { search(query: "{\"projects\":[' + p.id.to_s + ']}") { + id + number_of_results + medias(first: 1) { + edges { + node { + id + dbid + picture + title + description + virality + demand + linked_items_count + type + status + first_seen: created_at + last_seen + } } } - } - }} - ' + }} + ' - assert_queries 25, '<=' do - post :create, params: { query: query, team: 'team' } - end + assert_queries 25, '<=' do + post :create, params: { query: query, team: 'team' } + end - assert_response :success - result = JSON.parse(@response.body)['data']['search'] - assert_equal 1, result['number_of_results'] - assert_equal 1, result['medias']['edges'].size - result['medias']['edges'].each do |pm_node| - pm = pm_node['node'] - assert_equal 'Test', pm['title'] - assert_equal 'Test', pm['description'] - assert_equal 0, pm['virality'] - assert_equal 2, pm['linked_items_count'] - assert_equal 'UploadedImage', pm['type'] - assert_not_equal pm['first_seen'], pm['last_seen'] - assert_equal 2, pm['demand'] + assert_response :success + result = JSON.parse(@response.body)['data']['search'] + assert_equal 1, result['number_of_results'] + assert_equal 1, result['medias']['edges'].size + result['medias']['edges'].each do |pm_node| + pm = pm_node['node'] + assert_equal 'Test', pm['title'] + assert_equal 'Test', pm['description'] + assert_equal 0, pm['virality'] + assert_equal 2, pm['linked_items_count'] + assert_equal 'UploadedImage', pm['type'] + assert_not_equal pm['first_seen'], pm['last_seen'] + assert_equal 2, pm['demand'] + end end end @@ -307,22 +315,24 @@ def setup end test "should return updated offset from ES" do - RequestStore.store[:skip_cached_field_update] = false - u = create_user is_admin: true - authenticate_with_user(u) - t = create_team - pm1 = create_project_media team: t, disable_es_callbacks: false - create_relationship source_id: pm1.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type - pm2 = create_project_media team: t, disable_es_callbacks: false - create_relationship source_id: pm2.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type - create_relationship source_id: pm2.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type - sleep 2 - query = 'query CheckSearch { search(query: "{\"sort\":\"related\",\"id\":' + pm1.id.to_s + ',\"esoffset\":0,\"eslimit\":1}") {item_navigation_offset,medias(first:20){edges{node{dbid}}}}}' - post :create, params: { query: query, team: t.slug } - assert_response :success - response = JSON.parse(@response.body)['data']['search'] - assert_equal pm1.id, response['medias']['edges'][0]['node']['dbid'] - assert_equal 1, response['item_navigation_offset'] + Sidekiq::Testing.inline! do + RequestStore.store[:skip_cached_field_update] = false + u = create_user is_admin: true + authenticate_with_user(u) + t = create_team + pm1 = create_project_media team: t, disable_es_callbacks: false + create_relationship source_id: pm1.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type + pm2 = create_project_media team: t, disable_es_callbacks: false + create_relationship source_id: pm2.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type + create_relationship source_id: pm2.id, target_id: create_project_media(team: t).id, relationship_type: Relationship.confirmed_type + sleep 2 + query = 'query CheckSearch { search(query: "{\"sort\":\"related\",\"id\":' + pm1.id.to_s + ',\"esoffset\":0,\"eslimit\":1}") {item_navigation_offset,medias(first:20){edges{node{dbid}}}}}' + post :create, params: { query: query, team: t.slug } + assert_response :success + response = JSON.parse(@response.body)['data']['search'] + assert_equal pm1.id, response['medias']['edges'][0]['node']['dbid'] + assert_equal 1, response['item_navigation_offset'] + end end test "should get requests from media" do @@ -362,45 +372,49 @@ def setup end test "should filter by user in ElasticSearch" do - u = create_user - t = create_team - create_team_user user: u, team: t - pm = create_project_media team: t, quote: 'This is a test', media: nil, user: u, disable_es_callbacks: false - create_project_media team: t, user: u, disable_es_callbacks: false - create_project_media team: t, disable_es_callbacks: false - sleep 1 - authenticate_with_user(u) - - query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"users\":[' + u.id.to_s + ']}") { medias(first: 10) { edges { node { dbid } } } } }' - post :create, params: { query: query, team: t.slug } + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user user: u, team: t + pm = create_project_media team: t, quote: 'This is a test', media: nil, user: u, disable_es_callbacks: false + create_project_media team: t, user: u, disable_es_callbacks: false + create_project_media team: t, disable_es_callbacks: false + sleep 1 + authenticate_with_user(u) + + query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"users\":[' + u.id.to_s + ']}") { medias(first: 10) { edges { node { dbid } } } } }' + post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + assert_response :success + assert_equal [pm.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + end end test "should filter by read in ElasticSearch" do - u = create_user - t = create_team - create_team_user user: u, team: t - pm1 = create_project_media team: t, quote: 'This is a test', media: nil, read: true, disable_es_callbacks: false - pm2 = create_project_media team: t, quote: 'This is another test', media: nil, disable_es_callbacks: false - pm3 = create_project_media quote: 'This is another test', media: nil, disable_es_callbacks: false - sleep 1 - authenticate_with_user(u) - - query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"read\":[1]}") { medias(first: 10) { edges { node { dbid } } } } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + Sidekiq::Testing.inline! do + u = create_user + t = create_team + create_team_user user: u, team: t + pm1 = create_project_media team: t, quote: 'This is a test', media: nil, read: true, disable_es_callbacks: false + pm2 = create_project_media team: t, quote: 'This is another test', media: nil, disable_es_callbacks: false + pm3 = create_project_media quote: 'This is another test', media: nil, disable_es_callbacks: false + sleep 1 + authenticate_with_user(u) + + query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"read\":[1]}") { medias(first: 10) { edges { node { dbid } } } } }' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm1.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"read\":[0]}") { medias(first: 10) { edges { node { dbid } } } } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } + query = 'query CheckSearch { search(query: "{\"keyword\":\"test\",\"read\":[0]}") { medias(first: 10) { edges { node { dbid } } } } }' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm2.id], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] } - query = 'query CheckSearch { search(query: "{\"keyword\":\"test\"}") { medias(first: 10) { edges { node { dbid } } } } }' - post :create, params: { query: query, team: t.slug } - assert_response :success - assert_equal [pm1.id, pm2.id].sort, JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] }.sort + query = 'query CheckSearch { search(query: "{\"keyword\":\"test\"}") { medias(first: 10) { edges { node { dbid } } } } }' + post :create, params: { query: query, team: t.slug } + assert_response :success + assert_equal [pm1.id, pm2.id].sort, JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |x| x['node']['dbid'] }.sort + end end end diff --git a/test/controllers/graphql_controller_test.rb b/test/controllers/graphql_controller_test.rb index 786141273f..0da24ff8a3 100644 --- a/test/controllers/graphql_controller_test.rb +++ b/test/controllers/graphql_controller_test.rb @@ -242,7 +242,9 @@ def setup end test "should destroy project" do - assert_graphql_destroy('project') + Sidekiq::Testing.inline! do + assert_graphql_destroy('project') + end end test "should create source" do @@ -262,7 +264,9 @@ def setup end test "should destroy team" do - assert_graphql_destroy('team') + Sidekiq::Testing.inline! do + assert_graphql_destroy('team') + end end test "should update user" do diff --git a/test/controllers/project_medias_controller_test.rb b/test/controllers/project_medias_controller_test.rb index ac9dbad953..becc3d2f92 100644 --- a/test/controllers/project_medias_controller_test.rb +++ b/test/controllers/project_medias_controller_test.rb @@ -3,6 +3,7 @@ class ProjectMediasControllerTest < ActionController::TestCase def setup super + Sidekiq::Testing.fake! @controller = Api::V1::ProjectMediasController.new @request.env["devise.mapping"] = Devise.mappings[:api_user] sign_out('user') diff --git a/test/models/team_test.rb b/test/models/team_test.rb index 4134f078cd..6f0601e13b 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -1073,6 +1073,7 @@ def setup value[:statuses][1][:locales][:en][:label] = 'Custom Status 2 Changed' t.media_verification_statuses = value t.save! + Sidekiq::Worker.drain_all assert_equal 'Custom Status 2 Changed', r.reload.data.dig('options', 'status_label') end diff --git a/test/test_helper.rb b/test/test_helper.rb index 78bd12412e..9314263880 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -159,6 +159,7 @@ def before_all # This will run before any test def setup + Sidekiq::Testing.fake! [Account, Media, ProjectMedia, User, Source, Annotation, Team, TeamUser, Relationship, Project, TiplineResource, TiplineRequest].each{ |klass| klass.delete_all } # Some of our non-GraphQL tests rely on behavior that this requires. As a result, From f5139542323609293d49a625f4a9d45122330c5b Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 5 Feb 2025 10:45:09 -0300 Subject: [PATCH 52/52] Bumping version number --- config/initializers/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/version.rb b/config/initializers/version.rb index 749280e526..cb5e7b7e76 100644 --- a/config/initializers/version.rb +++ b/config/initializers/version.rb @@ -1 +1 @@ -VERSION = 'v0.185.3' +VERSION = 'v0.186.0'