diff --git a/app/lib/check_elastic_search.rb b/app/lib/check_elastic_search.rb index 77bbda06cb..b43a44ed9c 100644 --- a/app/lib/check_elastic_search.rb +++ b/app/lib/check_elastic_search.rb @@ -23,7 +23,7 @@ def create_elasticsearch_doc_bg(options) ms.attributes[:updated_at] = self.updated_at.utc ms.attributes[:source_id] = self.source_id # Intial nested objects with [] - ['comments', 'tags', 'task_responses', 'assigned_user_ids', 'requests'].each{ |f| ms.attributes[f] = [] } + ['tags', 'task_responses', 'assigned_user_ids', 'requests'].each{ |f| ms.attributes[f] = [] } self.add_nested_objects(ms) if options[:force_creation] self.add_extra_elasticsearch_data(ms) $repository.save(ms) @@ -37,6 +37,13 @@ def update_elasticsearch_doc(keys, data = {}, pm_id = nil, skip_get_data = false ElasticSearchWorker.perform_in(1.second, YAML::dump(model), YAML::dump(options), 'update_doc') end + def remove_fields_from_elasticsearch_doc(keys, pm_id) + return if self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] + options = { keys: keys, pm_id: pm_id } + model = { klass: self.class.name, id: self.id } + ElasticSearchWorker.perform_in(1.second, YAML::dump(model), YAML::dump(options), 'remove_fields') + end + def update_recent_activity(obj) # update `updated_at` date for both PG & ES updated_at = Time.now @@ -61,8 +68,14 @@ def update_elasticsearch_doc_bg(options) if fields.count create_doc_if_not_exists(options) sleep 1 - client = $repository.client - client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { doc: fields } + $repository.client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { doc: fields } + end + end + + def remove_fields_from_elasticsearch_doc_bg(options) + options[:keys].each do |k| + source = "ctx._source.remove('#{k}')" + $repository.client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { script: { source: source } } end end @@ -175,9 +188,8 @@ def destroy_elasticsearch_doc(options) def destroy_elasticsearch_doc_nested(options) nested_type = options[:es_type] begin - client = $repository.client source = "for (int i = 0; i < ctx._source.#{nested_type}.size(); i++) { if(ctx._source.#{nested_type}[i].id == params.id){ctx._source.#{nested_type}.remove(i);}}" - client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], + $repository.client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { script: { source: source, params: { id: options[:model_id] } } } rescue Rails.logger.info "[ES destroy] doc with id #{options[:doc_id]} not exists" diff --git a/app/models/claim_description.rb b/app/models/claim_description.rb index a740453215..cd9352ea83 100644 --- a/app/models/claim_description.rb +++ b/app/models/claim_description.rb @@ -35,15 +35,13 @@ def text_fields end def article_elasticsearch_data(action = 'create_or_update') - return if self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] - data = action == 'destroy' ? { - 'claim_description_content' => '', - 'claim_description_context' => '' - } : { - 'claim_description_content' => self.description, - 'claim_description_context' => self.context - } - self.index_in_elasticsearch(data) + return if self.project_media_id.nil? || self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] + if action == 'destroy' + self.remove_fields_from_elasticsearch_doc(['claim_description_content', 'claim_description_context'], self.project_media_id) + else + data = { 'claim_description_content' => self.description, 'claim_description_context' => self.context } + self.index_in_elasticsearch(self.project_media_id, data) + end end def project_media_was @@ -91,6 +89,14 @@ def update_report fact_check.report_status = 'paused' fact_check.save! end + # update ES + # Remove claim_description fields + self.remove_fields_from_elasticsearch_doc(['claim_description_content', 'claim_description_context'], pm.id) + # clear fact-check values + unless self.fact_check.nil? + data = { 'fact_check_title' => '', 'fact_check_summary' => '', 'fact_check_url' => '', 'fact_check_languages' => [] } + self.fact_check.index_in_elasticsearch(pm.id, data) + end end end diff --git a/app/models/concerns/article.rb b/app/models/concerns/article.rb index ee981866b5..0e42f206f3 100644 --- a/app/models/concerns/article.rb +++ b/app/models/concerns/article.rb @@ -51,11 +51,8 @@ def notify_bots protected - def index_in_elasticsearch(data) - # touch project media to update `updated_at` date - pm = self.project_media - return if pm.nil? - pm = ProjectMedia.find_by_id(pm.id) + def index_in_elasticsearch(pm_id, data) + pm = ProjectMedia.find_by_id(pm_id) unless pm.nil? updated_at = Time.now pm.update_columns(updated_at: updated_at) diff --git a/app/models/fact_check.rb b/app/models/fact_check.rb index 7ca6aebd67..cd84cf548e 100644 --- a/app/models/fact_check.rb +++ b/app/models/fact_check.rb @@ -152,7 +152,7 @@ def update_report end def article_elasticsearch_data(action = 'create_or_update') - return if self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] + return if self.project_media.nil? || self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] data = action == 'destroy' ? { 'fact_check_title' => '', 'fact_check_summary' => '', @@ -164,7 +164,7 @@ def article_elasticsearch_data(action = 'create_or_update') 'fact_check_url' => self.url, 'fact_check_languages' => [self.language] } - self.index_in_elasticsearch(data) + self.index_in_elasticsearch(self.project_media.id, data) end def set_initial_rating diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 4ddd6cb6f4..357ce0c1ef 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -617,9 +617,6 @@ def add_extra_elasticsearch_data(ms) end def add_nested_objects(ms) - # comments - comments = self.annotations('comment') - ms.attributes[:comments] = comments.collect{|c| {id: c.id, text: c.text}} # tags tags = self.get_annotations('tag').map(&:load) ms.attributes[:tags] = tags.collect{|t| {id: t.id, tag: t.tag_text}} diff --git a/app/workers/elastic_search_worker.rb b/app/workers/elastic_search_worker.rb index 87ca4da87c..33a79d5650 100644 --- a/app/workers/elastic_search_worker.rb +++ b/app/workers/elastic_search_worker.rb @@ -15,6 +15,7 @@ def perform(model_data, options, type) ops = { 'create_doc' => 'create_elasticsearch_doc_bg', 'update_doc' => 'update_elasticsearch_doc_bg', + 'remove_fields' => 'remove_fields_from_elasticsearch_doc_bg', 'update_doc_team' => 'update_elasticsearch_doc_team_bg', 'create_update_doc_nested' => 'create_update_nested_obj_bg', 'destroy_doc' => 'destroy_elasticsearch_doc', diff --git a/lib/tasks/data/project_media.rake b/lib/tasks/data/project_media.rake index 19cdb7ef54..90b88a43a5 100644 --- a/lib/tasks/data/project_media.rake +++ b/lib/tasks/data/project_media.rake @@ -67,18 +67,6 @@ namespace :check do pm_responses end - def self.comments(_team, pm_ids) - pm_comments = Hash.new {|hash, key| hash[key] = [] } - Comment.where(annotation_type: 'comment', annotated_id: pm_ids, annotated_type: 'ProjectMedia') - .find_each do |c| - pm_comments[c.annotated_id] << { - id: c.id, - text: c.text - } - end - pm_comments - end - def self.tags(_team, pm_ids) pm_tags = Hash.new {|hash, key| hash[key] = [] } Tag.where(annotation_type: 'tag', annotated_id: pm_ids, annotated_type: 'ProjectMedia') diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index 3c22995bde..14993cc7bd 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -16,13 +16,18 @@ def setup pm2 = create_project_media team: t, disable_es_callbacks: false pm3 = create_project_media team: t, disable_es_callbacks: false pm4 = create_project_media team: t, disable_es_callbacks: false - create_claim_description project_media: pm, disable_es_callbacks: false + cd = create_claim_description project_media: pm, disable_es_callbacks: false create_claim_description project_media: pm3, disable_es_callbacks: false - sleep 2 + sleep 1 results = CheckSearch.new({ has_claim: ['ANY_VALUE'] }.to_json) assert_equal [pm.id, pm3.id], results.medias.map(&:id).sort results = CheckSearch.new({ has_claim: ['NO_VALUE'] }.to_json) assert_equal [pm2.id, pm4.id], results.medias.map(&:id).sort + cd.project_media_id = nil + cd.save! + sleep 1 + results = CheckSearch.new({ has_claim: ['NO_VALUE'] }.to_json) + assert_equal [pm.id, pm2.id, pm4.id], results.medias.map(&:id).sort end end