Merge branch 'develop' into CV2-4072-review-fields-of-types-team-type…

…-and-user-type

app/graph/mutations/tipline_message_mutations.rb

@@ -6,10 +6,10 @@ class Send < Mutations::BaseMutation
     field :success, GraphQL::Types::Boolean, null: true
 
     def resolve(in_reply_to_id: nil, message: nil)
-      request = Annotation.find(in_reply_to_id).load
+      request = TiplineRequest.find(in_reply_to_id)
       ability = context[:ability] || Ability.new
       success = false
-      if Team.current&.id && User.current&.id && ability.can?(:send, TiplineMessage.new(team: Team.current)) && request.annotated.team_id == Team.current.id
+      if Team.current&.id && User.current&.id && ability.can?(:send, TiplineMessage.new(team: Team.current)) && request.team_id == Team.current.id
         success = Bot::Smooch.reply_to_request_with_custom_message(request, message)
       end
       { success: success }

app/graph/types/tipline_request_type.rb

@@ -4,10 +4,9 @@ class TiplineRequestType < DefaultObject
   implements GraphQL::Types::Relay::Node
 
   field :dbid, GraphQL::Types::Int, null: true
-  field :value_json, JsonStringType, null: true
-  field :annotation, AnnotationType, null: true
-  field :annotation_id, GraphQL::Types::Int, null: true
-  field :associated_graphql_id, GraphQL::Types::String, null: true
+  field :associated_id, GraphQL::Types::Int, null: true
+  field :associated_type, GraphQL::Types::String, null: true
+  field :smooch_data, JsonStringType, null: true
   field :smooch_user_slack_channel_url, GraphQL::Types::String, null: true
   field :smooch_user_external_identifier, GraphQL::Types::String, null: true
   field :smooch_report_received_at, GraphQL::Types::Int, null: true
@@ -16,4 +15,5 @@ class TiplineRequestType < DefaultObject
   field :smooch_report_sent_at, GraphQL::Types::Int, null: true
   field :smooch_report_correction_sent_at, GraphQL::Types::Int, null: true
   field :smooch_request_type, GraphQL::Types::String, null: true
+  field :associated_graphql_id, GraphQL::Types::String, null: true
 end

app/lib/smooch_nlu.rb

@@ -1,6 +1,6 @@
 class SmoochNlu
-  class SmoochBotNotInstalledError < ::ArgumentError
-  end
+  class SmoochBotNotInstalledError < ::ArgumentError; end
+  class SmoochNluError < ::StandardError; end
 
   # FIXME: Make it more flexible
   # FIXME: Once we support paraphrase-multilingual-mpnet-base-v2 make it the only model used
@@ -10,6 +10,8 @@ class SmoochBotNotInstalledError < ::ArgumentError
     Bot::Alegre::PARAPHRASE_MULTILINGUAL_MODEL => 0.6
   }
 
+  NLU_GLOBAL_COUNTER_KEY = 'nlu_global_counter'
+
   include SmoochNluMenus
 
   def initialize(team_slug)
@@ -59,13 +61,23 @@ def self.disambiguation_threshold
     CheckConfig.get('nlu_disambiguation_threshold', 0.11, :float).to_f
   end
 
-  def self.alegre_matches_from_message(message, language, context, alegre_result_key)
+  def self.alegre_matches_from_message(message, language, context, alegre_result_key, uid)
     # FIXME: Raise exception if not in a tipline context (so, if Bot::Smooch.config is nil)
     matches = []
-    team_slug = Team.find(Bot::Smooch.config['team_id']).slug
-    params = nil
-    response = nil
-    if Bot::Smooch.config.to_h['nlu_enabled']
+    unless Bot::Smooch.config.to_h['nlu_enabled']
+      return []
+    end
+    if self.nlu_global_rate_limit_reached?
+      CheckSentry.notify(SmoochNluError.new('NLU global rate limit reached.'))
+      return []
+    end
+    if self.nlu_user_rate_limit_reached?(uid)
+      CheckSentry.notify(SmoochNluError.new('NLU user rate limit reached.'), user_id: uid)
+      return []
+    end
+    begin
+      self.increment_global_counter
+      team_slug = Team.find(Bot::Smooch.config['team_id']).slug
       # FIXME: In the future we could consider matches across all languages when options is nil
       # FIXME: No need to call Alegre if it's an exact match to one of the keywords
       # FIXME: No need to call Alegre if message has no word characters
@@ -94,22 +106,45 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k
       ranked_options = sorted_options.map{ |o| { 'key' => o.dig('_source', 'context', alegre_result_key), 'score' => o['_score'] } }
       matches = ranked_options
 
-      # FIXME: Deal with ties (i.e., where two options have an equal _score or count)
+      # In all cases log for analysis
+      log = {
+        version: '0.1', # Update if schema changes
+        datetime: DateTime.current,
+        team_slug: team_slug,
+        user_query: message,
+        alegre_query: params,
+        alegre_response: response,
+        matches: matches
+      }
+      Rails.logger.info("[Smooch NLU] [Matches From Message] #{log.to_json}")
+    rescue StandardError => e
+      CheckSentry.notify(SmoochNluError.new("NLU exception: #{e.message}"), exception: e)
+      matches = []
+    ensure
+      self.decrement_global_counter
     end
-    # In all cases log for analysis
-    log = {
-      version: "0.1", # Update if schema changes
-      datetime: DateTime.current,
-      team_slug: team_slug,
-      user_query: message,
-      alegre_query: params,
-      alegre_response: response,
-      matches: matches
-    }
-    Rails.logger.info("[Smooch NLU] [Matches From Message] #{log.to_json}")
     matches
   end
 
+  def self.nlu_global_rate_limit_reached?
+    redis = Redis.new(REDIS_CONFIG)
+    redis.get(NLU_GLOBAL_COUNTER_KEY).to_i > CheckConfig.get('nlu_global_rate_limit', 100, :integer)
+  end
+
+  def self.nlu_user_rate_limit_reached?(uid)
+    TiplineMessage.where(uid: uid, created_at: Time.now.ago(1.minute)..Time.now, state: 'received').count > CheckConfig.get('nlu_user_rate_limit', 30, :integer)
+  end
+
+  def self.increment_global_counter
+    redis = Redis.new(REDIS_CONFIG)
+    redis.incr(NLU_GLOBAL_COUNTER_KEY)
+  end
+
+  def self.decrement_global_counter
+    redis = Redis.new(REDIS_CONFIG)
+    redis.decr(NLU_GLOBAL_COUNTER_KEY)
+  end
+
   private
 
   def toggle!(enabled)

app/lib/smooch_nlu_menus.rb

@@ -67,13 +67,13 @@ def update_menu_option_keywords(language, menu, menu_option_index, keyword, oper
   end
 
   module ClassMethods
-    def menu_options_from_message(message, language, options)
+    def menu_options_from_message(message, language, options, uid)
       return [{ 'smooch_menu_option_value' => 'main_state' }] if message == 'cancel_nlu'
       return [] if options.blank?
       context = {
         context: ALEGRE_CONTEXT_KEY_MENU
       }
-      matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id')
+      matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id', uid)
       # Select the top two menu options that exists in `options`
       top_options = []
       matches.each do |r|

app/models/ability.rb

@@ -89,7 +89,7 @@ def editor_perms
     can [:cud], DynamicAnnotation::Field do |obj|
       obj.annotation.team&.id == @context_team.id
     end
-    can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource], :team_id => @context_team.id
+    can [:create, :update, :read, :destroy], [Account, Source, TiplineNewsletter, TiplineResource, TiplineRequest], :team_id => @context_team.id
     can [:cud], AccountSource, source: { team: { team_users: { team_id: @context_team.id }}}
     %w(annotation comment dynamic task tag).each do |annotation_type|
       can [:cud], annotation_type.classify.constantize do |obj|
@@ -135,6 +135,10 @@ def collaborator_perms
         obj.team&.id == @context_team.id && !obj.annotated_is_trashed?
       end
     end
+    can [:cud], TiplineRequest do |obj|
+      is_trashed = obj.associated.respond_to?(:archived) && obj.associated.archived == CheckArchivedFlags::FlagCodes::TRASHED
+      obj.team_id == @context_team.id && !is_trashed
+    end
     can [:create, :destroy], Assignment do |obj|
       type = obj.assigned_type
       obj = obj.assigned

app/models/bot/smooch.rb

@@ -29,7 +29,6 @@ class CapiUnhandledMessageWarning < MessageDeliveryError; end
   include SmoochCapi
   include SmoochStrings
   include SmoochMenus
-  include SmoochFields
   include SmoochLanguage
   include SmoochBlocking
 
@@ -40,19 +39,18 @@ def report_image
       self.get_dynamic_annotation('report_design')&.report_design_image_url
     end
 
-    def get_deduplicated_smooch_annotations
+    def get_deduplicated_tipline_requests
       uids = []
-      annotations = []
+      tipline_requests = []
       ProjectMedia.where(id: self.related_items_ids).each do |pm|
-        pm.get_annotations('smooch').find_each do |annotation|
-          data = JSON.parse(annotation.load.get_field_value('smooch_data'))
-          uid = data['authorId']
+        pm.tipline_requests.find_each do |tr|
+          uid = tr.tipline_user_uid
           next if uids.include?(uid)
           uids << uid
-          annotations << annotation
+          tipline_requests << tr
         end
       end
-      annotations
+      tipline_requests
     end
   end
 
@@ -125,7 +123,9 @@ def self.inherit_status_and_send_report(rid)
                 '_id': Digest::MD5.hexdigest([self.action.to_s, Time.now.to_f.to_s].join(':')),
                 authorId: id,
                 type: 'text',
-                text: message[1]
+                text: message[1],
+                source: { type: "whatsapp" },
+                language: 'en'
               }.with_indifferent_access
               Bot::Smooch.save_message_later(payload, app_id)
             end
@@ -295,14 +295,14 @@ def self.run(body)
         'capi:verification'
       when 'message:appUser'
         json['messages'].each do |message|
-          self.parse_message(message, json['app']['_id'], json)
           SmoochTiplineMessageWorker.perform_async(message, json)
+          self.parse_message(message, json['app']['_id'], json)
         end
         true
       when 'message:delivery:failure'
         self.resend_message(json)
         true
-      when 'conversation:start'
+      when 'conversation:start', 'conversation:referral'
         message = {
           '_id': json['conversation']['_id'],
           authorId: json['appUser']['_id'],
@@ -570,12 +570,12 @@ def self.process_menu_option(message, state, app_id)
     end
     # ...if nothing is matched, try using the NLU feature
     if state != 'query'
-      options = SmoochNlu.menu_options_from_message(typed, language, options)
+      options = SmoochNlu.menu_options_from_message(typed, language, options, uid)
       unless options.blank?
         SmoochNlu.process_menu_options(uid, options, message, language, workflow, app_id)
         return true
       end
-      resource = TiplineResource.resource_from_message(typed, language)
+      resource = TiplineResource.resource_from_message(typed, language, uid)
       unless resource.nil?
         CheckStateMachine.new(uid).reset
         resource = self.send_resource_to_user(uid, workflow, resource.uuid, language)
@@ -602,10 +602,11 @@ def self.user_received_report(message)
       original = begin JSON.parse(original) rescue {} end
       if original['fallback_template'] =~ /report/
         pmids = ProjectMedia.find(original['project_media_id']).related_items_ids
-        DynamicAnnotation::Field.joins(:annotation).where(field_name: 'smooch_data', 'annotations.annotated_type' => 'ProjectMedia', 'annotations.annotated_id' => pmids).where("value_json ->> 'authorId' = ?", message['appUser']['_id']).each do |f|
-          a = f.annotation.load
-          a.set_fields = { smooch_report_received: Time.now.to_i }.to_json
-          a.save!
+        TiplineRequest.where(associated_type: 'ProjectMedia', associated_id: pmids, tipline_user_uid: message['appUser']['_id']).find_each do |tr|
+          field_name = tr.smooch_report_received_at == 0 ? 'smooch_report_received_at' : 'smooch_report_update_received_at'
+          tr.send("#{field_name}=", Time.now.to_i)
+          tr.skip_check_ability = true
+          tr.save!
         end
       end
     end
@@ -934,15 +935,15 @@ def self.send_report_to_users(pm, action)
     report = parent.get_annotations('report_design').last&.load
     return if report.nil?
     last_published_at = report.get_field_value('last_published').to_i
-    parent.get_deduplicated_smooch_annotations.each do |annotation|
-      data = JSON.parse(annotation.load.get_field_value('smooch_data'))
+    parent.get_deduplicated_tipline_requests.each do |tipline_request|
+      data = tipline_request.smooch_data
       self.get_installation(self.installation_setting_id_keys, data['app_id']) if self.config.blank?
-      self.send_correction_to_user(data, parent, annotation, last_published_at, action, report.get_field_value('published_count').to_i) unless self.config['smooch_disabled']
+      self.send_correction_to_user(data, parent, tipline_request, last_published_at, action, report.get_field_value('published_count').to_i) unless self.config['smooch_disabled']
     end
   end
 
-  def self.send_correction_to_user(data, pm, annotation, last_published_at, action, published_count = 0)
-    subscribed_at = annotation.created_at
+  def self.send_correction_to_user(data, pm, tipline_request, last_published_at, action, published_count = 0)
+    subscribed_at = tipline_request.created_at
     self.get_platform_from_message(data)
     uid = data['authorId']
     lang = data['language']
@@ -959,10 +960,9 @@ def self.send_correction_to_user(data, pm, annotation, last_published_at, action
       self.send_report_to_user(uid, data, pm, lang, 'fact_check_report')
     end
     unless field_name.blank?
-      annotation = annotation.load
-      annotation.skip_check_ability = true
-      annotation.set_fields = { "#{field_name}": Time.now.to_i }.to_json
-      annotation.save!
+      tipline_request.skip_check_ability = true
+      tipline_request.send("#{field_name}=", Time.now.to_i)
+      tipline_request.save!
     end
   end
 
@@ -1017,11 +1017,11 @@ def self.send_report_from_parent_to_child(parent_id, target_id)
     parent = ProjectMedia.where(id: parent_id).last
     child = ProjectMedia.where(id: target_id).last
     return if parent.nil? || child.nil?
-    child.get_annotations('smooch').find_each do |annotation|
-      data = JSON.parse(annotation.load.get_field_value('smooch_data'))
+    child.tipline_requests.find_each do |tr|
+      data = tr.smooch_data
       self.get_platform_from_message(data)
       self.get_installation(self.installation_setting_id_keys, data['app_id']) if self.config.blank?
-      self.send_report_to_user(data['authorId'], data, parent, data['language'], 'fact_check_report')
+      self.send_report_to_user(tr.tipline_user_uid, data, parent, tr.language, 'fact_check_report')
     end
   end
 

app/models/cluster.rb

@@ -85,9 +85,9 @@ def claim_descriptions
     recalculate: :recalculate_requests_count,
     update_on: [
         {
-          model: Dynamic,
-          if: proc { |d| d.annotation_type == 'smooch' && d.annotated_type == 'ProjectMedia' },
-          affected_ids: proc { |d| ProjectMedia.where(id: d.annotated.related_items_ids).group(:cluster_id).count.keys.reject{ |cid| cid.nil? } },
+          model: TiplineRequest,
+          if: proc { |tr| tr.associated_type == 'ProjectMedia' },
+          affected_ids: proc { |tr| ProjectMedia.where(id: tr.associated.related_items_ids).group(:cluster_id).count.keys.reject{ |cid| cid.nil? } },
           events: {
             create: :cached_field_cluster_requests_count_create,
             destroy: :cached_field_cluster_requests_count_destroy
@@ -154,14 +154,3 @@ def update_elasticsearch
     ElasticSearchWorker.perform_in(1.second, YAML::dump(model), YAML::dump(options), 'update_doc')
   end
 end
-
-
-Dynamic.class_eval do
-  def cached_field_cluster_requests_count_create(target)
-    target.requests_count + 1
-  end
-
-  def cached_field_cluster_requests_count_destroy(target)
-    target.requests_count - 1
-  end
-end

app/models/concerns/project_media_associations.rb

@@ -19,6 +19,7 @@ module ProjectMediaAssociations
     has_one :claim_description, dependent: :destroy
     belongs_to :cluster, counter_cache: :project_medias_count, optional: true
     belongs_to :source, optional: true
+    has_many :tipline_requests, as: :associated
     has_annotations
   end
 end