Merge branch 'develop' into refactor/CV2-2648-refactor-tipline-requests

app/models/bot/alegre.rb

@@ -33,7 +33,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {})
           'UploadedImage' => 'image',
         }[self.media.type].to_s
         threshold = [{value: thresholds.dig(media_type.to_sym, :value)}] || Bot::Alegre.get_threshold_for_query(media_type, self, true)
-        ids_and_scores = Bot::Alegre.get_items_with_similar_media(Bot::Alegre.media_file_url(self), threshold, team_ids, "/#{media_type}/similarity/search/").to_h
+        ids_and_scores = Bot::Alegre.get_items_with_similar_media_v2(Bot::Alegre.media_file_url(self), threshold, team_ids, media_type).to_h
       elsif self.is_text?
         ids_and_scores = {}
         threads = []

app/models/concerns/alegre_similarity.rb

@@ -39,7 +39,7 @@ def get_items_with_similarity(type, pm, threshold)
       if type == 'text'
         self.get_merged_items_with_similar_text(pm, threshold)
       else
-        results = self.get_items_with_similar_media(self.media_file_url(pm), threshold, pm.team_id, "/#{type}/similarity/search/").reject{ |id, _score_with_context| pm.id == id }
+        results = self.get_items_with_similar_media_v2(self.media_file_url(pm), threshold, pm.team_id, type).reject{ |id, _score_with_context| pm.id == id }
         self.merge_response_with_source_and_target_fields(results, type)
       end
     end

app/models/concerns/alegre_v2.rb

@@ -9,7 +9,11 @@ def host
     end
 
     def sync_path(project_media)
-      "/similarity/sync/#{get_type(project_media)}"
+      self.sync_path_for_type(get_type(project_media))
+    end
+
+    def sync_path_for_type(type)
+      "/similarity/sync/#{type}"
     end
 
     def async_path(project_media)
@@ -256,5 +260,12 @@ def get_similar_items_v2(project_media, field)
     def relate_project_media(project_media, field=nil)
       self.add_relationships(project_media, self.get_similar_items_v2(project_media, field)) unless project_media.is_blank?
     end
+
+    def get_items_with_similar_media_v2(media_url, threshold, team_ids, type)
+      alegre_path = ['audio', 'image'].include?(type) ? self.sync_path_for_type(type) : "/#{type}/similarity/search/"
+      # FIXME: Stop using this method from v1 once all media types are supported by v2
+      # FIXME: Alegre crashes if `media_url` was already requested before, this is why I append a hash
+      self.get_items_with_similar_media("#{media_url}?hash=#{SecureRandom.hex}", threshold, team_ids, alegre_path)
+    end
   end
 end

app/models/concerns/smooch_search.rb

@@ -112,7 +112,7 @@ def get_search_results(uid, last_message, team_id, language)
         type = message['type']
         after = self.date_filter(team_id)
         query = message['text']
-        query = message['mediaUrl'] unless type == 'text'
+        query = CheckS3.rewrite_url(message['mediaUrl']) unless type == 'text'
         results = self.search_for_similar_published_fact_checks(type, query, [team_id], after, nil, language).select{ |pm| is_a_valid_search_result(pm) }
       rescue StandardError => e
         self.handle_search_error(uid, e, language)
@@ -161,10 +161,11 @@ def search_for_similar_published_fact_checks_no_cache(type, query, team_ids, aft
         end
       else
         media_url = Twitter::TwitterText::Extractor.extract_urls(query)[0]
+        Rails.logger.info "[Smooch Bot] Got media_url #{media_url} from query #{query}"
         return [] if media_url.blank?
         media_url = self.save_locally_and_return_url(media_url, type, feed_id)
         threshold = Bot::Alegre.get_threshold_for_query(type, pm)[0][:value]
-        alegre_results = Bot::Alegre.get_items_with_similar_media(media_url, [{ value: threshold }], team_ids, "/#{type}/similarity/search/")
+        alegre_results = Bot::Alegre.get_items_with_similar_media_v2(media_url, [{ value: threshold }], team_ids, type)
         results = self.parse_search_results_from_alegre(alegre_results, after, feed_id, team_ids)
         Rails.logger.info "[Smooch Bot] Media similarity search got #{results.count} results while looking for '#{query}' after date #{after.inspect} for teams #{team_ids}"
       end

app/models/monthly_team_statistic.rb

@@ -13,6 +13,8 @@ class MonthlyTeamStatistic < ApplicationRecord
     language: 'Language',
     month: 'Month', # model method
     whatsapp_conversations: 'WhatsApp conversations',
+    whatsapp_conversations_business: 'Business Conversations',
+    whatsapp_conversations_user: 'User Conversations',
     unique_users: 'Unique users',
     returning_users: 'Returning users',
     published_reports: 'Published reports',

app/resources/api/v2/report_resource.rb

@@ -100,7 +100,7 @@ def self.apply_media_similarity_filter(organization_ids, threshold, media_path,
         unless media.blank?
           media[0].rewind
           CheckS3.write(media_path, media[0].content_type.gsub(/^video/, 'application'), media[0].read)
-          ids_and_scores = Bot::Alegre.get_items_with_similar_media(CheckS3.public_url(media_path), [{ value: threshold }], organization_ids, "/#{media_type}/similarity/search/")
+          ids_and_scores = Bot::Alegre.get_items_with_similar_media_v2(CheckS3.public_url(media_path), [{ value: threshold }], organization_ids, media_type)
           RequestStore.store[:scores] = ids_and_scores # Store the scores so we can return them
           ids = ids_and_scores.keys.uniq || [0]
           CheckS3.delete(media_path)

config/initializers/report_designer.rb

@@ -56,7 +56,7 @@
   def report_design_introduction(data, language)
     if self.annotation_type == 'report_design'
       introduction = self.report_design_field_value('introduction').to_s
-      introduction = introduction.gsub('{{status}}', self.report_design_field_value('status_label').to_s)
+      introduction = introduction.gsub('{{status}}', self.annotated&.status_i18n(nil, { locale: language }))
       introduction = introduction.gsub('{{query_date}}', self.report_design_date(Time.at(data['received']).to_date, language)) if data['received']
       introduction
     end

db/migrate/20240114024701_add_whats_app_user_and_business_conversations_to_monthly_team_statistic.rb

@@ -0,0 +1,6 @@
+class AddWhatsAppUserAndBusinessConversationsToMonthlyTeamStatistic < ActiveRecord::Migration[6.1]
+  def change
+    add_column :monthly_team_statistics, :whatsapp_conversations_user, :integer
+    add_column :monthly_team_statistics, :whatsapp_conversations_business, :integer
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_01_07_223820) do
+ActiveRecord::Schema.define(version: 2024_01_14_024701) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -402,6 +402,8 @@
     t.integer "positive_searches"
     t.integer "negative_searches"
     t.integer "newsletters_sent"
+    t.integer "whatsapp_conversations_user"
+    t.integer "whatsapp_conversations_business"
     t.index ["team_id", "platform", "language", "start_date"], name: "index_monthly_stats_team_platform_language_start", unique: true
     t.index ["team_id"], name: "index_monthly_team_statistics_on_team_id"
   end

lib/check_search.rb

@@ -262,7 +262,7 @@ def alegre_file_similar_items
       file_path = "check_search/#{hash}"
     end
     threshold = Bot::Alegre.get_threshold_for_query(@options['file_type'], ProjectMedia.new(team_id: Team.current.id))[0][:value]
-    results = Bot::Alegre.get_items_with_similar_media(CheckS3.public_url(file_path), [{ value: threshold }], @options['team_id'].first, "/#{@options['file_type']}/similarity/search/")
+    results = Bot::Alegre.get_items_with_similar_media_v2(CheckS3.public_url(file_path), [{ value: threshold }], @options['team_id'].first, @options['file_type'])
     results.blank? ? [0] : results.keys
   end
 

lib/check_statistics.rb

@@ -56,21 +56,35 @@ def number_of_newsletters_sent(team_id, start_date, end_date, language)
       old_count + new_count
     end
 
-    def number_of_whatsapp_conversations(team_id, start_date, end_date)
+    def number_of_whatsapp_conversations(team_id, start_date, end_date, type = 'all') # "type" is "all", "user" or "business"
       from = start_date.to_datetime.to_i
       to = end_date.to_datetime.to_i
 
       # Cache it so we don't recalculate when grabbing the statistics for different languages
-      Rails.cache.fetch("check_statistics:whatsapp_conversations:#{team_id}:#{from}:#{to}", expires_in: 12.hours, skip_nil: true) do
+      Rails.cache.fetch("check_statistics:whatsapp_conversations:#{team_id}:#{from}:#{to}:#{type}", expires_in: 12.hours, skip_nil: true) do
         response = OpenStruct.new({ body: nil, code: 0 })
         begin
           tbi = TeamBotInstallation.where(team_id: team_id, user: BotUser.smooch_user).last
 
           # Only available for tiplines using WhatsApp Cloud API
           unless tbi&.get_capi_whatsapp_business_account_id.blank?
             uri = URI(URI.join('https://graph.facebook.com/v17.0/', tbi.get_capi_whatsapp_business_account_id.to_s))
+            # Account for changes in WhatsApp pricing model
+            # Until May 2023: User-initiated conversations and business-initiated conversations are defined by the dimension CONVERSATION_DIRECTION, values BUSINESS_INITIATED or USER_INITIATED
+            # Starting June 2023: The dimension is CONVERSATION_CATEGORY, where SERVICE is user-initiated and business-initiated is defined by UTILITY, MARKETING or AUTHENTICATION
+            # https://developers.facebook.com/docs/whatsapp/business-management-api/analytics/#conversation-analytics-parameters
+            dimension_field = ''
+            unless type == 'all'
+              dimension = ''
+              if to < Time.parse('2023-06-01').beginning_of_day.to_i
+                dimension = 'CONVERSATION_DIRECTION'
+              else
+                dimension = 'CONVERSATION_CATEGORY'
+              end
+              dimension_field = ".dimensions(#{dimension})"
+            end
             params = {
-              fields: "conversation_analytics.start(#{from}).end(#{to}).granularity(DAILY).phone_numbers(#{tbi.get_capi_phone_number})",
+              fields: "conversation_analytics.start(#{from}).end(#{to}).granularity(DAILY)#{dimension_field}.phone_numbers(#{tbi.get_capi_phone_number})",
               access_token: tbi.get_capi_permanent_token
             }
             uri.query = Rack::Utils.build_query(params)
@@ -80,11 +94,20 @@ def number_of_whatsapp_conversations(team_id, start_date, end_date)
             response = http.request(request)
             raise 'Unexpected response' if response.code.to_i >= 300
             data = JSON.parse(response.body)
-            count = 0
+            all = 0
+            user = 0
+            business = 0
             data['conversation_analytics']['data'][0]['data_points'].each do |data_point|
-              count += data_point['conversation']
+              count = data_point['conversation']
+              all += count
+              user += count if data_point['conversation_direction'] == 'USER_INITIATED' || data_point['conversation_category'] == 'SERVICE'
+              business += count if data_point['conversation_direction'] == 'BUSINESS_INITIATED' || ['UTILITY', 'MARKETING', 'AUTHENTICATION'].include?(data_point['conversation_category'])
             end
-            count
+            {
+              all: all,
+              user: user,
+              business: business
+            }[type.to_sym]
           else
             nil
           end
@@ -189,8 +212,18 @@ def get_statistics(start_date, end_date, team_id, platform, language, tracing_at
           statistics[:newsletters_delivered] = TiplineMessage.where(created_at: start_date..end_date, team_id: team_id, platform: platform_name, language: language, direction: 'outgoing', state: 'delivered', event: 'newsletter').count
         end
 
-        CheckTracer.in_span('CheckStatistics#whatsapp_conversations', attributes: tracing_attributes) do
-          statistics[:whatsapp_conversations] = number_of_whatsapp_conversations(team_id, start_date, end_date) if platform_name == 'WhatsApp'
+        if platform_name == 'WhatsApp'
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'all')
+          end
+
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations_user', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations_user] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'user')
+          end
+
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations_business', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations_business] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'business')
+          end
         end
 
         CheckTracer.in_span('CheckStatistics#published_reports', attributes: tracing_attributes) do

test/lib/check_statistics_test.rb

@@ -20,7 +20,7 @@ def setup
   def teardown
   end
 
-  test 'should calculate number of WhatsApp conversations' do
+  test 'should calculate number of all WhatsApp conversations' do
     WebMock.stub_request(:get, @url).to_return(status: 200, body: {
       conversation_analytics: {
         data: [
@@ -62,7 +62,7 @@ def teardown
       },
       id: '123456'
     }.to_json)
-    assert_equal 2300, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to)
+    assert_equal 2300, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'all')
   end
 
   test 'should not calculate number of WhatsApp conversations if WhatsApp Insights API returns an error' do
@@ -82,4 +82,35 @@ def teardown
     data = CheckStatistics.get_statistics(Time.now.yesterday, Time.now.tomorrow, @team.id, 'whatsapp', 'en')
     assert_equal 1, data[:newsletters_delivered]
   end
+
+  test 'should calculate number of WhatsApp user-initiated and business-initiated conversations' do
+    url = 'https://graph.facebook.com/v17.0/123456?fields=conversation_analytics.start(1672531200).end(1675123200).granularity(DAILY).dimensions(CONVERSATION_DIRECTION).phone_numbers(12345678)&access_token=654321'
+    WebMock.stub_request(:get, url).to_return(status: 200, body: {
+      conversation_analytics: {
+        data: [
+          {
+            data_points: [
+              {
+                start: 1688454000,
+                end: 1688540400,
+                conversation: 40,
+                conversation_direction: 'USER_INITIATED',
+                cost: 0.8866
+              },
+              {
+                start: 1688281200,
+                end: 1688367600,
+                conversation: 10,
+                conversation_direction: 'BUSINESS_INITIATED',
+                cost: 0
+              }
+            ]
+          }
+        ]
+      },
+      id: '123456'
+    }.to_json)
+    assert_equal 40, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'user')
+    assert_equal 10, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'business')
+  end
 end