Collecting two new data points for workspace tipline statistics: "Wha…

…tsApp Service Conversations" and "WhatsApp Business Conversations". (#1779)

Reference: CV2-4121.

app/models/monthly_team_statistic.rb

@@ -13,6 +13,8 @@ class MonthlyTeamStatistic < ApplicationRecord
     language: 'Language',
     month: 'Month', # model method
     whatsapp_conversations: 'WhatsApp conversations',
+    whatsapp_conversations_business: 'WhatsApp marketing conversations (business-initiated)',
+    whatsapp_conversations_user: 'WhatsApp service conversations (user-initiated)',
     unique_users: 'Unique users',
     returning_users: 'Returning users',
     published_reports: 'Published reports',

db/migrate/20240114024701_add_whats_app_user_and_business_conversations_to_monthly_team_statistic.rb

@@ -0,0 +1,6 @@
+class AddWhatsAppUserAndBusinessConversationsToMonthlyTeamStatistic < ActiveRecord::Migration[6.1]
+  def change
+    add_column :monthly_team_statistics, :whatsapp_conversations_user, :integer
+    add_column :monthly_team_statistics, :whatsapp_conversations_business, :integer
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_01_07_223820) do
+ActiveRecord::Schema.define(version: 2024_01_14_024701) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -402,6 +402,8 @@
     t.integer "positive_searches"
     t.integer "negative_searches"
     t.integer "newsletters_sent"
+    t.integer "whatsapp_conversations_user"
+    t.integer "whatsapp_conversations_business"
     t.index ["team_id", "platform", "language", "start_date"], name: "index_monthly_stats_team_platform_language_start", unique: true
     t.index ["team_id"], name: "index_monthly_team_statistics_on_team_id"
   end

lib/check_statistics.rb

@@ -65,21 +65,35 @@ def number_of_newsletters_sent(team_id, start_date, end_date, language)
       old_count + new_count
     end
 
-    def number_of_whatsapp_conversations(team_id, start_date, end_date)
+    def number_of_whatsapp_conversations(team_id, start_date, end_date, type = 'all') # "type" is "all", "user" or "business"
       from = start_date.to_datetime.to_i
       to = end_date.to_datetime.to_i
 
       # Cache it so we don't recalculate when grabbing the statistics for different languages
-      Rails.cache.fetch("check_statistics:whatsapp_conversations:#{team_id}:#{from}:#{to}", expires_in: 12.hours, skip_nil: true) do
+      Rails.cache.fetch("check_statistics:whatsapp_conversations:#{team_id}:#{from}:#{to}:#{type}", expires_in: 12.hours, skip_nil: true) do
         response = OpenStruct.new({ body: nil, code: 0 })
         begin
           tbi = TeamBotInstallation.where(team_id: team_id, user: BotUser.smooch_user).last
 
           # Only available for tiplines using WhatsApp Cloud API
           unless tbi&.get_capi_whatsapp_business_account_id.blank?
             uri = URI(URI.join('https://graph.facebook.com/v17.0/', tbi.get_capi_whatsapp_business_account_id.to_s))
+            # Account for changes in WhatsApp pricing model
+            # Until May 2023: User-initiated conversations and business-initiated conversations are defined by the dimension CONVERSATION_DIRECTION, values BUSINESS_INITIATED or USER_INITIATED
+            # Starting June 2023: The dimension is CONVERSATION_CATEGORY, where SERVICE is user-initiated and business-initiated is defined by UTILITY, MARKETING or AUTHENTICATION
+            # https://developers.facebook.com/docs/whatsapp/business-management-api/analytics/#conversation-analytics-parameters
+            dimension_field = ''
+            unless type == 'all'
+              dimension = ''
+              if to < Time.parse('2023-06-01').beginning_of_day.to_i
+                dimension = 'CONVERSATION_DIRECTION'
+              else
+                dimension = 'CONVERSATION_CATEGORY'
+              end
+              dimension_field = ".dimensions(#{dimension})"
+            end
             params = {
-              fields: "conversation_analytics.start(#{from}).end(#{to}).granularity(DAILY).phone_numbers(#{tbi.get_capi_phone_number})",
+              fields: "conversation_analytics.start(#{from}).end(#{to}).granularity(DAILY)#{dimension_field}.phone_numbers(#{tbi.get_capi_phone_number})",
               access_token: tbi.get_capi_permanent_token
             }
             uri.query = Rack::Utils.build_query(params)
@@ -89,11 +103,20 @@ def number_of_whatsapp_conversations(team_id, start_date, end_date)
             response = http.request(request)
             raise 'Unexpected response' if response.code.to_i >= 300
             data = JSON.parse(response.body)
-            count = 0
+            all = 0
+            user = 0
+            business = 0
             data['conversation_analytics']['data'][0]['data_points'].each do |data_point|
-              count += data_point['conversation']
+              count = data_point['conversation']
+              all += count
+              user += count if data_point['conversation_direction'] == 'USER_INITIATED' || data_point['conversation_category'] == 'SERVICE'
+              business += count if data_point['conversation_direction'] == 'BUSINESS_INITIATED' || ['UTILITY', 'MARKETING', 'AUTHENTICATION'].include?(data_point['conversation_category'])
             end
-            count
+            {
+              all: all,
+              user: user,
+              business: business
+            }[type.to_sym]
           else
             nil
           end
@@ -198,8 +221,18 @@ def get_statistics(start_date, end_date, team_id, platform, language, tracing_at
           statistics[:newsletters_delivered] = TiplineMessage.where(created_at: start_date..end_date, team_id: team_id, platform: platform_name, language: language, direction: 'outgoing', state: 'delivered', event: 'newsletter').count
         end
 
-        CheckTracer.in_span('CheckStatistics#whatsapp_conversations', attributes: tracing_attributes) do
-          statistics[:whatsapp_conversations] = number_of_whatsapp_conversations(team_id, start_date, end_date) if platform_name == 'WhatsApp'
+        if platform_name == 'WhatsApp'
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'all')
+          end
+
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations_user', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations_user] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'user')
+          end
+
+          CheckTracer.in_span('CheckStatistics#whatsapp_conversations_business', attributes: tracing_attributes) do
+            statistics[:whatsapp_conversations_business] = number_of_whatsapp_conversations(team_id, start_date, end_date, 'business')
+          end
         end
 
         CheckTracer.in_span('CheckStatistics#published_reports', attributes: tracing_attributes) do

test/lib/check_statistics_test.rb

@@ -20,7 +20,7 @@ def setup
   def teardown
   end
 
-  test 'should calculate number of WhatsApp conversations' do
+  test 'should calculate number of all WhatsApp conversations' do
     WebMock.stub_request(:get, @url).to_return(status: 200, body: {
       conversation_analytics: {
         data: [
@@ -62,7 +62,7 @@ def teardown
       },
       id: '123456'
     }.to_json)
-    assert_equal 2300, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to)
+    assert_equal 2300, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'all')
   end
 
   test 'should not calculate number of WhatsApp conversations if WhatsApp Insights API returns an error' do
@@ -82,4 +82,35 @@ def teardown
     data = CheckStatistics.get_statistics(Time.now.yesterday, Time.now.tomorrow, @team.id, 'whatsapp', 'en')
     assert_equal 1, data[:newsletters_delivered]
   end
+
+  test 'should calculate number of WhatsApp user-initiated and business-initiated conversations' do
+    url = 'https://graph.facebook.com/v17.0/123456?fields=conversation_analytics.start(1672531200).end(1675123200).granularity(DAILY).dimensions(CONVERSATION_DIRECTION).phone_numbers(12345678)&access_token=654321'
+    WebMock.stub_request(:get, url).to_return(status: 200, body: {
+      conversation_analytics: {
+        data: [
+          {
+            data_points: [
+              {
+                start: 1688454000,
+                end: 1688540400,
+                conversation: 40,
+                conversation_direction: 'USER_INITIATED',
+                cost: 0.8866
+              },
+              {
+                start: 1688281200,
+                end: 1688367600,
+                conversation: 10,
+                conversation_direction: 'BUSINESS_INITIATED',
+                cost: 0
+              }
+            ]
+          }
+        ]
+      },
+      id: '123456'
+    }.to_json)
+    assert_equal 40, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'user')
+    assert_equal 10, CheckStatistics.number_of_whatsapp_conversations(@team.id, @from, @to, 'business')
+  end
 end