CV2-4985: Export dashboard data as CSV (#2190)

* CV2-4985: export dashboard as CSV

* CV2-4985: add export to articles dashboard

* CV2-4985: fix export

* CV2-4985: add missing tests

app/models/concerns/team_private.rb

@@ -137,4 +137,71 @@ def empty_data_structure
     data_structure["Org"] = self.name
     [data_structure]
   end
+
+  def get_dashboard_export_headers(ts, dashboard_type)
+    # Get dashboard headers for both types (articles_dashboard & tipline_dashboard) in format { key: value }
+    # key(string): header label
+    # Value(Hash): { method_name: 'callback that should apply to the method output'}
+    # In some cases, the value may be empty ({}) as some methods will populate more than one row.
+
+    # Common header between articles_dashboard and tipline_dashboard
+    header = {
+      'Articles Sent': { number_of_articles_sent: 'to_i' },
+      'Matched Results (Fact-Checks)': { number_of_matched_results_by_article_type: 'values' },
+      'Matched Results (Explainers)': {},
+    }
+    # Hash to include top items as the header label depend on top_items size
+    top_items = {}
+    # tipline_dashboard columns
+    if dashboard_type == 'tipline_dashboard'
+      header.merge!({
+        'Conversations': { number_of_conversations: 'to_i' },
+        'Messages': { number_of_messages: 'to_i' },
+        'Conversations (Positive)': { number_of_search_results_by_feedback_type: 'values' },
+        'Conversations (Negative)': {}, 'Conversations (No Response)': {},
+        'Avg. Response Time': { average_response_time: 'to_i' },
+        'Users (Total)': { number_of_total_users: 'to_i' },
+        'Users (Unique)': { number_of_unique_users: 'to_i' },
+        'Users (Returning)': { number_of_returning_users: 'to_i' },
+        'Subscribers': { number_of_subscribers: 'to_i' },
+        'Subscribers (New)': { number_of_new_subscribers: 'to_i' },
+        'Newsletters (Sent)': { number_of_newsletters_sent: 'to_i' },
+        'Newsletters (Delivered)': { number_of_newsletters_delivered: 'to_i' },
+        'Media Received (Text)': { number_of_media_received_by_media_type: 'values' },
+        'Media Received (Link)': {}, 'Media Received (Audio)': {}, 'Media Received (Image)': {}, 'Media Received (Video)': {},
+      })
+      top_items = { top_media_tags: 'Top tag', top_requested_media_clusters: 'Top Requested' }
+    else
+      # article_dashboard columns
+      header.merge!({
+        'Published Fact-Checks': { number_of_published_fact_checks: 'to_i' },
+        'Explainers Created': { number_of_explainers_created: 'to_i' },
+        'Fact-Checks Created': { number_of_fact_checks_created: 'to_i' },
+      })
+      rates = ts.send('number_of_fact_checks_by_rating').keys
+      unless rates.blank?
+        # Get the first element to fill the label and callback methods as other element will calling with empty callbacks
+        f_rate = rates.delete_at(0)
+        header.merge!({ "Claim & Fact-Checks (#{f_rate})": { number_of_fact_checks_by_rating: 'values' }})
+        rates.each{ |rate| header.merge!({"Claim & Fact-Checks (#{rate})": {}}) }
+      end
+      top_items = { top_articles_tags: 'Top Article Tags', top_articles_sent: 'Top Fact-Checks Sent' }
+    end
+    unless top_items.blank?
+      top_callback = proc { |output| output.collect{|item| "#{item[:label]} (#{item[:value]})"} }
+      # Append Top tags/requested header based on result count
+      top_items.each do |method, prefix|
+        col_numbers = ts.send(method).size
+        if col_numbers > 0
+          # Add a first one with method callback
+          header.merge!({"#{prefix} (1)": { "#{method}": top_callback } })
+          (col_numbers - 1).times do |i|
+            # Append other columns with empty method
+            header.merge!({"#{prefix} (#{i+2})": {} })
+          end
+        end
+      end
+    end
+    header
+  end
 end

app/models/team.rb

@@ -611,6 +611,32 @@ def get_shorten_outgoing_urls
     self.settings.to_h.with_indifferent_access[:shorten_outgoing_urls] || self.tipline_newsletters.where(content_type: 'rss', enabled: true).exists?
   end
 
+  def get_dashboard_exported_data(filters, dashboard_type)
+    filters = filters.with_indifferent_access
+    ts = TeamStatistics.new(self, filters[:period], filters[:language], filters[:platform])
+    headers = get_dashboard_export_headers(ts, dashboard_type)
+    data = []
+    # Add header labels
+    data << headers.keys
+    header_methods = headers.values.delete_if{|v| v.blank?}
+    # Merging multiple hashes as single hash
+    header_methods = Hash[*header_methods.map{|v|v.to_a}.flatten]
+    raw = []
+    header_methods.each do |method, type|
+      unless type.blank?
+        output = ts.send(method) if ts.respond_to?(method)
+        if type.is_a?(Proc)
+          output = type.call(output)
+        else
+          output = output.send(type)
+        end
+        raw << output
+      end
+    end
+    data << raw.flatten
+    data
+  end
+
   # private
   #
   # Please add private methods to app/models/concerns/team_private.rb

lib/list_export.rb

@@ -1,5 +1,5 @@
 class ListExport
-  TYPES = [:media, :feed, :fact_check, :explainer]
+  TYPES = [:media, :feed, :fact_check, :explainer, :articles_dashboard, :tipline_dashboard ]
 
   def initialize(type, query, team_id)
     @type = type
@@ -21,6 +21,8 @@ def number_of_rows
       @team.filtered_fact_checks(@parsed_query).count
     when :explainer
       @team.filtered_explainers(@parsed_query).count
+    when :articles_dashboard, :tipline_dashboard
+      1 # Always maintain one row for dashboard data, but use different columns for export.
     end
   end
 
@@ -62,6 +64,8 @@ def export_data
       FactCheck.get_exported_data(@parsed_query, @team)
     when :explainer
       Explainer.get_exported_data(@parsed_query, @team)
+    when :articles_dashboard, :tipline_dashboard
+      @team.get_dashboard_exported_data(@parsed_query, @type)
     end
   end
 end

test/lib/list_export_test.rb

@@ -118,4 +118,24 @@ def teardown
     assert_equal 2, csv_content.size
     assert_equal 2, export.number_of_rows
   end
+
+  test "should export dashboard CSV" do
+    t = create_team
+    # tipline_dashboard
+    export = ListExport.new(:tipline_dashboard, { period: "past_week", platform: "whatsapp", language: "en" }.to_json, t.id)
+    csv_url = export.generate_csv_and_send_email(create_user)
+    response = Net::HTTP.get_response(URI(csv_url))
+    assert_equal 200, response.code.to_i
+    csv_content = CSV.parse(response.body, headers: true)
+    assert_equal 1, csv_content.size
+    assert_equal 1, export.number_of_rows
+    # articles_dashboard
+    export = ListExport.new(:articles_dashboard, { period: "past_week", platform: "whatsapp", language: "en" }.to_json, t.id)
+    csv_url = export.generate_csv_and_send_email(create_user)
+    response = Net::HTTP.get_response(URI(csv_url))
+    assert_equal 200, response.code.to_i
+    csv_content = CSV.parse(response.body, headers: true)
+    assert_equal 1, csv_content.size
+    assert_equal 1, export.number_of_rows
+  end
 end

test/models/team_test.rb

@@ -1315,4 +1315,24 @@ def setup
     tn.destroy!
     assert !t.get_shorten_outgoing_urls
   end
+
+  test "should get dashboard data" do
+    setup_elasticsearch
+    RequestStore.store[:skip_cached_field_update] = false
+    team = create_team
+    pm = create_project_media team: team, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }, disable_es_callbacks: false
+    pm2 = create_project_media team: team, channel: { main: CheckChannels::ChannelCodes::WHATSAPP }, disable_es_callbacks: false
+    create_tipline_request team: team.id, associated: pm, disable_es_callbacks: false
+    create_tipline_request team: team.id, associated: pm2, disable_es_callbacks: false
+    sleep 1
+    filters = { period: "past_week", platform: "whatsapp", language: "en" }
+    data = team.get_dashboard_exported_data(filters, 'tipline_dashboard')
+    assert_not_nil data
+    assert_equal data[0].length, data[1].length
+    cd = create_claim_description project_media: pm
+    fc = create_fact_check claim_description: cd, rating: 'in_progress'
+    data = team.get_dashboard_exported_data(filters, 'articles_dashboard')
+    assert_not_nil data
+    assert_equal data[0].length, data[1].length
+  end
 end