Data dashboard GraphQL API

.codeclimate.yml

@@ -2,7 +2,7 @@ version: "2"
 checks:
   argument-count:
     config:
-      threshold: 8
+      threshold: 9
   complex-logic:
     config:
       threshold: 4

.rubocop.yml

@@ -243,10 +243,10 @@ Metrics/ModuleLength:
   Max: 250
 
 Metrics/ParameterLists:
-  Description: 'Avoid parameter lists longer than three or four parameters.'
+  Description: 'Avoid parameter lists longer than 9 parameters.'
   StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#too-many-params'
   Enabled: true
-  Max: 8
+  Max: 9
 
 Metrics/PerceivedComplexity:
   Description: >-

app/graph/types/team_statistics_type.rb

@@ -0,0 +1,40 @@
+class TeamStatisticsType < DefaultObject
+  description 'Workspace statistics.'
+
+  implements GraphQL::Types::Relay::Node
+
+  # For articles
+
+  field :number_of_articles_created_by_date, JsonStringType, null: true
+  field :number_of_articles_updated_by_date, JsonStringType, null: true
+  field :number_of_explainers_created, GraphQL::Types::Int, null: true
+  field :number_of_fact_checks_created, GraphQL::Types::Int, null: true
+  field :number_of_published_fact_checks, GraphQL::Types::Int, null: true
+  field :number_of_fact_checks_by_rating, JsonStringType, null: true
+  field :top_articles_sent, JsonStringType, null: true
+  field :top_articles_tags, JsonStringType, null: true
+
+  # For tiplines
+
+  field :number_of_messages, GraphQL::Types::Int, null: true
+  field :number_of_conversations, GraphQL::Types::Int, null: true
+  field :number_of_messages_by_date, JsonStringType, null: true
+  field :number_of_conversations_by_date, JsonStringType, null: true
+  field :number_of_search_results_by_feedback_type, JsonStringType, null: true
+  field :average_response_time, GraphQL::Types::Int, null: true
+  field :number_of_unique_users, GraphQL::Types::Int, null: true
+  field :number_of_total_users, GraphQL::Types::Int, null: true
+  field :number_of_returning_users, GraphQL::Types::Int, null: true
+  field :number_of_subscribers, GraphQL::Types::Int, null: true
+  field :number_of_new_subscribers, GraphQL::Types::Int, null: true
+  field :number_of_newsletters_sent, GraphQL::Types::Int, null: true
+  field :number_of_newsletters_delivered, GraphQL::Types::Int, null: true
+  field :top_media_tags, JsonStringType, null: true
+  field :top_requested_media_clusters, JsonStringType, null: true
+  field :number_of_media_received_by_media_type, JsonStringType, null: true
+
+  # For both articles and tiplines
+
+  field :number_of_articles_sent, GraphQL::Types::Int, null: true
+  field :number_of_matched_results_by_article_type, JsonStringType, null: true
+end

app/graph/types/team_type.rb

@@ -380,6 +380,7 @@ def api_key(dbid:)
   end
 
   field :api_keys, ApiKeyType.connection_type, null: true
+
   def api_keys
     ability = context[:ability] || Ability.new
     api_keys = object.api_keys.order(created_at: :desc)
@@ -388,4 +389,15 @@ def api_keys
       ability.can?(:read, api_key)
     end
   end
+
+  field :statistics, TeamStatisticsType, null: true do
+    argument :period, GraphQL::Types::String, required: true # FIXME: List/validate possible values
+    argument :language, GraphQL::Types::String, required: false
+    argument :platform, GraphQL::Types::String, required: false # FIXME: List/validate possible values
+  end
+
+  def statistics(period:, language: nil, platform: nil)
+    # FIXME: Check for permissions
+    TeamStatistics.new(object, period, language, platform)
+  end
 end

app/models/tipline_request.rb

@@ -2,6 +2,7 @@ class TiplineRequest < ApplicationRecord
   include CheckElasticSearch
 
   belongs_to :associated, polymorphic: true
+  belongs_to :project_media, -> { where(tipline_requests: { associated_type: 'ProjectMedia' }) }, foreign_key: 'associated_id', optional: true
   belongs_to :user, optional: true
 
   before_validation :set_team_and_user, :set_smooch_data_fields, on: :create

db/migrate/20241015223059_add_index_to_created_at_for_articles.rb

@@ -0,0 +1,8 @@
+class AddIndexToCreatedAtForArticles < ActiveRecord::Migration[6.1]
+  def change
+    execute "CREATE INDEX fact_check_created_at_day ON fact_checks (date_trunc('day', created_at))"
+    add_index :fact_checks, :created_at
+    execute "CREATE INDEX explainer_created_at_day ON explainers (date_trunc('day', created_at))"
+    add_index :explainers, :created_at
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_10_09_192811) do
+ActiveRecord::Schema.define(version: 2024_10_15_223059) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -292,7 +292,7 @@
     t.jsonb "value_json", default: "{}"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))"
+    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))"
     t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name"
     t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id"
     t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type"
@@ -326,6 +326,8 @@
     t.datetime "updated_at", precision: 6, null: false
     t.string "tags", default: [], array: true
     t.boolean "trashed", default: false
+    t.index "date_trunc('day'::text, created_at)", name: "explainer_created_at_day"
+    t.index ["created_at"], name: "index_explainers_on_created_at"
     t.index ["tags"], name: "index_explainers_on_tags", using: :gin
     t.index ["team_id"], name: "index_explainers_on_team_id"
     t.index ["user_id"], name: "index_explainers_on_user_id"
@@ -347,7 +349,9 @@
     t.string "rating"
     t.boolean "imported", default: false
     t.boolean "trashed", default: false
+    t.index "date_trunc('day'::text, created_at)", name: "fact_check_created_at_day"
     t.index ["claim_description_id"], name: "index_fact_checks_on_claim_description_id", unique: true
+    t.index ["created_at"], name: "index_fact_checks_on_created_at"
     t.index ["imported"], name: "index_fact_checks_on_imported"
     t.index ["language"], name: "index_fact_checks_on_language"
     t.index ["publisher_id"], name: "index_fact_checks_on_publisher_id"

lib/check_data_points.rb

@@ -4,44 +4,47 @@ class << self
     GRANULARITY_VALUES = ['year', 'quarter', 'month', 'week', 'day']
 
     # Number of tipline messages
-    def tipline_messages(team_id, start_date, end_date, granularity = nil)
+    def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
       query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date)
-      query_based_on_granularity(query, granularity)
+      query_based_on_granularity(query, platform, language, granularity)
     end
 
     # Number of tipline requests
-    def tipline_requests(team_id, start_date, end_date, granularity = nil)
+    def tipline_requests(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
       query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date)
-      query_based_on_granularity(query, granularity)
+      query_based_on_granularity(query, platform, language, granularity)
     end
 
     # Number of tipline requests grouped by type of search result
-    def tipline_requests_by_search_type(team_id, start_date, end_date)
+    def tipline_requests_by_search_type(team_id, start_date, end_date, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
-      TiplineRequest.where(
+      query = TiplineRequest.where(
         team_id: team_id,
         smooch_request_type: SEARCH_RESULT_TYPES,
         created_at: start_date..end_date,
-      ).group('smooch_request_type').count
+      )
+      query = query.where(platform: platform) unless platform.blank?
+      query = query.where(language: language) unless language.blank?
+      query.group('smooch_request_type').count
     end
 
     # Number of Subscribers
-    def tipline_subscriptions(team_id, start_date, end_date, granularity = nil)
+    def tipline_subscriptions(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
       query = TiplineSubscription.where(team_id: team_id, created_at: start_date..end_date)
-      query_based_on_granularity(query, granularity)
+      query_based_on_granularity(query, platform, language, granularity)
     end
 
     # Number of Newsletters sent
-    def newsletters_sent(team_id, start_date, end_date, granularity = nil)
+    def newsletters_sent(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
       query = TiplineNewsletterDelivery
       .joins(:tipline_newsletter)
       .where('tipline_newsletters.team_id': team_id)
       .where(created_at: start_date..end_date)
-      query_based_on_granularity(query, granularity, 'newsletter')
+      query_based_on_granularity(query, platform, language, granularity, 'newsletter')
     end
 
     # Number of Media received, by type
@@ -53,48 +56,62 @@ def media_received_by_type(team_id, start_date, end_date)
     end
 
     # Top clusters
-    def top_clusters(team_id, start_date, end_date, limit = 5)
-      elastic_search_top_items(team_id, start_date, end_date, limit)
+    def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil)
+      elastic_search_top_items(team_id, start_date, end_date, limit, false, range_field, language, language_field, platform)
     end
 
     # Top media tags
-    def top_media_tags(team_id, start_date, end_date, limit = 5)
-      elastic_search_top_items(team_id, start_date, end_date, limit, true)
+    def top_media_tags(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil)
+      elastic_search_top_items(team_id, start_date, end_date, limit, true, range_field, language, language_field, platform)
     end
 
     # Articles sent
-    def articles_sent(team_id, start_date, end_date)
+    def articles_sent(team_id, start_date, end_date, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
       # Get number of articles sent as search results
-      search_result_c = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date).count
+      search_results_query = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date)
+      search_results_query = search_results_query.where(platform: platform) unless platform.blank?
+      search_results_query = search_results_query.where(language: language) unless language.blank?
+      search_results_count = search_results_query.count
       # Get the number of articles sent as reports
-      reports_c = TiplineRequest
+      reports_query = TiplineRequest
       .where(team_id: team_id, created_at: start_date..end_date)
-      .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0').count
-      search_result_c + reports_c
+      .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0')
+      reports_query = reports_query.where(platform: platform) unless platform.blank?
+      reports_query = reports_query.where(language: language) unless language.blank?
+      reports_count = reports_query.count
+      search_results_count + reports_count
     end
 
     # Average response time
-    def average_response_time(team_id, start_date, end_date)
-      TiplineRequest
-      .where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i)
-      .average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f
+    def average_response_time(team_id, start_date, end_date, platform = nil, language = nil)
+      query = TiplineRequest.where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i)
+      query = query.where(platform: platform) unless platform.blank?
+      query = query.where(language: language) unless language.blank?
+      query.average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f
     end
 
     # All users
-    def all_users(team_id, start_date, end_date)
+    def all_users(team_id, start_date, end_date, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
-      TiplineRequest.where(team_id: team_id, created_at: start_date..end_date)
-      .count('DISTINCT(tipline_user_uid)')
+      query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date)
+      query = query.where(platform: platform) unless platform.blank?
+      query = query.where(language: language) unless language.blank?
+      query.count('DISTINCT(tipline_user_uid)')
     end
 
     # Returning users
-    def returning_users(team_id, start_date, end_date)
+    def returning_users(team_id, start_date, end_date, platform = nil, language = nil)
       # Number of returning users (at least one session in the current month, and at least one session in the last previous 2 months)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
-      uids = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date).map(&:tipline_user_uid).uniq
-      TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date)
-      .count('DISTINCT(tipline_user_uid)')
+      uids_query = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date)
+      uids_query = uids_query.where(platform: platform) unless platform.blank?
+      uids_query = uids_query.where(language: language) unless language.blank?
+      uids = uids_query.select(:tipline_user_uid).map(&:tipline_user_uid).uniq
+      query = TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date)
+      query = query.where(platform: platform) unless platform.blank?
+      query = query.where(language: language) unless language.blank?
+      query.count('DISTINCT(tipline_user_uid)')
     end
 
     # New users
@@ -116,7 +133,9 @@ def parse_start_end_dates(start_date, end_date)
       return start_date, end_date
     end
 
-    def query_based_on_granularity(query, granularity, type = nil)
+    def query_based_on_granularity(query, platform, language, granularity, type = nil)
+      query = query.where(platform: platform) unless platform.blank?
+      query = query.where(language: language) unless language.blank?
       # For PG the allowed values for granularity can be one of the following
       # [millennium, century, decade, year, quarter, month, week, day, hour,
       # minute, second, milliseconds, microseconds]
@@ -132,15 +151,17 @@ def query_based_on_granularity(query, granularity, type = nil)
       end
     end
 
-    def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false)
+    def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false, range_field = 'created_at', language = nil, language_field = 'language', platform = nil)
       data = {}
       query = {
-        range: { 'created_at': { start_time: start_date, end_time: end_date } },
+        range: { range_field => { start_time: start_date, end_time: end_date } },
         demand: { min: 1 },
         sort: 'demand',
         eslimit: limit
       }
       query[:tags_as_sentence] = { min: 1 } if with_tags
+      query[language_field.to_sym] = [language].flatten if language
+      query[:channels] = [CheckChannels::ChannelCodes.all_channels['TIPLINE'][platform.upcase]] if platform
       result = CheckSearch.new(query.to_json, nil, team_id)
       result.medias.each{ |pm| data[pm.id] = pm.demand }
       data

lib/relay.idl

@@ -13372,6 +13372,7 @@ type Team implements Node {
   ): SourceConnection
   sources_count(keyword: String): Int
   spam_count: Int
+  statistics(language: String, period: String!, platform: String): TeamStatistics
   tag_texts(
     """
     Returns the elements in the list that come after the specified cursor.
@@ -13688,6 +13689,42 @@ type TeamEdge {
   node: Team
 }
 
+"""
+Workspace statistics.
+"""
+type TeamStatistics implements Node {
+  average_response_time: Int
+  created_at: String
+  id: ID!
+  number_of_articles_created_by_date: JsonStringType
+  number_of_articles_sent: Int
+  number_of_articles_updated_by_date: JsonStringType
+  number_of_conversations: Int
+  number_of_conversations_by_date: JsonStringType
+  number_of_explainers_created: Int
+  number_of_fact_checks_by_rating: JsonStringType
+  number_of_fact_checks_created: Int
+  number_of_matched_results_by_article_type: JsonStringType
+  number_of_media_received_by_media_type: JsonStringType
+  number_of_messages: Int
+  number_of_messages_by_date: JsonStringType
+  number_of_new_subscribers: Int
+  number_of_newsletters_delivered: Int
+  number_of_newsletters_sent: Int
+  number_of_published_fact_checks: Int
+  number_of_returning_users: Int
+  number_of_search_results_by_feedback_type: JsonStringType
+  number_of_subscribers: Int
+  number_of_total_users: Int
+  number_of_unique_users: Int
+  permissions: String
+  top_articles_sent: JsonStringType
+  top_articles_tags: JsonStringType
+  top_media_tags: JsonStringType
+  top_requested_media_clusters: JsonStringType
+  updated_at: String
+}
+
 """
 Team task type
 """