From 632f210c141dce9607f4dd5ac7f5adb3edae5804 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 9 Oct 2024 23:27:08 -0300 Subject: [PATCH 01/26] Ticket CV2-5389: First commit, just the structure of the GraphQL API --- app/graph/types/team_statistics_type.rb | 37 ++ app/graph/types/team_type.rb | 12 + lib/relay.idl | 34 ++ lib/team_statistics.rb | 148 ++++++++ public/relay.json | 457 ++++++++++++++++++++++++ 5 files changed, 688 insertions(+) create mode 100644 app/graph/types/team_statistics_type.rb create mode 100644 lib/team_statistics.rb diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb new file mode 100644 index 000000000..5aa164f81 --- /dev/null +++ b/app/graph/types/team_statistics_type.rb @@ -0,0 +1,37 @@ +class TeamStatisticsType < DefaultObject + description 'Workspace statistics.' + + implements GraphQL::Types::Relay::Node + + # For articles + + field :number_of_articles_created, JsonStringType, null: true + field :number_of_articles_updated, JsonStringType, null: true + field :number_of_explainers_created, GraphQL::Types::Int, null: true + field :number_of_fact_checks_created, GraphQL::Types::Int, null: true + field :number_of_published_fact_checks, GraphQL::Types::Int, null: true + field :number_of_fact_checks_by_rating, JsonStringType, null: true + field :top_articles_sent, JsonStringType, null: true + field :top_articles_tags, JsonStringType, null: true + + # For tiplines + + field :number_of_messages, GraphQL::Types::Int, null: true + field :number_of_conversations, GraphQL::Types::Int, null: true + field :number_of_search_results_by_type, JsonStringType, null: true + field :average_response_type, GraphQL::Types::Int, null: true + field :number_of_unique_users, GraphQL::Types::Int, null: true + field :number_of_total_users, GraphQL::Types::Int, null: true + field :number_of_returning_users, GraphQL::Types::Int, null: true + field :number_of_subscribers, GraphQL::Types::Int, null: true + field :number_of_newsletters_sent, GraphQL::Types::Int, null: true + field :number_of_newsletters_delivered, GraphQL::Types::Int, null: true + field :top_media_tags, JsonStringType, null: true + field :top_requested_media_clusters, JsonStringType, null: true + field :number_of_media_received_by_type, JsonStringType, null: true + + # For both articles and tiplines + + field :number_of_articles_sent, GraphQL::Types::Int, null: true + field :number_of_matched_results, GraphQL::Types::Int, null: true +end diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index f8d96424a..f04fcc30c 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -363,6 +363,7 @@ def api_key(dbid:) end field :api_keys, ApiKeyType.connection_type, null: true + def api_keys ability = context[:ability] || Ability.new api_keys = object.api_keys.order(created_at: :desc) @@ -371,4 +372,15 @@ def api_keys ability.can?(:read, api_key) end end + + field :statistics, TeamStatisticsType, null: true do + argument :period, GraphQL::Types::String, required: true # FIXME: List/validate possible values + argument :language, GraphQL::Types::String, required: false + argument :platform, GraphQL::Types::String, required: false # FIXME: List/validate possible values + end + + def statistics(period:, language: nil, platform: nil) + # FIXME: Check for permissions + TeamStatistics.new(object, period, language, platform) + end end diff --git a/lib/relay.idl b/lib/relay.idl index f327c115e..a323842fe 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13372,6 +13372,7 @@ type Team implements Node { ): SourceConnection sources_count(keyword: String): Int spam_count: Int + statistics(language: String, period: String!, platform: String): TeamStatistics tag_texts( """ Returns the elements in the list that come after the specified cursor. @@ -13665,6 +13666,39 @@ type TeamEdge { node: Team } +""" +Workspace statistics. +""" +type TeamStatistics implements Node { + average_response_type: Int + created_at: String + id: ID! + number_of_articles_created: JsonStringType + number_of_articles_sent: Int + number_of_articles_updated: JsonStringType + number_of_conversations: Int + number_of_explainers_created: Int + number_of_fact_checks_by_rating: JsonStringType + number_of_fact_checks_created: Int + number_of_matched_results: Int + number_of_media_received_by_type: JsonStringType + number_of_messages: Int + number_of_newsletters_delivered: Int + number_of_newsletters_sent: Int + number_of_published_fact_checks: Int + number_of_returning_users: Int + number_of_search_results_by_type: JsonStringType + number_of_subscribers: Int + number_of_total_users: Int + number_of_unique_users: Int + permissions: String + top_articles_sent: JsonStringType + top_articles_tags: JsonStringType + top_media_tags: JsonStringType + top_requested_media_clusters: JsonStringType + updated_at: String +} + """ Team task type """ diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb new file mode 100644 index 000000000..81e7e670f --- /dev/null +++ b/lib/team_statistics.rb @@ -0,0 +1,148 @@ +class TeamStatistics + def initialize(team, period, language, platform) + @team = team + @period = period + @language = language + @platform = platform + end + + # For articles + + # TODO + def number_of_articles_created + data = {} + time_range.each do |day| + data[day] = rand(100) + end + data + end + + # TODO + def number_of_articles_updated + data = {} + time_range.each do |day| + data[day] = rand(100) + end + data + end + + # TODO + def number_of_explainers_created + rand(1000) + end + + # TODO + def number_of_fact_checks_created + rand(1000) + end + + # TODO + def number_of_published_fact_checks + rand(1000) + end + + # TODO + def number_of_fact_checks_by_rating + { 'Unstarted' => rand(100), 'In Progress' => rand(100), 'False' => rand(100), 'True' => rand(100) } + end + + # TODO + def top_articles_sent + { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } + end + + # TODO + def top_articles_tags + { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + end + + # For tiplines + + # TODO + def number_of_messages + rand(1000) + end + + # TODO + def number_of_conversations + rand(1000) + end + + # TODO + def number_of_search_results_by_type + { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } + end + + # TODO + def average_response_type + 24.hours + end + + # TODO + def number_of_unique_users + rand(1000) + end + + # TODO + def number_of_total_users + rand(1000) + end + + # TODO + def number_of_returning_users + rand(1000) + end + + # TODO + def number_of_subscribers + rand(1000) + end + + # TODO + def number_of_newsletters_sent + rand(1000) + end + + # TODO + def number_of_newsletters_delivered + rand(1000) + end + + # TODO + def top_media_tags + { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + end + + # TODO + def top_requested_media_clusters + { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } + end + + # TODO + def number_of_media_received_by_type + { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } + end + + # For both articles and tiplines + + # TODO + def number_of_articles_sent + rand(1000) + end + + # TODO + def number_of_matched_results + rand(1000) + end + + private + + def time_range + ago = { + last_week: 1.week, + last_month: 1.month, + last_year: 1.year + }[@period.to_sym] + (Time.now.ago(ago).to_datetime..Time.now.to_datetime).to_a + end +end diff --git a/public/relay.json b/public/relay.json index 89102460d..ea2777039 100644 --- a/public/relay.json +++ b/public/relay.json @@ -56461,6 +56461,11 @@ "name": "TeamBotInstallation", "ofType": null }, + { + "kind": "OBJECT", + "name": "TeamStatistics", + "ofType": null + }, { "kind": "OBJECT", "name": "TeamTask", @@ -70508,6 +70513,59 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "statistics", + "description": null, + "args": [ + { + "name": "period", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "language", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "platform", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TeamStatistics", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "tag_texts", "description": null, @@ -71784,6 +71842,405 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "TeamStatistics", + "description": "Workspace statistics.", + "fields": [ + { + "name": "average_response_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_created", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_updated", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_conversations", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_explainers_created", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_fact_checks_by_rating", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_fact_checks_created", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_matched_results", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_media_received_by_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_messages", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_newsletters_delivered", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_newsletters_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_published_fact_checks", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_returning_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_search_results_by_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_subscribers", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_total_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_unique_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "permissions", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_articles_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_articles_tags", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_media_tags", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_requested_media_clusters", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "Node", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "TeamTask", From c936d1b0ff078595620226b00395584f5357badf Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:45:54 -0300 Subject: [PATCH 02/26] Ticket CV2-5389: Adding ID field to TeamStatistics --- lib/team_statistics.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 81e7e670f..de88ae72b 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -6,6 +6,11 @@ def initialize(team, period, language, platform) @platform = platform end + # For GraphQL + def id + Base64.encode64("TeamStatistics/#{@team.id}") + end + # For articles # TODO From ee3f35f5294256e9bf106d65995bd93a27ee1e2b Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:44:17 -0300 Subject: [PATCH 03/26] Fixing Sentry error --- app/models/bot/alegre.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index 2001f6e16..9471af181 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -668,6 +668,7 @@ def self.can_create_relationship?(source, target, relationship_type) end def self.send_post_create_message(source, target, relationship) + return if relationship.nil? message_type = relationship.is_confirmed? ? 'related_to_confirmed_similar' : 'related_to_suggested_similar' message_opts = {item_title: target.title, similar_item_title: source.title} CheckNotification::InfoMessages.send( From d0d6281b6c1a8417e82929d334d0aef5162b8034 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Tue, 15 Oct 2024 19:34:56 -0300 Subject: [PATCH 04/26] Ticket CV2-5389: Actual implementation for articles methods --- ...59_add_index_to_created_at_for_articles.rb | 8 +++ db/schema.rb | 8 ++- lib/team_statistics.rb | 54 +++++++++------- test/lib/team_statistics_test.rb | 61 +++++++++++++++++++ 4 files changed, 107 insertions(+), 24 deletions(-) create mode 100644 db/migrate/20241015223059_add_index_to_created_at_for_articles.rb create mode 100644 test/lib/team_statistics_test.rb diff --git a/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb b/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb new file mode 100644 index 000000000..cc06680c6 --- /dev/null +++ b/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb @@ -0,0 +1,8 @@ +class AddIndexToCreatedAtForArticles < ActiveRecord::Migration[6.1] + def change + execute "CREATE INDEX fact_check_created_at_day ON fact_checks (date_trunc('day', created_at))" + add_index :fact_checks, :created_at + execute "CREATE INDEX explainer_created_at_day ON explainers (date_trunc('day', created_at))" + add_index :explainers, :created_at + end +end diff --git a/db/schema.rb b/db/schema.rb index bccdebba3..dfb106d58 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_10_09_192811) do +ActiveRecord::Schema.define(version: 2024_10_15_223059) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -292,7 +292,7 @@ t.jsonb "value_json", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))" + t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))" t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name" t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id" t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type" @@ -326,6 +326,8 @@ t.datetime "updated_at", precision: 6, null: false t.string "tags", default: [], array: true t.boolean "trashed", default: false + t.index "date_trunc('day'::text, created_at)", name: "explainer_created_at_day" + t.index ["created_at"], name: "index_explainers_on_created_at" t.index ["tags"], name: "index_explainers_on_tags", using: :gin t.index ["team_id"], name: "index_explainers_on_team_id" t.index ["user_id"], name: "index_explainers_on_user_id" @@ -347,7 +349,9 @@ t.string "rating" t.boolean "imported", default: false t.boolean "trashed", default: false + t.index "date_trunc('day'::text, created_at)", name: "fact_check_created_at_day" t.index ["claim_description_id"], name: "index_fact_checks_on_claim_description_id", unique: true + t.index ["created_at"], name: "index_fact_checks_on_created_at" t.index ["imported"], name: "index_fact_checks_on_imported" t.index ["language"], name: "index_fact_checks_on_language" t.index ["publisher_id"], name: "index_fact_checks_on_publisher_id" diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index de88ae72b..f7803f48b 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -1,7 +1,11 @@ class TeamStatistics - def initialize(team, period, language, platform) + PERIODS = ['last_day', 'last_week', 'last_month', 'last_year'] + + def initialize(team, period, language, platform = nil) @team = team + raise ArgumentError.new('Invalid workspace provided') unless @team.is_a?(Team) @period = period + raise ArgumentError.new("Invalid period provided. Allowed values: #{PERIODS.join(', ')}") unless PERIODS.include?(@period) @language = language @platform = platform end @@ -13,42 +17,28 @@ def id # For articles - # TODO def number_of_articles_created - data = {} - time_range.each do |day| - data[day] = rand(100) - end - data + number_of_articles_saved(:created_at) end - # TODO def number_of_articles_updated - data = {} - time_range.each do |day| - data[day] = rand(100) - end - data + number_of_articles_saved(:updated_at) end - # TODO def number_of_explainers_created - rand(1000) + explainers_base_query.count end - # TODO def number_of_fact_checks_created - rand(1000) + fact_checks_base_query.count end - # TODO def number_of_published_fact_checks - rand(1000) + fact_checks_base_query.where(report_status: 'published').count end - # TODO def number_of_fact_checks_by_rating - { 'Unstarted' => rand(100), 'In Progress' => rand(100), 'False' => rand(100), 'True' => rand(100) } + fact_checks_base_query.group(:rating).count.sort.to_h end # TODO @@ -144,10 +134,30 @@ def number_of_matched_results def time_range ago = { + last_day: 1.day, last_week: 1.week, last_month: 1.month, last_year: 1.year }[@period.to_sym] - (Time.now.ago(ago).to_datetime..Time.now.to_datetime).to_a + Time.now.ago(ago).to_datetime..Time.now.to_datetime + end + + def fact_checks_base_query(timestamp_field = :created_at) + FactCheck.joins(:claim_description).where('language' => @language, timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) + end + + def explainers_base_query(timestamp_field = :created_at) + Explainer.where('language' => @language, timestamp_field => time_range, 'team_id' => @team.id) + end + + def number_of_articles_saved(timestamp_field) # timestamp_field = :created_at or :updated_at + raise ArgumentError if timestamp_field != :created_at && timestamp_field != :updated_at + number_of_fact_checks = fact_checks_base_query(timestamp_field).group("date_trunc('day', fact_checks.#{timestamp_field})").count + number_of_explainers = explainers_base_query(timestamp_field).group("date_trunc('day', explainers.#{timestamp_field})").count + number_of_articles = {} + (number_of_fact_checks.keys + number_of_explainers.keys).uniq.sort.each do |day| + number_of_articles[day.to_s] = number_of_fact_checks[day].to_i + number_of_explainers[day].to_i + end + number_of_articles end end diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb new file mode 100644 index 000000000..ec28bf0a2 --- /dev/null +++ b/test/lib/team_statistics_test.rb @@ -0,0 +1,61 @@ +require_relative '../test_helper' + +class TeamStatisticsTest < ActiveSupport::TestCase + def setup + @team = create_team + @team.set_languages = ['en', 'pt'] + @team.save! + end + + def teardown + end + + test "should provide a valid period" do + assert_raises ArgumentError do + TeamStatistics.new(@team, 'last_century', 'en', 'whatsapp') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + end + end + + test "should provide a valid workspace" do + assert_raises ArgumentError do + TeamStatistics.new(Class.new, 'last_month', 'en', 'whatsapp') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + end + end + + test "should return articles statistics" do + team = create_team + + travel_to Time.parse('2024-01-01') do + create_fact_check(language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_explainer team: @team, language: 'en' + create_explainer team: @team + create_explainer language: 'en', team: team + end + + travel_to Time.parse('2024-01-02') do + create_fact_check(report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_explainer team: @team, language: 'en' + create_explainer team: @team + create_explainer language: 'en', team: team + end + + travel_to Time.parse('2024-01-03') do + expected = { '2024-01-01 00:00:00 UTC' => 2, '2024-01-02 00:00:00 UTC' => 2 } + object = TeamStatistics.new(@team, 'last_month', 'en') + assert_equal expected, object.number_of_articles_created + assert_equal expected, object.number_of_articles_updated + assert_equal 2, object.number_of_explainers_created + assert_equal 2, object.number_of_fact_checks_created + assert_equal 1, object.number_of_published_fact_checks + assert_equal({ 'false' => 1, 'verified' => 1 }, object.number_of_fact_checks_by_rating) + end + end +end From 75cfd76163f6ca0a4c935b6e511af4223bd7c0ff Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:14:52 -0300 Subject: [PATCH 05/26] Ticket CV2-5389: Applying changes as per conversation with Alex --- lib/team_statistics.rb | 22 +++++++++++++++++----- test/lib/team_statistics_test.rb | 14 ++++++++------ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index f7803f48b..32437c424 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -143,21 +143,33 @@ def time_range end def fact_checks_base_query(timestamp_field = :created_at) - FactCheck.joins(:claim_description).where('language' => @language, timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) + query = FactCheck.joins(:claim_description).where('language' => @language, timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) + query = query.where('fact_checks.created_at != fact_checks.updated_at') if timestamp_field.to_sym == :updated_at + query end def explainers_base_query(timestamp_field = :created_at) - Explainer.where('language' => @language, timestamp_field => time_range, 'team_id' => @team.id) + query = Explainer.where('language' => @language, timestamp_field => time_range, 'team_id' => @team.id) + query = query.where('explainers.created_at != explainers.updated_at') if timestamp_field.to_sym == :updated_at + query end def number_of_articles_saved(timestamp_field) # timestamp_field = :created_at or :updated_at raise ArgumentError if timestamp_field != :created_at && timestamp_field != :updated_at - number_of_fact_checks = fact_checks_base_query(timestamp_field).group("date_trunc('day', fact_checks.#{timestamp_field})").count - number_of_explainers = explainers_base_query(timestamp_field).group("date_trunc('day', explainers.#{timestamp_field})").count + number_of_fact_checks = fact_checks_base_query(timestamp_field).group("date_trunc('day', #{Arel.sql("fact_checks.#{timestamp_field}")})").count + number_of_explainers = explainers_base_query(timestamp_field).group("date_trunc('day', #{Arel.sql("explainers.#{timestamp_field}")})").count number_of_articles = {} + + # Pre-fill with zeros + time_range.to_a.each do |day| + number_of_articles[day.strftime("%Y-%m-%d")] = 0 + end + + # Replace zeros by the days we have data for (number_of_fact_checks.keys + number_of_explainers.keys).uniq.sort.each do |day| - number_of_articles[day.to_s] = number_of_fact_checks[day].to_i + number_of_explainers[day].to_i + number_of_articles[day.strftime("%Y-%m-%d")] = number_of_fact_checks[day].to_i + number_of_explainers[day].to_i end + number_of_articles end end diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index ec28bf0a2..11b0f5fb2 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -32,10 +32,11 @@ def teardown test "should return articles statistics" do team = create_team + exp = nil travel_to Time.parse('2024-01-01') do create_fact_check(language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) - create_explainer team: @team, language: 'en' + exp = create_explainer team: @team, language: 'en' create_explainer team: @team create_explainer language: 'en', team: team end @@ -45,13 +46,14 @@ def teardown create_explainer team: @team, language: 'en' create_explainer team: @team create_explainer language: 'en', team: team + exp.updated_at = Time.now + exp.save! end - travel_to Time.parse('2024-01-03') do - expected = { '2024-01-01 00:00:00 UTC' => 2, '2024-01-02 00:00:00 UTC' => 2 } - object = TeamStatistics.new(@team, 'last_month', 'en') - assert_equal expected, object.number_of_articles_created - assert_equal expected, object.number_of_articles_updated + travel_to Time.parse('2024-01-08') do + object = TeamStatistics.new(@team, 'last_week', 'en') + assert_equal({ '2024-01-01' => 2, '2024-01-02' => 2, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_articles_created) + assert_equal({ '2024-01-01' => 0, '2024-01-02' => 1, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_articles_updated) assert_equal 2, object.number_of_explainers_created assert_equal 2, object.number_of_fact_checks_created assert_equal 1, object.number_of_published_fact_checks From cb033752fe1bb4ea0220fed3fe42273288647871 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:17:19 -0300 Subject: [PATCH 06/26] Ticket CV2-5389: Applying changes as per conversation with Alex --- app/graph/types/team_statistics_type.rb | 4 +-- lib/relay.idl | 4 +-- lib/team_statistics.rb | 34 ++++++++++++++++++------- public/relay.json | 4 +-- test/lib/team_statistics_test.rb | 6 +++-- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb index 5aa164f81..a1d2087a7 100644 --- a/app/graph/types/team_statistics_type.rb +++ b/app/graph/types/team_statistics_type.rb @@ -5,8 +5,8 @@ class TeamStatisticsType < DefaultObject # For articles - field :number_of_articles_created, JsonStringType, null: true - field :number_of_articles_updated, JsonStringType, null: true + field :number_of_articles_created_by_date, JsonStringType, null: true + field :number_of_articles_updated_by_date, JsonStringType, null: true field :number_of_explainers_created, GraphQL::Types::Int, null: true field :number_of_fact_checks_created, GraphQL::Types::Int, null: true field :number_of_published_fact_checks, GraphQL::Types::Int, null: true diff --git a/lib/relay.idl b/lib/relay.idl index a323842fe..2b4b1956d 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13673,9 +13673,9 @@ type TeamStatistics implements Node { average_response_type: Int created_at: String id: ID! - number_of_articles_created: JsonStringType + number_of_articles_created_by_date: JsonStringType number_of_articles_sent: Int - number_of_articles_updated: JsonStringType + number_of_articles_updated_by_date: JsonStringType number_of_conversations: Int number_of_explainers_created: Int number_of_fact_checks_by_rating: JsonStringType diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 32437c424..50ed30d21 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -17,12 +17,12 @@ def id # For articles - def number_of_articles_created - number_of_articles_saved(:created_at) + def number_of_articles_created_by_date + number_of_articles_saved_by_date(:created_at) end - def number_of_articles_updated - number_of_articles_saved(:updated_at) + def number_of_articles_updated_by_date + number_of_articles_saved_by_date(:updated_at) end def number_of_explainers_created @@ -142,22 +142,38 @@ def time_range Time.now.ago(ago).to_datetime..Time.now.to_datetime end - def fact_checks_base_query(timestamp_field = :created_at) + def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false) query = FactCheck.joins(:claim_description).where('language' => @language, timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) query = query.where('fact_checks.created_at != fact_checks.updated_at') if timestamp_field.to_sym == :updated_at + if group_by_day + # Avoid SQL injection warning + group = { + created_at: "date_trunc('day', fact_checks.created_at)", + updated_at: "date_trunc('day', fact_checks.updated_at)" + }[timestamp_field.to_sym] + query = query.group(group) + end query end - def explainers_base_query(timestamp_field = :created_at) + def explainers_base_query(timestamp_field = :created_at, group_by_day = false) query = Explainer.where('language' => @language, timestamp_field => time_range, 'team_id' => @team.id) query = query.where('explainers.created_at != explainers.updated_at') if timestamp_field.to_sym == :updated_at + if group_by_day + # Avoid SQL injection warning + group = { + created_at: "date_trunc('day', explainers.created_at)", + updated_at: "date_trunc('day', explainers.updated_at)" + }[timestamp_field.to_sym] + query = query.group(group) + end query end - def number_of_articles_saved(timestamp_field) # timestamp_field = :created_at or :updated_at + def number_of_articles_saved_by_date(timestamp_field) # timestamp_field = :created_at or :updated_at raise ArgumentError if timestamp_field != :created_at && timestamp_field != :updated_at - number_of_fact_checks = fact_checks_base_query(timestamp_field).group("date_trunc('day', #{Arel.sql("fact_checks.#{timestamp_field}")})").count - number_of_explainers = explainers_base_query(timestamp_field).group("date_trunc('day', #{Arel.sql("explainers.#{timestamp_field}")})").count + number_of_fact_checks = fact_checks_base_query(timestamp_field, true).count + number_of_explainers = explainers_base_query(timestamp_field, true).count number_of_articles = {} # Pre-fill with zeros diff --git a/public/relay.json b/public/relay.json index ea2777039..c9940e8a8 100644 --- a/public/relay.json +++ b/public/relay.json @@ -71894,7 +71894,7 @@ "deprecationReason": null }, { - "name": "number_of_articles_created", + "name": "number_of_articles_created_by_date", "description": null, "args": [ @@ -71922,7 +71922,7 @@ "deprecationReason": null }, { - "name": "number_of_articles_updated", + "name": "number_of_articles_updated_by_date", "description": null, "args": [ diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 11b0f5fb2..37b1b2894 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -52,8 +52,10 @@ def teardown travel_to Time.parse('2024-01-08') do object = TeamStatistics.new(@team, 'last_week', 'en') - assert_equal({ '2024-01-01' => 2, '2024-01-02' => 2, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_articles_created) - assert_equal({ '2024-01-01' => 0, '2024-01-02' => 1, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_articles_updated) + assert_equal({ '2024-01-01' => 2, '2024-01-02' => 2, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_articles_created_by_date) + assert_equal({ '2024-01-01' => 0, '2024-01-02' => 1, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_articles_updated_by_date) assert_equal 2, object.number_of_explainers_created assert_equal 2, object.number_of_fact_checks_created assert_equal 1, object.number_of_published_fact_checks From 4e16e430b33dbbbb3336296ecfaaa0a6bce37a4b Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 20 Oct 2024 12:58:21 -0300 Subject: [PATCH 07/26] Ticket CV2-5389: Finished data points for articles --- lib/check_data_points.rb | 9 +++++---- lib/team_statistics.rb | 31 +++++++++++++++++++++++++++---- test/lib/team_statistics_test.rb | 29 +++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 160056f0a..03bb38cb4 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -53,8 +53,8 @@ def media_received_by_type(team_id, start_date, end_date) end # Top clusters - def top_clusters(team_id, start_date, end_date, limit = 5) - elastic_search_top_items(team_id, start_date, end_date, limit) + def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil) + elastic_search_top_items(team_id, start_date, end_date, limit, false, range_field, language) end # Top media tags @@ -132,15 +132,16 @@ def query_based_on_granularity(query, granularity, type = nil) end end - def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false) + def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false, range_field = 'created_at', language = nil) data = {} query = { - range: { 'created_at': { start_time: start_date, end_time: end_date } }, + range: { range_field => { start_time: start_date, end_time: end_date } }, demand: { min: 1 }, sort: 'demand', eslimit: limit } query[:tags_as_sentence] = { min: 1 } if with_tags + query[:fc_language] = [language] if language result = CheckSearch.new(query.to_json, nil, team_id) result.medias.each{ |pm| data[pm.id] = pm.demand } data diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 50ed30d21..bf5127b5a 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -41,14 +41,37 @@ def number_of_fact_checks_by_rating fact_checks_base_query.group(:rating).count.sort.to_h end - # TODO + # FIXME: Only fact-checks for now - add explainers def top_articles_sent - { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } + data = {} + range = time_range.to_a + start_date, end_date = range.first, range.last + clusters = CheckDataPoints.top_clusters(@team.id, start_date, end_date, 5, 'last_seen', @language) + clusters.each do |pm_id, demand| + data[ProjectMedia.find(pm_id).fact_check_title] = demand + end + data end - # TODO def top_articles_tags - { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + sql = <<-SQL + SELECT tag, COUNT(*) as tag_count + FROM ( + SELECT unnest(fact_checks.tags) AS tag FROM fact_checks + UNION ALL + SELECT unnest(explainers.tags) AS tag FROM explainers + ) AS all_tags + GROUP BY tag + ORDER BY tag_count DESC + LIMIT 5 + SQL + + result = ActiveRecord::Base.connection.execute(sql) + data = {} + result.each do |row| + data[row['tag']] = row['tag_count'].to_i + end + data.sort.reverse.to_h end # For tiplines diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 37b1b2894..47efa5538 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -35,15 +35,15 @@ def teardown exp = nil travel_to Time.parse('2024-01-01') do - create_fact_check(language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) - exp = create_explainer team: @team, language: 'en' + create_fact_check(tags: ['foo', 'bar'], language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + exp = create_explainer team: @team, language: 'en', tags: ['foo'] create_explainer team: @team create_explainer language: 'en', team: team end travel_to Time.parse('2024-01-02') do - create_fact_check(report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) - create_explainer team: @team, language: 'en' + create_fact_check(tags: ['bar'], report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_explainer team: @team, language: 'en', tags: ['foo'] create_explainer team: @team create_explainer language: 'en', team: team exp.updated_at = Time.now @@ -60,6 +60,27 @@ def teardown assert_equal 2, object.number_of_fact_checks_created assert_equal 1, object.number_of_published_fact_checks assert_equal({ 'false' => 1, 'verified' => 1 }, object.number_of_fact_checks_by_rating) + assert_equal({ 'foo' => 3, 'bar' => 2 }, object.top_articles_tags) end end + + test "should return number of articles sent" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + + pm1 = create_project_media team: @team, disable_es_callbacks: false + create_fact_check title: 'Bar', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm1), disable_es_callbacks: false + create_tipline_request team: @team.id, associated: pm1 + + pm2 = create_project_media team: @team, disable_es_callbacks: false + create_fact_check title: 'Foo', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm2), disable_es_callbacks: false + create_tipline_request team: @team.id, associated: pm2 + create_tipline_request team: @team.id, associated: pm2 + + sleep 2 + + object = TeamStatistics.new(@team, 'last_week', 'en') + expected = { 'Foo' => 2, 'Bar' => 1 } + assert_equal expected, object.top_articles_sent + end end From 5b5d07a5b533225a5104ca80e59dd8912cf81ede Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 20 Oct 2024 13:40:13 -0300 Subject: [PATCH 08/26] Ticket CV2-5389: Fixing articles top tags data point --- lib/team_statistics.rb | 10 ++++++++-- test/lib/team_statistics_test.rb | 10 ++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index bf5127b5a..1bc80dd19 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -57,16 +57,22 @@ def top_articles_tags sql = <<-SQL SELECT tag, COUNT(*) as tag_count FROM ( - SELECT unnest(fact_checks.tags) AS tag FROM fact_checks + SELECT unnest(fcs.tags) AS tag FROM fact_checks fcs + INNER JOIN claim_descriptions cds ON fcs.claim_description_id = cds.id + WHERE cds.team_id = :team_id AND fcs.created_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language) UNION ALL SELECT unnest(explainers.tags) AS tag FROM explainers + WHERE explainers.team_id = :team_id AND explainers.created_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language) ) AS all_tags GROUP BY tag ORDER BY tag_count DESC LIMIT 5 SQL - result = ActiveRecord::Base.connection.execute(sql) + range = time_range.to_a + start_date, end_date = range.first, range.last + language = @language ? [@language] : @team.get_languages.to_a + result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: start_date, end_date: end_date, language: language])) data = {} result.each do |row| data[row['tag']] = row['tag_count'].to_i diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 47efa5538..252692637 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -36,16 +36,18 @@ def teardown travel_to Time.parse('2024-01-01') do create_fact_check(tags: ['foo', 'bar'], language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) exp = create_explainer team: @team, language: 'en', tags: ['foo'] - create_explainer team: @team - create_explainer language: 'en', team: team + create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer language: 'en', team: team, tags: ['foo', 'bar'] end travel_to Time.parse('2024-01-02') do create_fact_check(tags: ['bar'], report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) create_explainer team: @team, language: 'en', tags: ['foo'] - create_explainer team: @team - create_explainer language: 'en', team: team + create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer language: 'en', team: team, tags: ['foo', 'bar'] exp.updated_at = Time.now exp.save! end From 504a6074a0dc1c5e98e31157d6a6b16826f66b52 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 20 Oct 2024 14:09:10 -0300 Subject: [PATCH 09/26] Ticket CV2-5389: Adding two new fields to the schema --- app/graph/types/team_statistics_type.rb | 2 ++ lib/relay.idl | 2 ++ lib/team_statistics.rb | 18 ++++++++++++++++ public/relay.json | 28 +++++++++++++++++++++++++ 4 files changed, 50 insertions(+) diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb index a1d2087a7..aa86a7625 100644 --- a/app/graph/types/team_statistics_type.rb +++ b/app/graph/types/team_statistics_type.rb @@ -18,6 +18,8 @@ class TeamStatisticsType < DefaultObject field :number_of_messages, GraphQL::Types::Int, null: true field :number_of_conversations, GraphQL::Types::Int, null: true + field :number_of_messages_by_date, JsonStringType, null: true + field :number_of_conversations_by_date, JsonStringType, null: true field :number_of_search_results_by_type, JsonStringType, null: true field :average_response_type, GraphQL::Types::Int, null: true field :number_of_unique_users, GraphQL::Types::Int, null: true diff --git a/lib/relay.idl b/lib/relay.idl index 2b4b1956d..2434b9213 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13677,12 +13677,14 @@ type TeamStatistics implements Node { number_of_articles_sent: Int number_of_articles_updated_by_date: JsonStringType number_of_conversations: Int + number_of_conversations_by_date: JsonStringType number_of_explainers_created: Int number_of_fact_checks_by_rating: JsonStringType number_of_fact_checks_created: Int number_of_matched_results: Int number_of_media_received_by_type: JsonStringType number_of_messages: Int + number_of_messages_by_date: JsonStringType number_of_newsletters_delivered: Int number_of_newsletters_sent: Int number_of_published_fact_checks: Int diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 1bc80dd19..6452dde78 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -92,6 +92,24 @@ def number_of_conversations rand(1000) end + # TODO + def number_of_messages_by_date + data = {} + time_range.each do |day| + data[day] = rand(1000) + end + data + end + + # TODO + def number_of_conversations_by_date + data = {} + time_range.each do |day| + data[day] = rand(1000) + end + data + end + # TODO def number_of_search_results_by_type { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } diff --git a/public/relay.json b/public/relay.json index c9940e8a8..7ea77d1ef 100644 --- a/public/relay.json +++ b/public/relay.json @@ -71949,6 +71949,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "number_of_conversations_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "number_of_explainers_created", "description": null, @@ -72033,6 +72047,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "number_of_messages_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "number_of_newsletters_delivered", "description": null, From ecfeef074870a29eb10fa86dc47cb9b34d31806e Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 20 Oct 2024 18:06:41 -0300 Subject: [PATCH 10/26] Ticket CV2-5389: Implementing 4 more data points --- lib/check_data_points.rb | 8 +++- lib/team_statistics.rb | 67 ++++++++++++++++++++------------ test/lib/team_statistics_test.rb | 56 ++++++++++++++++++++++++-- 3 files changed, 101 insertions(+), 30 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 03bb38cb4..ff4dc5204 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -4,16 +4,20 @@ class << self GRANULARITY_VALUES = ['year', 'quarter', 'month', 'week', 'day'] # Number of tipline messages - def tipline_messages(team_id, start_date, end_date, granularity = nil) + def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? query_based_on_granularity(query, granularity) end # Number of tipline requests - def tipline_requests(team_id, start_date, end_date, granularity = nil) + def tipline_requests(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? query_based_on_granularity(query, granularity) end diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 6452dde78..acabae9c4 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -1,13 +1,30 @@ class TeamStatistics PERIODS = ['last_day', 'last_week', 'last_month', 'last_year'] + PLATFORMS = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES def initialize(team, period, language, platform = nil) @team = team - raise ArgumentError.new('Invalid workspace provided') unless @team.is_a?(Team) + unless @team.is_a?(Team) + raise ArgumentError.new('Invalid workspace provided') + end + @period = period - raise ArgumentError.new("Invalid period provided. Allowed values: #{PERIODS.join(', ')}") unless PERIODS.include?(@period) - @language = language + unless PERIODS.include?(@period) + raise ArgumentError.new("Invalid period provided. Allowed values: #{PERIODS.join(', ')}") + end + + range = time_range.to_a + @start_date, @end_date = range.first, range.last + @start_date_str, @end_date_str = @start_date.strftime('%Y-%m-%d'), @end_date.strftime('%Y-%m-%d') + @platform = platform + if !@platform.blank? && !PLATFORMS.keys.include?(@platform) + # For `Bot::Smooch::SUPPORTED_INTEGRATION_NAMES`, the keys (e.g., 'whatsapp') are used by `TiplineRequest`, + # while the values (e.g., 'WhatsApp') are used by `TiplineMessage` + raise ArgumentError.new("Invalid platform provided. Allowed values: #{PLATFORMS.keys.join(', ')}") + end + + @language = language end # For GraphQL @@ -44,9 +61,7 @@ def number_of_fact_checks_by_rating # FIXME: Only fact-checks for now - add explainers def top_articles_sent data = {} - range = time_range.to_a - start_date, end_date = range.first, range.last - clusters = CheckDataPoints.top_clusters(@team.id, start_date, end_date, 5, 'last_seen', @language) + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) clusters.each do |pm_id, demand| data[ProjectMedia.find(pm_id).fact_check_title] = demand end @@ -69,10 +84,8 @@ def top_articles_tags LIMIT 5 SQL - range = time_range.to_a - start_date, end_date = range.first, range.last language = @language ? [@language] : @team.get_languages.to_a - result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: start_date, end_date: end_date, language: language])) + result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: @start_date, end_date: @end_date, language: language])) data = {} result.each do |row| data[row['tag']] = row['tag_count'].to_i @@ -82,32 +95,24 @@ def top_articles_tags # For tiplines - # TODO def number_of_messages - rand(1000) + platform = PLATFORMS[@platform] + CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, nil, platform, @language) end - # TODO def number_of_conversations - rand(1000) + CheckDataPoints.tipline_requests(@team.id, @start_date_str, @end_date_str, nil, @platform, @language) end - # TODO def number_of_messages_by_date - data = {} - time_range.each do |day| - data[day] = rand(1000) - end - data + platform = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES[@platform] + data = CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, 'day', platform, @language) + number_of_tipline_data_points_by_date(data) end - # TODO def number_of_conversations_by_date - data = {} - time_range.each do |day| - data[day] = rand(1000) - end - data + data = CheckDataPoints.tipline_requests(@team.id, @start_date_str, @end_date_str, 'day', @platform, @language) + number_of_tipline_data_points_by_date(data) end # TODO @@ -235,4 +240,16 @@ def number_of_articles_saved_by_date(timestamp_field) # timestamp_field = :creat number_of_articles end + + def number_of_tipline_data_points_by_date(results) + data = {} + # Pre-fill with zeros + time_range.to_a.each do |day| + data[day.strftime("%Y-%m-%d")] = 0 + end + results.each do |day, count| + data[day.strftime("%Y-%m-%d")] = count + end + data + end end diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 252692637..ebd3c6533 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -30,6 +30,16 @@ def teardown end end + test "should provide a valid platform" do + assert_raises ArgumentError do + TeamStatistics.new(Class.new, 'last_month', 'en', 'icq') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + end + end + test "should return articles statistics" do team = create_team exp = nil @@ -72,12 +82,12 @@ def teardown pm1 = create_project_media team: @team, disable_es_callbacks: false create_fact_check title: 'Bar', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm1), disable_es_callbacks: false - create_tipline_request team: @team.id, associated: pm1 + create_tipline_request team_id: @team.id, associated: pm1 pm2 = create_project_media team: @team, disable_es_callbacks: false create_fact_check title: 'Foo', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm2), disable_es_callbacks: false - create_tipline_request team: @team.id, associated: pm2 - create_tipline_request team: @team.id, associated: pm2 + create_tipline_request team_id: @team.id, associated: pm2 + create_tipline_request team_id: @team.id, associated: pm2 sleep 2 @@ -85,4 +95,44 @@ def teardown expected = { 'Foo' => 2, 'Bar' => 1 } assert_equal expected, object.top_articles_sent end + + test "should return tipline statistics" do + team = create_team + pm1 = create_project_media team: @team + pm2 = create_project_media team: team + + travel_to Time.parse('2024-01-01') do + 2.times { create_tipline_message team_id: @team.id, language: 'en', platform: 'WhatsApp' } + create_tipline_message team_id: @team.id, language: 'en', platform: 'Telegram' + create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' + create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' + + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp' + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram' + end + + travel_to Time.parse('2024-01-03') do + 3.times { create_tipline_message team_id: @team.id, language: 'en', platform: 'WhatsApp' } + create_tipline_message team_id: @team.id, language: 'en', platform: 'Telegram' + create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' + create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' + + 2.times { create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp' } + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram' + end + + travel_to Time.parse('2024-01-08') do + object = TeamStatistics.new(@team, 'last_week', 'en', 'whatsapp') + assert_equal 5, object.number_of_messages + assert_equal({ '2024-01-01' => 2, '2024-01-02' => 0, '2024-01-03' => 3, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_messages_by_date) + assert_equal 3, object.number_of_conversations + assert_equal({ '2024-01-01' => 1, '2024-01-02' => 0, '2024-01-03' => 2, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_conversations_by_date) + end + end end From 61d3c7ef187f528a05cf460f8248184c1d7ec20c Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:02:51 -0300 Subject: [PATCH 11/26] Ticket CV2-5389: Fixing CC issues, implementing missing tests and implementing one more data point --- lib/check_data_points.rb | 29 +++++++------ lib/team_statistics.rb | 16 ++++++- .../controllers/graphql_controller_11_test.rb | 43 +++++++++++++++++++ test/lib/team_statistics_test.rb | 23 ++++++---- 4 files changed, 86 insertions(+), 25 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index ff4dc5204..4350181f8 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -7,45 +7,44 @@ class << self def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date) - query = query.where(platform: platform) unless platform.blank? - query = query.where(language: language) unless language.blank? - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of tipline requests def tipline_requests(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) - query = query.where(platform: platform) unless platform.blank? - query = query.where(language: language) unless language.blank? - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of tipline requests grouped by type of search result - def tipline_requests_by_search_type(team_id, start_date, end_date) + def tipline_requests_by_search_type(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) - TiplineRequest.where( + query = TiplineRequest.where( team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date, - ).group('smooch_request_type').count + ) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.group('smooch_request_type').count end # Number of Subscribers - def tipline_subscriptions(team_id, start_date, end_date, granularity = nil) + def tipline_subscriptions(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineSubscription.where(team_id: team_id, created_at: start_date..end_date) - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of Newsletters sent - def newsletters_sent(team_id, start_date, end_date, granularity = nil) + def newsletters_sent(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineNewsletterDelivery .joins(:tipline_newsletter) .where('tipline_newsletters.team_id': team_id) .where(created_at: start_date..end_date) - query_based_on_granularity(query, granularity, 'newsletter') + query_based_on_granularity(query, platform, language, granularity, 'newsletter') end # Number of Media received, by type @@ -120,7 +119,9 @@ def parse_start_end_dates(start_date, end_date) return start_date, end_date end - def query_based_on_granularity(query, granularity, type = nil) + def query_based_on_granularity(query, platform, language, granularity, type = nil) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? # For PG the allowed values for granularity can be one of the following # [millennium, century, decade, year, quarter, month, week, day, hour, # minute, second, milliseconds, microseconds] diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index acabae9c4..42da772be 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -115,9 +115,21 @@ def number_of_conversations_by_date number_of_tipline_data_points_by_date(data) end - # TODO def number_of_search_results_by_type - { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } + mapping = { + relevant_search_result_requests: 'Positive', + irrelevant_search_result_requests: 'Negative', + timeout_search_requests: 'No Response' + } + data = { + 'Positive' => 0, + 'Negative' => 0, + 'No Response' => 0 + } + CheckDataPoints.tipline_requests_by_search_type(@team.id, @start_date_str, @end_date_str, @platform, @language).each do |type, count| + data[mapping[type.to_sym]] = count + end + data end # TODO diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 2cad8e262..280fa460c 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -200,4 +200,47 @@ def teardown assert !JSON.parse(@response.body)['data']['exportList']['success'] end end + + test "should get team statistics" do + user = create_user + team = create_team + create_team_user user: user, team: team, role: 'admin' + + authenticate_with_user(user) + query = <<~GRAPHQL + query { + team(slug: "#{team.slug}") { + statistics(period: "last_week") { + number_of_articles_created_by_date + number_of_articles_updated_by_date + number_of_explainers_created + number_of_fact_checks_created + number_of_published_fact_checks + number_of_fact_checks_by_rating + top_articles_sent + top_articles_tags + number_of_messages + number_of_conversations + number_of_messages_by_date + number_of_conversations_by_date + number_of_search_results_by_type + average_response_type + number_of_unique_users + number_of_total_users + number_of_returning_users + number_of_subscribers + number_of_newsletters_sent + number_of_newsletters_delivered + top_media_tags + top_requested_media_clusters + number_of_media_received_by_type + number_of_articles_sent + number_of_matched_results + } + } + } + GRAPHQL + post :create, params: { query: query } + assert_response :success + end end diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index ebd3c6533..be2a2297f 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -32,7 +32,7 @@ def teardown test "should provide a valid platform" do assert_raises ArgumentError do - TeamStatistics.new(Class.new, 'last_month', 'en', 'icq') + TeamStatistics.new(@team, 'last_month', 'en', 'icq') end assert_nothing_raised do @@ -40,6 +40,10 @@ def teardown end end + test "should have a GraphQL ID" do + assert_kind_of String, TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp').id + end + test "should return articles statistics" do team = create_team exp = nil @@ -107,10 +111,10 @@ def teardown create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' - create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp' - create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp' - create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp' - create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram', smooch_request_type: 'relevant_search_result_requests' end travel_to Time.parse('2024-01-03') do @@ -119,10 +123,10 @@ def teardown create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' - 2.times { create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp' } - create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp' - create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp' - create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram' + 2.times { create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp', smooch_request_type: 'irrelevant_search_result_requests' } + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram', smooch_request_type: 'relevant_search_result_requests' end travel_to Time.parse('2024-01-08') do @@ -133,6 +137,7 @@ def teardown assert_equal 3, object.number_of_conversations assert_equal({ '2024-01-01' => 1, '2024-01-02' => 0, '2024-01-03' => 2, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_conversations_by_date) + assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_type) end end end From 860f8a7a2258f0a7d4adca675da3cf9c87eb4e18 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:51:11 -0300 Subject: [PATCH 12/26] Ticket CV2-5389: Implementing changes to date ranges as requested by Alex --- lib/team_statistics.rb | 16 +++++++++------ .../controllers/graphql_controller_11_test.rb | 2 +- test/lib/team_statistics_test.rb | 20 +++++++++---------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 42da772be..d42605d9c 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -1,5 +1,6 @@ class TeamStatistics - PERIODS = ['last_day', 'last_week', 'last_month', 'last_year'] + PERIODS = ['past_week', 'past_2_weeks', 'past_month', 'past_3_months', 'past_6_months', 'year_to_date'] + PLATFORMS = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES def initialize(team, period, language, platform = nil) @@ -198,12 +199,15 @@ def number_of_matched_results def time_range ago = { - last_day: 1.day, - last_week: 1.week, - last_month: 1.month, - last_year: 1.year + past_week: 1.week, + past_2_weeks: 2.weeks, + past_month: 1.month, + past_3_months: 3.months, + past_6_months: 6.months }[@period.to_sym] - Time.now.ago(ago).to_datetime..Time.now.to_datetime + from = Time.now.ago(ago) unless ago.nil? + from = Time.now.beginning_of_year if @period.to_s == 'year_to_date' + from.to_datetime..Time.now.to_datetime end def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false) diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 280fa460c..7da137925 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -210,7 +210,7 @@ def teardown query = <<~GRAPHQL query { team(slug: "#{team.slug}") { - statistics(period: "last_week") { + statistics(period: "past_week") { number_of_articles_created_by_date number_of_articles_updated_by_date number_of_explainers_created diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index be2a2297f..4dbcd6c3d 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -12,36 +12,36 @@ def teardown test "should provide a valid period" do assert_raises ArgumentError do - TeamStatistics.new(@team, 'last_century', 'en', 'whatsapp') + TeamStatistics.new(@team, 'past_century', 'en', 'whatsapp') end assert_nothing_raised do - TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') end end test "should provide a valid workspace" do assert_raises ArgumentError do - TeamStatistics.new(Class.new, 'last_month', 'en', 'whatsapp') + TeamStatistics.new(Class.new, 'past_month', 'en', 'whatsapp') end assert_nothing_raised do - TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') end end test "should provide a valid platform" do assert_raises ArgumentError do - TeamStatistics.new(@team, 'last_month', 'en', 'icq') + TeamStatistics.new(@team, 'past_month', 'en', 'icq') end assert_nothing_raised do - TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp') + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') end end test "should have a GraphQL ID" do - assert_kind_of String, TeamStatistics.new(@team, 'last_month', 'en', 'whatsapp').id + assert_kind_of String, TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp').id end test "should return articles statistics" do @@ -67,7 +67,7 @@ def teardown end travel_to Time.parse('2024-01-08') do - object = TeamStatistics.new(@team, 'last_week', 'en') + object = TeamStatistics.new(@team, 'past_week', 'en') assert_equal({ '2024-01-01' => 2, '2024-01-02' => 2, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_articles_created_by_date) assert_equal({ '2024-01-01' => 0, '2024-01-02' => 1, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, @@ -95,7 +95,7 @@ def teardown sleep 2 - object = TeamStatistics.new(@team, 'last_week', 'en') + object = TeamStatistics.new(@team, 'past_week', 'en') expected = { 'Foo' => 2, 'Bar' => 1 } assert_equal expected, object.top_articles_sent end @@ -130,7 +130,7 @@ def teardown end travel_to Time.parse('2024-01-08') do - object = TeamStatistics.new(@team, 'last_week', 'en', 'whatsapp') + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') assert_equal 5, object.number_of_messages assert_equal({ '2024-01-01' => 2, '2024-01-02' => 0, '2024-01-03' => 3, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_messages_by_date) From f98d43b018cd716462a11d527e559744dda22473 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:58:29 -0300 Subject: [PATCH 13/26] Ticket CV2-5389: Fixing typo on field name --- app/graph/types/team_statistics_type.rb | 2 +- lib/relay.idl | 2 +- lib/team_statistics.rb | 2 +- public/relay.json | 2 +- test/controllers/graphql_controller_11_test.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb index aa86a7625..035fe08a3 100644 --- a/app/graph/types/team_statistics_type.rb +++ b/app/graph/types/team_statistics_type.rb @@ -21,7 +21,7 @@ class TeamStatisticsType < DefaultObject field :number_of_messages_by_date, JsonStringType, null: true field :number_of_conversations_by_date, JsonStringType, null: true field :number_of_search_results_by_type, JsonStringType, null: true - field :average_response_type, GraphQL::Types::Int, null: true + field :average_response_time, GraphQL::Types::Int, null: true field :number_of_unique_users, GraphQL::Types::Int, null: true field :number_of_total_users, GraphQL::Types::Int, null: true field :number_of_returning_users, GraphQL::Types::Int, null: true diff --git a/lib/relay.idl b/lib/relay.idl index 2434b9213..281b597bd 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13670,7 +13670,7 @@ type TeamEdge { Workspace statistics. """ type TeamStatistics implements Node { - average_response_type: Int + average_response_time: Int created_at: String id: ID! number_of_articles_created_by_date: JsonStringType diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index d42605d9c..175bb29d4 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -134,7 +134,7 @@ def number_of_search_results_by_type end # TODO - def average_response_type + def average_response_time 24.hours end diff --git a/public/relay.json b/public/relay.json index 7ea77d1ef..4951ae067 100644 --- a/public/relay.json +++ b/public/relay.json @@ -71848,7 +71848,7 @@ "description": "Workspace statistics.", "fields": [ { - "name": "average_response_type", + "name": "average_response_time", "description": null, "args": [ diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 7da137925..957e3ddef 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -224,7 +224,7 @@ def teardown number_of_messages_by_date number_of_conversations_by_date number_of_search_results_by_type - average_response_type + average_response_time number_of_unique_users number_of_total_users number_of_returning_users From 7702acb8748357dd26ab50f2328e2f675ccf44de Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 21 Oct 2024 23:23:06 -0300 Subject: [PATCH 14/26] Ticket CV2-5389: Implementing 4 more data points --- lib/check_data_points.rb | 25 +++++++++++-------- lib/team_statistics.rb | 12 +++------ .../controllers/graphql_controller_11_test.rb | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 4350181f8..3498d80cc 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -78,26 +78,31 @@ def articles_sent(team_id, start_date, end_date) end # Average response time - def average_response_time(team_id, start_date, end_date) - TiplineRequest - .where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i) - .average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f + def average_response_time(team_id, start_date, end_date, platform = nil, language = nil) + query = TiplineRequest.where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f end # All users - def all_users(team_id, start_date, end_date) + def all_users(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) - TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) - .count('DISTINCT(tipline_user_uid)') + query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.count('DISTINCT(tipline_user_uid)') end # Returning users - def returning_users(team_id, start_date, end_date) + def returning_users(team_id, start_date, end_date, platform = nil, language = nil) # Number of returning users (at least one session in the current month, and at least one session in the last previous 2 months) start_date, end_date = parse_start_end_dates(start_date, end_date) uids = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date).map(&:tipline_user_uid).uniq - TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date) - .count('DISTINCT(tipline_user_uid)') + query = TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.count('DISTINCT(tipline_user_uid)') end # New users diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 175bb29d4..1bb5d34e8 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -133,24 +133,20 @@ def number_of_search_results_by_type data end - # TODO def average_response_time - 24.hours + CheckDataPoints.average_response_time(@team.id, @start_date, @end_date, @platform, @language) end - # TODO def number_of_unique_users - rand(1000) + number_of_total_users - number_of_returning_users end - # TODO def number_of_total_users - rand(1000) + CheckDataPoints.all_users(@team.id, @start_date_str, @end_date_str, @platform, @language) end - # TODO def number_of_returning_users - rand(1000) + CheckDataPoints.returning_users(@team.id, @start_date_str, @end_date_str, @platform, @language) end # TODO diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 957e3ddef..554a152a1 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -210,7 +210,7 @@ def teardown query = <<~GRAPHQL query { team(slug: "#{team.slug}") { - statistics(period: "past_week") { + statistics(period: "past_week", platform: "whatsapp", language: "en") { number_of_articles_created_by_date number_of_articles_updated_by_date number_of_explainers_created From 702fe4c652df59fe9e2d70e350fd9282e6f2cbbb Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:55:43 -0300 Subject: [PATCH 15/26] Ticket CV2-5389: 3 more fields implemented, 5 missing --- lib/team_statistics.rb | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 1bb5d34e8..b7ee97abd 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -24,6 +24,7 @@ def initialize(team, period, language, platform = nil) # while the values (e.g., 'WhatsApp') are used by `TiplineMessage` raise ArgumentError.new("Invalid platform provided. Allowed values: #{PLATFORMS.keys.join(', ')}") end + @platform_name = PLATFORMS[@platform] @language = language end @@ -97,8 +98,7 @@ def top_articles_tags # For tiplines def number_of_messages - platform = PLATFORMS[@platform] - CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, nil, platform, @language) + CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, nil, @platform_name, @language) end def number_of_conversations @@ -106,8 +106,7 @@ def number_of_conversations end def number_of_messages_by_date - platform = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES[@platform] - data = CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, 'day', platform, @language) + data = CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, 'day', @platform_name, @language) number_of_tipline_data_points_by_date(data) end @@ -149,19 +148,16 @@ def number_of_returning_users CheckDataPoints.returning_users(@team.id, @start_date_str, @end_date_str, @platform, @language) end - # TODO def number_of_subscribers - rand(1000) + CheckDataPoints.tipline_subscriptions(@team.id, @start_date_str, @end_date_str, nil, @platform, @language) end - # TODO def number_of_newsletters_sent - rand(1000) + TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'sent', event: 'newsletter').count end - # TODO def number_of_newsletters_delivered - rand(1000) + TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'delivered', event: 'newsletter').count end # TODO From 0e93e15b605f43142cf4f18cd8e638c54569069c Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:09:58 -0300 Subject: [PATCH 16/26] Ticket CV2-5389: Small fix to query --- lib/team_statistics.rb | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index b7ee97abd..47acf6158 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -60,16 +60,6 @@ def number_of_fact_checks_by_rating fact_checks_base_query.group(:rating).count.sort.to_h end - # FIXME: Only fact-checks for now - add explainers - def top_articles_sent - data = {} - clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) - clusters.each do |pm_id, demand| - data[ProjectMedia.find(pm_id).fact_check_title] = demand - end - data - end - def top_articles_tags sql = <<-SQL SELECT tag, COUNT(*) as tag_count @@ -149,7 +139,7 @@ def number_of_returning_users end def number_of_subscribers - CheckDataPoints.tipline_subscriptions(@team.id, @start_date_str, @end_date_str, nil, @platform, @language) + CheckDataPoints.tipline_subscriptions(@team.id, @start_date_str, @end_date_str, nil, @platform_name, @language) end def number_of_newsletters_sent @@ -160,9 +150,14 @@ def number_of_newsletters_delivered TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'delivered', event: 'newsletter').count end - # TODO - def top_media_tags - { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + # FIXME: Only fact-checks for now - add explainers + def top_articles_sent + data = {} + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) + clusters.each do |pm_id, demand| + data[ProjectMedia.find(pm_id).fact_check_title] = demand + end + data end # TODO @@ -175,6 +170,11 @@ def number_of_media_received_by_type { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } end + # TODO + def top_media_tags + { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + end + # For both articles and tiplines # TODO From 24e09177821ad74fa52a590c9f065da462993b70 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:01:10 -0300 Subject: [PATCH 17/26] Ticket CV2-5389: Adding number_of_new_subscribers --- app/graph/types/team_statistics_type.rb | 1 + lib/relay.idl | 1 + lib/team_statistics.rb | 24 ++++++++++++++---------- public/relay.json | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb index 035fe08a3..f43b16d92 100644 --- a/app/graph/types/team_statistics_type.rb +++ b/app/graph/types/team_statistics_type.rb @@ -26,6 +26,7 @@ class TeamStatisticsType < DefaultObject field :number_of_total_users, GraphQL::Types::Int, null: true field :number_of_returning_users, GraphQL::Types::Int, null: true field :number_of_subscribers, GraphQL::Types::Int, null: true + field :number_of_new_subscribers, GraphQL::Types::Int, null: true field :number_of_newsletters_sent, GraphQL::Types::Int, null: true field :number_of_newsletters_delivered, GraphQL::Types::Int, null: true field :top_media_tags, JsonStringType, null: true diff --git a/lib/relay.idl b/lib/relay.idl index 857986846..c49e79c91 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13708,6 +13708,7 @@ type TeamStatistics implements Node { number_of_media_received_by_type: JsonStringType number_of_messages: Int number_of_messages_by_date: JsonStringType + number_of_new_subscribers: Int number_of_newsletters_delivered: Int number_of_newsletters_sent: Int number_of_published_fact_checks: Int diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 47acf6158..6f9a03853 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -60,6 +60,16 @@ def number_of_fact_checks_by_rating fact_checks_base_query.group(:rating).count.sort.to_h end + # FIXME: Only fact-checks for now - add explainers + def top_articles_sent + data = {} + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) + clusters.each do |pm_id, demand| + data[ProjectMedia.find(pm_id).fact_check_title] = demand + end + data + end + def top_articles_tags sql = <<-SQL SELECT tag, COUNT(*) as tag_count @@ -139,6 +149,10 @@ def number_of_returning_users end def number_of_subscribers + CheckDataPoints.tipline_subscriptions(@team.id, @team.created_at.strftime('%Y-%m-%d'), @end_date_str, nil, @platform_name, @language) + end + + def number_of_new_subscribers CheckDataPoints.tipline_subscriptions(@team.id, @start_date_str, @end_date_str, nil, @platform_name, @language) end @@ -150,16 +164,6 @@ def number_of_newsletters_delivered TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'delivered', event: 'newsletter').count end - # FIXME: Only fact-checks for now - add explainers - def top_articles_sent - data = {} - clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) - clusters.each do |pm_id, demand| - data[ProjectMedia.find(pm_id).fact_check_title] = demand - end - data - end - # TODO def top_requested_media_clusters { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } diff --git a/public/relay.json b/public/relay.json index f6b9e6cf6..dffcc27cc 100644 --- a/public/relay.json +++ b/public/relay.json @@ -72154,6 +72154,20 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "number_of_new_subscribers", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "number_of_newsletters_delivered", "description": null, From ec205eb3c8677b880c76f34d98864193bffaaab9 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:07:37 -0300 Subject: [PATCH 18/26] Ticket CV2-5389: Small fix to query --- lib/team_statistics.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 6f9a03853..61514d2e2 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -86,7 +86,7 @@ def top_articles_tags LIMIT 5 SQL - language = @language ? [@language] : @team.get_languages.to_a + language = @language ? [@language] : [@team.get_languages.to_a, 'und'].flatten result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: @start_date, end_date: @end_date, language: language])) data = {} result.each do |row| From 95b4fc6392d007389107cceda184d76bab8f8afe Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:13:10 -0300 Subject: [PATCH 19/26] Ticket CV2-5389: Small fix to query --- lib/check_data_points.rb | 2 +- lib/team_statistics.rb | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 3498d80cc..e61ddf0fe 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -151,7 +151,7 @@ def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = f eslimit: limit } query[:tags_as_sentence] = { min: 1 } if with_tags - query[:fc_language] = [language] if language + query[:fc_language] = [language].flatten if language result = CheckSearch.new(query.to_json, nil, team_id) result.medias.each{ |pm| data[pm.id] = pm.demand } data diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 61514d2e2..8aab372e0 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -27,6 +27,7 @@ def initialize(team, period, language, platform = nil) @platform_name = PLATFORMS[@platform] @language = language + @all_languages = [@team.get_languages.to_a, 'und'].flatten end # For GraphQL @@ -63,7 +64,7 @@ def number_of_fact_checks_by_rating # FIXME: Only fact-checks for now - add explainers def top_articles_sent data = {} - clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language) + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages) clusters.each do |pm_id, demand| data[ProjectMedia.find(pm_id).fact_check_title] = demand end @@ -86,7 +87,7 @@ def top_articles_tags LIMIT 5 SQL - language = @language ? [@language] : [@team.get_languages.to_a, 'und'].flatten + language = @language ? [@language] : @all_languages result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: @start_date, end_date: @end_date, language: language])) data = {} result.each do |row| From 3c7f97ea42f8a40564ad4376e71dd9a0ad526fb6 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:19:44 -0300 Subject: [PATCH 20/26] Ticket CV2-5389: Adding missing test --- test/controllers/graphql_controller_11_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index ed68b3bcc..2a4daa6cc 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -229,6 +229,7 @@ def teardown number_of_total_users number_of_returning_users number_of_subscribers + number_of_new_subscribers number_of_newsletters_sent number_of_newsletters_delivered top_media_tags From ecd4babbe1f994a5d413f8a07269e5437a0f18b7 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Thu, 24 Oct 2024 23:33:01 -0300 Subject: [PATCH 21/26] Ticket CV2-5389: Adding field number_of_media_received_by_type plus some other small fixes --- lib/team_statistics.rb | 37 +++++++++++++++++++++++--------- test/lib/team_statistics_test.rb | 3 ++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 8aab372e0..35ef27fc5 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -24,7 +24,7 @@ def initialize(team, period, language, platform = nil) # while the values (e.g., 'WhatsApp') are used by `TiplineMessage` raise ArgumentError.new("Invalid platform provided. Allowed values: #{PLATFORMS.keys.join(', ')}") end - @platform_name = PLATFORMS[@platform] + @platform_name = PLATFORMS[@platform] unless @platform.blank? @language = language @all_languages = [@team.get_languages.to_a, 'und'].flatten @@ -158,21 +158,29 @@ def number_of_new_subscribers end def number_of_newsletters_sent - TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'sent', event: 'newsletter').count + number_of_newsletters('sent') end def number_of_newsletters_delivered - TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, platform: @platform_name, language: @language, state: 'delivered', event: 'newsletter').count + number_of_newsletters('delivered') end - # TODO - def top_requested_media_clusters - { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } + def number_of_media_received_by_type + conditions = { team_id: @team.id, created_at: @start_date..@end_date } + conditions[:language] = @language unless @language.blank? + conditions[:platform] = @platform unless @platform.blank? + data = TiplineRequest + .joins("INNER JOIN project_medias pm ON tipline_requests.associated_type = 'ProjectMedia' AND pm.id = tipline_requests.associated_id") + .joins("INNER JOIN medias m ON m.id = pm.media_id") + .where(conditions) + .group('m.type') + .count + { 'Claim' => 0, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }.merge(data).reject{ |k, _v| k == 'Blank' } end # TODO - def number_of_media_received_by_type - { 'Image' => rand(100), 'Text' => rand(100), 'Audio' => rand(100), 'Video' => rand(100), 'Link' => rand(100) } + def top_requested_media_clusters + { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } end # TODO @@ -208,8 +216,9 @@ def time_range end def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false) - query = FactCheck.joins(:claim_description).where('language' => @language, timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) + query = FactCheck.joins(:claim_description).where(timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) query = query.where('fact_checks.created_at != fact_checks.updated_at') if timestamp_field.to_sym == :updated_at + query = query.where(language: @language) unless @language.blank? if group_by_day # Avoid SQL injection warning group = { @@ -222,7 +231,8 @@ def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false) end def explainers_base_query(timestamp_field = :created_at, group_by_day = false) - query = Explainer.where('language' => @language, timestamp_field => time_range, 'team_id' => @team.id) + query = Explainer.where(timestamp_field => time_range, 'team_id' => @team.id) + query = query.where(language: @language) unless @language.blank? query = query.where('explainers.created_at != explainers.updated_at') if timestamp_field.to_sym == :updated_at if group_by_day # Avoid SQL injection warning @@ -265,4 +275,11 @@ def number_of_tipline_data_points_by_date(results) end data end + + def number_of_newsletters(state) + query = TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, state: state, event: 'newsletter') + query = query.where(language: @language) unless @language.blank? + query = query.where(platform: @platform_name) unless @platform.blank? + query.count + end end diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 4dbcd6c3d..d0d650c51 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -102,7 +102,7 @@ def teardown test "should return tipline statistics" do team = create_team - pm1 = create_project_media team: @team + pm1 = create_project_media team: @team, quote: 'Test' pm2 = create_project_media team: team travel_to Time.parse('2024-01-01') do @@ -138,6 +138,7 @@ def teardown assert_equal({ '2024-01-01' => 1, '2024-01-02' => 0, '2024-01-03' => 2, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_conversations_by_date) assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_type) + assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_type) end end end From c9539b3f338bd5337170287cae2899e2c9185bba Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sat, 26 Oct 2024 13:55:18 -0300 Subject: [PATCH 22/26] Ticket CV2-5389: Renaming the *_by_type fields to be more specific --- app/graph/types/team_statistics_type.rb | 6 +++--- lib/check_data_points.rb | 5 ++++- lib/relay.idl | 6 +++--- lib/team_statistics.rb | 11 ++++++----- public/relay.json | 8 ++++---- test/controllers/graphql_controller_11_test.rb | 6 +++--- test/lib/team_statistics_test.rb | 4 ++-- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb index f43b16d92..5c98bfddd 100644 --- a/app/graph/types/team_statistics_type.rb +++ b/app/graph/types/team_statistics_type.rb @@ -20,7 +20,7 @@ class TeamStatisticsType < DefaultObject field :number_of_conversations, GraphQL::Types::Int, null: true field :number_of_messages_by_date, JsonStringType, null: true field :number_of_conversations_by_date, JsonStringType, null: true - field :number_of_search_results_by_type, JsonStringType, null: true + field :number_of_search_results_by_feedback_type, JsonStringType, null: true field :average_response_time, GraphQL::Types::Int, null: true field :number_of_unique_users, GraphQL::Types::Int, null: true field :number_of_total_users, GraphQL::Types::Int, null: true @@ -31,10 +31,10 @@ class TeamStatisticsType < DefaultObject field :number_of_newsletters_delivered, GraphQL::Types::Int, null: true field :top_media_tags, JsonStringType, null: true field :top_requested_media_clusters, JsonStringType, null: true - field :number_of_media_received_by_type, JsonStringType, null: true + field :number_of_media_received_by_media_type, JsonStringType, null: true # For both articles and tiplines field :number_of_articles_sent, GraphQL::Types::Int, null: true - field :number_of_matched_results, GraphQL::Types::Int, null: true + field :number_of_matched_results_by_article_type, JsonStringType, null: true end diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index e61ddf0fe..69534cbf6 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -98,7 +98,10 @@ def all_users(team_id, start_date, end_date, platform = nil, language = nil) def returning_users(team_id, start_date, end_date, platform = nil, language = nil) # Number of returning users (at least one session in the current month, and at least one session in the last previous 2 months) start_date, end_date = parse_start_end_dates(start_date, end_date) - uids = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date).map(&:tipline_user_uid).uniq + uids_query = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date) + uids_query = uids_query.where(platform: platform) unless platform.blank? + uids_query = uids_query.where(language: language) unless language.blank? + uids = uids_query.select(:tipline_user_uid).map(&:tipline_user_uid).uniq query = TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date) query = query.where(platform: platform) unless platform.blank? query = query.where(language: language) unless language.blank? diff --git a/lib/relay.idl b/lib/relay.idl index c49e79c91..b672574b3 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13704,8 +13704,8 @@ type TeamStatistics implements Node { number_of_explainers_created: Int number_of_fact_checks_by_rating: JsonStringType number_of_fact_checks_created: Int - number_of_matched_results: Int - number_of_media_received_by_type: JsonStringType + number_of_matched_results_by_article_type: JsonStringType + number_of_media_received_by_media_type: JsonStringType number_of_messages: Int number_of_messages_by_date: JsonStringType number_of_new_subscribers: Int @@ -13713,7 +13713,7 @@ type TeamStatistics implements Node { number_of_newsletters_sent: Int number_of_published_fact_checks: Int number_of_returning_users: Int - number_of_search_results_by_type: JsonStringType + number_of_search_results_by_feedback_type: JsonStringType number_of_subscribers: Int number_of_total_users: Int number_of_unique_users: Int diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 35ef27fc5..694e529e2 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -66,7 +66,8 @@ def top_articles_sent data = {} clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages) clusters.each do |pm_id, demand| - data[ProjectMedia.find(pm_id).fact_check_title] = demand + item = ProjectMedia.find(pm_id) + data[item.fact_check_title || item.title] = demand end data end @@ -116,7 +117,7 @@ def number_of_conversations_by_date number_of_tipline_data_points_by_date(data) end - def number_of_search_results_by_type + def number_of_search_results_by_feedback_type mapping = { relevant_search_result_requests: 'Positive', irrelevant_search_result_requests: 'Negative', @@ -165,7 +166,7 @@ def number_of_newsletters_delivered number_of_newsletters('delivered') end - def number_of_media_received_by_type + def number_of_media_received_by_media_type conditions = { team_id: @team.id, created_at: @start_date..@end_date } conditions[:language] = @language unless @language.blank? conditions[:platform] = @platform unless @platform.blank? @@ -196,8 +197,8 @@ def number_of_articles_sent end # TODO - def number_of_matched_results - rand(1000) + def number_of_matched_results_by_article_type + { 'FactCheck' => rand(1000), 'Explainer' => rand(1000) } end private diff --git a/public/relay.json b/public/relay.json index dffcc27cc..2fc923085 100644 --- a/public/relay.json +++ b/public/relay.json @@ -72099,21 +72099,21 @@ "deprecationReason": null }, { - "name": "number_of_matched_results", + "name": "number_of_matched_results_by_article_type", "description": null, "args": [ ], "type": { "kind": "SCALAR", - "name": "Int", + "name": "JsonStringType", "ofType": null }, "isDeprecated": false, "deprecationReason": null }, { - "name": "number_of_media_received_by_type", + "name": "number_of_media_received_by_media_type", "description": null, "args": [ @@ -72225,7 +72225,7 @@ "deprecationReason": null }, { - "name": "number_of_search_results_by_type", + "name": "number_of_search_results_by_feedback_type", "description": null, "args": [ diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index 2a4daa6cc..08eb5ded8 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -223,7 +223,7 @@ def teardown number_of_conversations number_of_messages_by_date number_of_conversations_by_date - number_of_search_results_by_type + number_of_search_results_by_feedback_type average_response_time number_of_unique_users number_of_total_users @@ -234,9 +234,9 @@ def teardown number_of_newsletters_delivered top_media_tags top_requested_media_clusters - number_of_media_received_by_type + number_of_media_received_by_media_type number_of_articles_sent - number_of_matched_results + number_of_matched_results_by_article_type } } } diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index d0d650c51..a93b69c59 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -137,8 +137,8 @@ def teardown assert_equal 3, object.number_of_conversations assert_equal({ '2024-01-01' => 1, '2024-01-02' => 0, '2024-01-03' => 2, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, object.number_of_conversations_by_date) - assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_type) - assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_type) + assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_feedback_type) + assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_media_type) end end end From aa98200db863324424c95aa6be12b0a3bcbb564a Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Sun, 27 Oct 2024 21:55:35 -0300 Subject: [PATCH 23/26] Ticket CV2-5389: 2 more fields implemented, 2 missing --- lib/check_data_points.rb | 25 ++++++++++++++++--------- lib/team_statistics.rb | 17 +++++++++++------ test/lib/team_statistics_test.rb | 27 +++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 69534cbf6..4dea6afa5 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -56,8 +56,8 @@ def media_received_by_type(team_id, start_date, end_date) end # Top clusters - def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil) - elastic_search_top_items(team_id, start_date, end_date, limit, false, range_field, language) + def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) + elastic_search_top_items(team_id, start_date, end_date, limit, false, range_field, language, language_field, platform) end # Top media tags @@ -66,15 +66,21 @@ def top_media_tags(team_id, start_date, end_date, limit = 5) end # Articles sent - def articles_sent(team_id, start_date, end_date) + def articles_sent(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) # Get number of articles sent as search results - search_result_c = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date).count + search_results_query = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date) + search_results_query = search_results_query.where(platform: platform) unless platform.blank? + search_results_query = search_results_query.where(language: language) unless language.blank? + search_results_count = search_results_query.count # Get the number of articles sent as reports - reports_c = TiplineRequest + reports_query = TiplineRequest .where(team_id: team_id, created_at: start_date..end_date) - .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0').count - search_result_c + reports_c + .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0') + reports_query = reports_query.where(platform: platform) unless platform.blank? + reports_query = reports_query.where(language: language) unless language.blank? + reports_count = reports_query.count + search_results_count + reports_count end # Average response time @@ -145,7 +151,7 @@ def query_based_on_granularity(query, platform, language, granularity, type = ni end end - def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false, range_field = 'created_at', language = nil) + def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) data = {} query = { range: { range_field => { start_time: start_date, end_time: end_date } }, @@ -154,7 +160,8 @@ def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = f eslimit: limit } query[:tags_as_sentence] = { min: 1 } if with_tags - query[:fc_language] = [language].flatten if language + query[language_field.to_sym] = [language].flatten if language + query[:channels] = [CheckChannels::ChannelCodes.all_channels['TIPLINE'][platform.upcase]] if platform result = CheckSearch.new(query.to_json, nil, team_id) result.medias.each{ |pm| data[pm.id] = pm.demand } data diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 694e529e2..60ef9d51e 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -61,10 +61,10 @@ def number_of_fact_checks_by_rating fact_checks_base_query.group(:rating).count.sort.to_h end - # FIXME: Only fact-checks for now - add explainers + # FIXME: Only fact-checks for now (need to add explainers) and the "demand" is across languages and platforms def top_articles_sent data = {} - clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages) + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'fc_language') clusters.each do |pm_id, demand| item = ProjectMedia.find(pm_id) data[item.fact_check_title || item.title] = demand @@ -179,9 +179,15 @@ def number_of_media_received_by_media_type { 'Claim' => 0, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }.merge(data).reject{ |k, _v| k == 'Blank' } end - # TODO + # FIXME: The "demand" is across languages and platforms def top_requested_media_clusters - { 'The sky is blue' => rand(100), 'Earth is round' => rand(100), 'Soup is dinner' => rand(100) } + data = {} + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'request_language', @platform) + clusters.each do |pm_id, demand| + item = ProjectMedia.find(pm_id) + data[item.title] = demand + end + data end # TODO @@ -191,9 +197,8 @@ def top_media_tags # For both articles and tiplines - # TODO def number_of_articles_sent - rand(1000) + CheckDataPoints.articles_sent(@team.id, @start_date_str, @end_date_str, @platform, @language) end # TODO diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index a93b69c59..9717da657 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -139,6 +139,33 @@ def teardown object.number_of_conversations_by_date) assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_feedback_type) assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_media_type) + assert_equal 3, object.number_of_articles_sent + end + end + + test "should return top requested media clusters" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + channel = CheckChannels::ChannelCodes::WHATSAPP + Sidekiq::Testing.inline! do + pm1 = create_project_media team: @team, quote: 'Bar', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm1, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm2 = create_project_media team: @team, quote: 'Foo', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm3 = create_project_media team: @team, quote: 'Test 1', channel: { main: 0, others: [0] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm3, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm4 = create_project_media team: @team, quote: 'Test 2', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'pt', disable_es_callbacks: false + + sleep 3 + + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') + expected = { 'Foo' => 2, 'Bar' => 1 } + assert_equal expected, object.top_requested_media_clusters end end end From 8d7131fe91a24f05171dd843646fcacb1cf93676 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 28 Oct 2024 00:27:39 -0300 Subject: [PATCH 24/26] Ticket CV2-5389: Implementing the last two fields --- app/models/tipline_request.rb | 1 + lib/check_data_points.rb | 4 +-- lib/team_statistics.rb | 19 ++++++++++++--- test/lib/team_statistics_test.rb | 42 +++++++++++++++++++++++++++++++- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/app/models/tipline_request.rb b/app/models/tipline_request.rb index 62e5bf28d..3b753a78d 100644 --- a/app/models/tipline_request.rb +++ b/app/models/tipline_request.rb @@ -2,6 +2,7 @@ class TiplineRequest < ApplicationRecord include CheckElasticSearch belongs_to :associated, polymorphic: true + belongs_to :project_media, -> { where(tipline_requests: { associated_type: 'ProjectMedia' }) }, foreign_key: 'associated_id', optional: true belongs_to :user, optional: true before_validation :set_team_and_user, :set_smooch_data_fields, on: :create diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 4dea6afa5..744312abc 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -61,8 +61,8 @@ def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'create end # Top media tags - def top_media_tags(team_id, start_date, end_date, limit = 5) - elastic_search_top_items(team_id, start_date, end_date, limit, true) + def top_media_tags(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) + elastic_search_top_items(team_id, start_date, end_date, limit, true, range_field, language, language_field, platform) end # Articles sent diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index 60ef9d51e..e53ef4bb3 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -190,9 +190,18 @@ def top_requested_media_clusters data end - # TODO + # FIXME: The "demand" is across languages and platforms def top_media_tags - { 'tag1' => rand(100), 'tag2' => rand(100), 'tag3' => rand(100), 'tag4' => rand(100), 'tag5' => rand(100) } + data = {} + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'language', @platform) + clusters.each do |pm_id, demand| + item = ProjectMedia.find(pm_id) + item.tags_as_sentence.split(',').map(&:strip).each do |tag| + data[tag] ||= 0 + data[tag] += demand + end + end + data.sort_by{ |key, value| value }.reverse.first(5).to_h end # For both articles and tiplines @@ -201,9 +210,11 @@ def number_of_articles_sent CheckDataPoints.articles_sent(@team.id, @start_date_str, @end_date_str, @platform, @language) end - # TODO def number_of_matched_results_by_article_type - { 'FactCheck' => rand(1000), 'Explainer' => rand(1000) } + query = TiplineRequest.where(team_id: @team.id, smooch_request_type: ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'], created_at: @start_date..@end_date) + query = query.where(platform: @platform) unless @platform.blank? + query = query.where(language: @language) unless @language.blank? + { 'FactCheck' => query.joins(project_media: { claim_description: :fact_check }).count, 'Explainer' => query.joins(project_media: :explainers).count } end private diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb index 9717da657..8e0b51dfe 100644 --- a/test/lib/team_statistics_test.rb +++ b/test/lib/team_statistics_test.rb @@ -101,8 +101,11 @@ def teardown end test "should return tipline statistics" do - team = create_team pm1 = create_project_media team: @team, quote: 'Test' + create_fact_check claim_description: create_claim_description(project_media: pm1) + exp = create_explainer team: @team + pm1.explainers << exp + team = create_team pm2 = create_project_media team: team travel_to Time.parse('2024-01-01') do @@ -140,6 +143,7 @@ def teardown assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_feedback_type) assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_media_type) assert_equal 3, object.number_of_articles_sent + assert_equal({ 'FactCheck' => 3, 'Explainer' => 3 }, object.number_of_matched_results_by_article_type) end end @@ -168,4 +172,40 @@ def teardown assert_equal expected, object.top_requested_media_clusters end end + + test "should return top media tags" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + channel = CheckChannels::ChannelCodes::WHATSAPP + TestDynamicAnnotationTables.load! + create_annotation_type_and_fields('Language', { 'Language' => ['Text', true] }) + Sidekiq::Testing.inline! do + pm1 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['foo', 'bar'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm1, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm1, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm2 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['foo', 'test'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm2, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm3 = create_project_media team: @team, channel: { main: 0, others: [0] }, tags: ['test-1'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm3, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm3, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm4 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['test-2'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm4, set_fields: { language: 'pt' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'pt', disable_es_callbacks: false + + pm5 = create_project_media team: @team, channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm5, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + sleep 3 + + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') + expected = { 'foo' => 3, 'test' => 2, 'bar' => 1 } + assert_equal expected, object.top_media_tags + end + end end From 802cd0f440591cff056a1fc06937fe19f33681d9 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 28 Oct 2024 00:32:06 -0300 Subject: [PATCH 25/26] Ticket CV2-5389: Fixing Code Climate issues --- .codeclimate.yml | 2 +- lib/team_statistics.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index ff19a8b13..3a28cf1b7 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -2,7 +2,7 @@ version: "2" checks: argument-count: config: - threshold: 8 + threshold: 9 complex-logic: config: threshold: 4 diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb index e53ef4bb3..e8178b828 100644 --- a/lib/team_statistics.rb +++ b/lib/team_statistics.rb @@ -201,7 +201,7 @@ def top_media_tags data[tag] += demand end end - data.sort_by{ |key, value| value }.reverse.first(5).to_h + data.sort_by{ |_key, value| value }.reverse.first(5).to_h end # For both articles and tiplines From d340513603ec0d91e94cd7503741c99559034a43 Mon Sep 17 00:00:00 2001 From: Caio <117518+caiosba@users.noreply.github.com> Date: Mon, 28 Oct 2024 00:32:41 -0300 Subject: [PATCH 26/26] Ticket CV2-5389: Fixing Code Climate issues --- .rubocop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 2ff1fbf74..78aae1b28 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -243,10 +243,10 @@ Metrics/ModuleLength: Max: 250 Metrics/ParameterLists: - Description: 'Avoid parameter lists longer than three or four parameters.' + Description: 'Avoid parameter lists longer than 9 parameters.' StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#too-many-params' Enabled: true - Max: 8 + Max: 9 Metrics/PerceivedComplexity: Description: >-