From 33d83fa7305db57d4b222a2f6b1d582466bbe32b Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Mon, 1 Jan 2024 20:23:34 +0100 Subject: [PATCH] fix the object analytics fetching after the new year change --- Gemfile.lock | 5 ++++ bin/import_google_ua_analytics_data | 12 ++++++---- lib/ncbo_cron/analytics/object_analytics.rb | 23 +++++++++++-------- .../analytics/page_visits_analytics.rb | 4 ++-- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d752fe7d..182c2074 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -135,6 +135,7 @@ GEM faraday (>= 1.0, < 3.a) google-cloud-errors (1.3.1) google-protobuf (3.25.1-x86_64-darwin) + google-protobuf (3.25.1-x86_64-linux) googleapis-common-protos (1.4.0) google-protobuf (~> 3.14) googleapis-common-protos-types (~> 1.2) @@ -151,6 +152,9 @@ GEM grpc (1.60.0-x86_64-darwin) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) + grpc (1.60.0-x86_64-linux) + google-protobuf (~> 3.25) + googleapis-common-protos-types (~> 1.0) htmlentities (4.3.4) http-accept (1.7.0) http-cookie (1.0.5) @@ -252,6 +256,7 @@ GEM PLATFORMS x86_64-darwin-23 + x86_64-linux DEPENDENCIES binding_of_caller (~> 1.0) diff --git a/bin/import_google_ua_analytics_data b/bin/import_google_ua_analytics_data index eadfd936..efc216d1 100755 --- a/bin/import_google_ua_analytics_data +++ b/bin/import_google_ua_analytics_data @@ -41,9 +41,10 @@ module NcboCron (start_year..Date.today.year).each do |y| aggregated_results[acronym] = Hash.new if aggregated_results[acronym].nil? aggregated_results[acronym][y.to_s] = Hash.new unless aggregated_results[acronym].has_key?(y.to_s) + # fill up non existent months with zeros + last_month = y.eql?(Date.today.year) ? Date.today.month : 12 + (1..last_month).each { |n| aggregated_results[acronym][y.to_s][n.to_s] = 0 if aggregated_results[acronym][y.to_s].is_a?(Hash) && !aggregated_results[acronym][y.to_s].has_key?(n.to_s)} end - # fill up non existent months with zeros - (1..12).each { |n| aggregated_results[acronym].values.each { |v| v[n.to_s] = 0 unless v.has_key?(n.to_s) } } break end end @@ -82,9 +83,12 @@ module NcboCron (start_year..Date.today.year).each do |y| aggregated_results = Hash.new if aggregated_results.nil? aggregated_results[y.to_s] = Hash.new unless aggregated_results.has_key?(y.to_s) + + # fill up non existent months with zeros + last_month = y.eql?(Date.today.year) ? Date.today.month.to_i : 12 + (1..last_month).each { |n| aggregated_results[y.to_s][n.to_s] = 0 if aggregated_results[y.to_s].is_a?(Hash) && !aggregated_results[y.to_s].has_key?(n.to_s)} end - # fill up non existent months with zeros - (1..12).each { |n| aggregated_results.values.each { |v| v[n.to_s] = 0 unless v.has_key?(n.to_s) } } + break end end diff --git a/lib/ncbo_cron/analytics/object_analytics.rb b/lib/ncbo_cron/analytics/object_analytics.rb index d4a7f3da..b5cca90d 100644 --- a/lib/ncbo_cron/analytics/object_analytics.rb +++ b/lib/ncbo_cron/analytics/object_analytics.rb @@ -113,7 +113,7 @@ def fetch_object_analytics(logger, ga_conn) private - def merge_and_fill_missing_data(new_data, old_data,logger, start_date = @start_date) + def merge_and_fill_missing_data(new_data, old_data, logger, start_date = @start_date) if !new_data.empty? logger.info "Merging old Google Analytics and the new data..." logger.flush @@ -123,7 +123,8 @@ def merge_and_fill_missing_data(new_data, old_data,logger, start_date = @start_d year = year.to_s if new_data[acronym].has_key?(year) if old_data[acronym].has_key?(year) - (1..Date.today.month).each do |month| + last_month = year.eql?(Date.today.year) ? Date.today.month : 12 + (1..last_month).each do |month| month = month.to_s old_data[acronym][year][month] ||= 0 unless old_data[acronym][year][month].eql?(new_data[acronym][year][month]) @@ -136,16 +137,14 @@ def merge_and_fill_missing_data(new_data, old_data,logger, start_date = @start_d end end else - old_data[acronym]= new_data[acronym] + old_data[acronym] = new_data[acronym] end end # fill missing years and months logger.info "Filling in missing years data..." old_data = fill_missing_data(old_data) end - - # sort_ga_data(old_data) - old_data + sort_ga_data(old_data) end def aggregate_results(aggregated_results, results) @@ -178,16 +177,22 @@ def fill_missing_data(ga_data) (start_year..Date.today.year).each do |y| ga_data[acronym] = Hash.new if ga_data[acronym].nil? ga_data[acronym][y.to_s] = Hash.new unless ga_data[acronym].has_key?(y.to_s) + + # fill up non existent months with zeros + last_month = y.eql?(Date.today.year) ? Date.today.month.to_i : 12 + (1..last_month).each { |n| ga_data[acronym][y.to_s][n.to_s] = 0 if ga_data[acronym][y.to_s].is_a?(Hash) && !ga_data[acronym][y.to_s].has_key?(n.to_s) } end - # fill up non existent months with zeros - (1..12).each { |n| ga_data[acronym].values.each { |v| v[n.to_s] = 0 if v.is_a?(Hash) && !v.has_key?(n.to_s) } } end end def sort_ga_data(ga_data) ga_data.transform_values { |value| value.transform_values { |val| - val.sort_by { |key, _| key.to_i }.to_h + if val.is_a?(Hash) + val.sort_by { |key, _| key.to_i }.to_h + else + val + end }.sort_by { |k, _| k.to_i }.to_h }.sort.to_h end diff --git a/lib/ncbo_cron/analytics/page_visits_analytics.rb b/lib/ncbo_cron/analytics/page_visits_analytics.rb index b2354c00..fd88588a 100644 --- a/lib/ncbo_cron/analytics/page_visits_analytics.rb +++ b/lib/ncbo_cron/analytics/page_visits_analytics.rb @@ -6,8 +6,8 @@ module NcboCron module Models class PageVisitsAnalytics < ObjectAnalytics - def initialize(start_date: Date.today.prev_month, old_data: {}) - super(redis_field: 'pages_analytics', start_date: Date.today.prev_month, old_data: { }) + def initialize(start_date: Date.today.prev_month.to_s, old_data: {}) + super(redis_field: 'pages_analytics', start_date: Date.today.prev_month.to_s, old_data: { }) end private