From 1a06c8e99d9f65cc6ca4c42a87bae6b7ecdfb941 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 10 Aug 2024 17:18:26 +0200 Subject: [PATCH 01/20] Merge class resolver and add new spec --- Gemfile | 1 + app/services/class_resolver.rb | 12 ++++++++++ spec/services/class_resolver_spec.rb | 36 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 app/services/class_resolver.rb create mode 100644 spec/services/class_resolver_spec.rb diff --git a/Gemfile b/Gemfile index f2b2ab43..45fc470e 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,7 @@ source "https://rubygems.org" ruby "3.3.4" +gem "aasm", "~> 5.5" gem "amazing_print" gem "bootsnap", require: false gem "jsbundling-rails" diff --git a/app/services/class_resolver.rb b/app/services/class_resolver.rb new file mode 100644 index 00000000..f8d97d74 --- /dev/null +++ b/app/services/class_resolver.rb @@ -0,0 +1,12 @@ +class ClassResolver + attr_reader :class_name, :suffix + + def initialize(class_name, suffix: nil) + @class_name = class_name + @suffix = suffix + end + + def resolve + [class_name, suffix].join("_").classify.constantize + end +end diff --git a/spec/services/class_resolver_spec.rb b/spec/services/class_resolver_spec.rb new file mode 100644 index 00000000..0f7d87a9 --- /dev/null +++ b/spec/services/class_resolver_spec.rb @@ -0,0 +1,36 @@ +require "rails_helper" + +RSpec.describe ClassResolver do + describe "initialization" do + it "sets expected parameters" do + resolver = described_class.new("test", suffix: "suffix") + expect(resolver.class_name).to eq("test") + expect(resolver.suffix).to eq("suffix") + end + + it "allows suffix to be nil" do + resolver = described_class.new("test") + expect(resolver.class_name).to eq("test") + expect(resolver.suffix).to be_nil + end + end + + describe "#resolve" do + it "resolves class by name" do + stub_const("TestService", Class.new) + resolver = described_class.new("test_service") + expect(resolver.resolve).to eq(TestService) + end + + it "resolves class with suffix" do + stub_const("TestService", Class.new) + resolver = described_class.new("test", suffix: "service") + expect(resolver.resolve).to eq(TestService) + end + + it "raises a NameError for missing class" do + resolver = described_class.new("non_existent") + expect { resolver.resolve }.to raise_error(NameError) + end + end +end From 77c21bcdeb608c333dc27b2534db1bc57b4b7d92 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 10 Aug 2024 17:28:28 +0200 Subject: [PATCH 02/20] Update gems --- Gemfile | 1 + Gemfile.lock | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index 45fc470e..5c2ecb75 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,7 @@ gem "pry-byebug" gem "pry-rails", "~> 0.3.9" gem "puma", "~> 6.4" gem "rails", "~> 7.1.3", ">= 7.1.3.4" +gem "rexml", ">= 3.3.4" # No direct dependency; added to mitigate a CVE gem "stimulus-rails" gem "turbo-rails" gem "tzinfo-data" diff --git a/Gemfile.lock b/Gemfile.lock index 559e1bfd..39290231 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -87,7 +87,7 @@ GEM base64 (0.2.0) bigdecimal (3.1.8) bindex (0.8.1) - bootsnap (1.18.3) + bootsnap (1.18.4) msgpack (~> 1.2) brakeman (6.1.2) racc @@ -97,7 +97,7 @@ GEM thor (~> 1.0) byebug (11.1.3) coderay (1.1.3) - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) crack (1.0.0) bigdecimal @@ -145,14 +145,14 @@ GEM railties (>= 5.0.0) globalid (1.2.1) activesupport (>= 6.1) - hashdiff (1.1.0) + hashdiff (1.1.1) i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.2) irb (1.14.0) rdoc (>= 4.0.0) reline (>= 0.4.2) - jsbundling-rails (1.3.0) + jsbundling-rails (1.3.1) railties (>= 6.0.0) json (2.7.2) language_server-protocol (3.17.0.3) @@ -196,8 +196,8 @@ GEM racc (~> 1.4) nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) - parallel (1.25.1) - parser (3.3.4.0) + parallel (1.26.1) + parser (3.3.4.2) ast (~> 2.4.1) racc pg (1.5.7) @@ -219,7 +219,7 @@ GEM public_suffix (6.0.1) puma (6.4.2) nio4r (~> 2.0) - racc (1.8.0) + racc (1.8.1) rack (3.1.7) rack-session (2.0.0) rack (>= 3.0.0) @@ -269,7 +269,7 @@ GEM regexp_parser (2.9.2) reline (0.5.9) io-console (~> 0.5) - rexml (3.3.2) + rexml (3.3.4) strscan rspec-core (3.13.0) rspec-support (~> 3.13.0) @@ -299,7 +299,7 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.3) + rubocop-ast (1.32.0) parser (>= 3.3.1.0) rubocop-factory_bot (2.26.1) rubocop (~> 1.61) @@ -311,7 +311,7 @@ GEM rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rspec (3.0.3) + rubocop-rspec (3.0.4) rubocop (~> 1.61) rubocop-rspec_rails (2.30.0) rubocop (~> 1.61) @@ -364,7 +364,7 @@ GEM websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) yaml-lint (0.1.2) - zeitwerk (2.6.16) + zeitwerk (2.6.17) PLATFORMS aarch64-linux @@ -393,6 +393,7 @@ DEPENDENCIES puma (~> 6.4) rails (~> 7.1.3, >= 7.1.3.4) reek (~> 6.3) + rexml (>= 3.3.4) rspec-rails (~> 6.1) rubocop rubocop-factory_bot From fd029ac7d1a187a16c9e0fa105efcea1326307f4 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sun, 11 Aug 2024 15:47:11 +0200 Subject: [PATCH 03/20] Require only necessary engines --- config/application.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config/application.rb b/config/application.rb index 360fe3d7..3d35200a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,17 +1,18 @@ require_relative "boot" require "rails" -# Pick the frameworks you want: -require "active_model/railtie" -require "active_job/railtie" + +# Picking only the required gems +# SEE: https://github.com/rails/rails/blob/main/railties/lib/rails/all.rb require "active_record/railtie" # require "active_storage/engine" require "action_controller/railtie" +require "action_view/railtie" # require "action_mailer/railtie" +require "active_job/railtie" +# require "action_cable/engine" # require "action_mailbox/engine" # require "action_text/engine" -require "action_view/railtie" -# require "action_cable/engine" require "rails/test_unit/railtie" # Require the gems listed in Gemfile, including any gems From 55cf57077f456fd2535ed100b9ead34a96fbf57d Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sun, 11 Aug 2024 15:47:28 +0200 Subject: [PATCH 04/20] Add app configuration --- config/application.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/application.rb b/config/application.rb index 3d35200a..ff60ccfa 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,5 +39,18 @@ class Application < Rails::Application # Don't generate system test files. config.generators.system_tests = nil + + # + # Manually added to the originally generated configuration + # + + config.hosts << "feeder.local" + config.hosts << "frf.im" + config.hosts << "localhost" + + # Use Redis cache store on all environments (see docker-compose.yml) + config.cache_store = :redis_cache_store, {url: ENV.fetch("REDIS_URL")} + + config.ssl_options = {hsts: {subdomains: true}} end end From 7b03afc8e37eb5671873474ca1a1a166431689e5 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 14:57:44 +0200 Subject: [PATCH 05/20] Add comments to `ClassResolver` --- app/services/class_resolver.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/services/class_resolver.rb b/app/services/class_resolver.rb index f8d97d74..7f03948f 100644 --- a/app/services/class_resolver.rb +++ b/app/services/class_resolver.rb @@ -1,11 +1,18 @@ +# Resolver a class from the class name and optional suffix. +# class ClassResolver attr_reader :class_name, :suffix + # @param [String] a string representing the target class name with words + # separated by underscores + # @param suffix: [String] optional suffix for the class name def initialize(class_name, suffix: nil) @class_name = class_name @suffix = suffix end + # @return [Class] + # @raise [NameError] if the target class is missing def resolve [class_name, suffix].join("_").classify.constantize end From ad3d89273eec6853ac90b4b94d7b1b5523db8e02 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 14:58:03 +0200 Subject: [PATCH 06/20] Define custom error classes --- app/errors/abstract_method_error.rb | 2 ++ app/errors/feed_configuration_error.rb | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 app/errors/abstract_method_error.rb create mode 100644 app/errors/feed_configuration_error.rb diff --git a/app/errors/abstract_method_error.rb b/app/errors/abstract_method_error.rb new file mode 100644 index 00000000..3b71eca1 --- /dev/null +++ b/app/errors/abstract_method_error.rb @@ -0,0 +1,2 @@ +class AbstractMethodError < StandardError +end diff --git a/app/errors/feed_configuration_error.rb b/app/errors/feed_configuration_error.rb new file mode 100644 index 00000000..69c81de1 --- /dev/null +++ b/app/errors/feed_configuration_error.rb @@ -0,0 +1,2 @@ +class FeedConfigurationError < StandardError +end From 16f1ee48ae3280023304a9d13dda00fecbf56563 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 14:58:32 +0200 Subject: [PATCH 07/20] Define basic service abstractions --- app/loaders/base_loader.rb | 12 ++++++++++++ app/normalizers/base_normalizer.rb | 13 +++++++++++++ app/processors/base_processor.rb | 13 +++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 app/loaders/base_loader.rb create mode 100644 app/normalizers/base_normalizer.rb create mode 100644 app/processors/base_processor.rb diff --git a/app/loaders/base_loader.rb b/app/loaders/base_loader.rb new file mode 100644 index 00000000..fc4e0b56 --- /dev/null +++ b/app/loaders/base_loader.rb @@ -0,0 +1,12 @@ +class BaseLoader + attr_reader :feed + + def initialize(feed:) + @feed = feed + end + + # @return [FeedContent] + def load + raise AbstractMethodError + end +end diff --git a/app/normalizers/base_normalizer.rb b/app/normalizers/base_normalizer.rb new file mode 100644 index 00000000..dfedabb9 --- /dev/null +++ b/app/normalizers/base_normalizer.rb @@ -0,0 +1,13 @@ +class BaseNormalizer + attr_reader :feed, :entity + + def initialize(feed:, entity:) + @feed = feed + @entity = entity + end + + # @return [Hash] + def normalize + raise AbstractMethodError + end +end diff --git a/app/processors/base_processor.rb b/app/processors/base_processor.rb new file mode 100644 index 00000000..66537d41 --- /dev/null +++ b/app/processors/base_processor.rb @@ -0,0 +1,13 @@ +class BaseProcessor + attr_reader :feed, :content + + def initialize(feed:, content:) + @feed = feed + @content = content + end + + # @return [Array] + def process + raise AbstractMethodError + end +end From 1579cd23a9a3defeecd2e04dea3c0b0e493d7533 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 14:59:15 +0200 Subject: [PATCH 08/20] Introduce Feed model and data objects --- app/models/feed.rb | 39 ++++++++++++++++++++++++++++++++++++++ app/models/feed_content.rb | 12 ++++++++++++ app/models/feed_entity.rb | 13 +++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 app/models/feed.rb create mode 100644 app/models/feed_content.rb create mode 100644 app/models/feed_entity.rb diff --git a/app/models/feed.rb b/app/models/feed.rb new file mode 100644 index 00000000..16409bee --- /dev/null +++ b/app/models/feed.rb @@ -0,0 +1,39 @@ +class Feed < ApplicationRecord + def supported? + !!(loader_class && processor_class && normalizer_class) + rescue NameError + false + end + + def service_classes + { + loader_class: loader_class rescue nil, + processor_class: processor_class rescue nil, + normalizer_class: normalizer_class rescue nil + } + end + + def loader_class + ClassResolver.new(loader, suffix: "loader").resolve + end + + def loader_instance + loader_class.new(self) + end + + def processor_class + ClassResolver.new(processor, suffix: "processor").resolve + end + + def processor_instance + processor_class.new(self) + end + + def normalizer_class + ClassResolver.new(normalizer, suffix: "normalizer").resolve + end + + def normalizer_instance + normalizer_class.new(self) + end +end diff --git a/app/models/feed_content.rb b/app/models/feed_content.rb new file mode 100644 index 00000000..f3534d25 --- /dev/null +++ b/app/models/feed_content.rb @@ -0,0 +1,12 @@ +# Data object for raw feed content representation. All loaders return +# a FeedContent instance. +# +# @see BaseLoader +# +class FeedContent + attr_reader :raw_content + + def initialize(raw_content) + @raw_content = raw_content + end +end diff --git a/app/models/feed_entity.rb b/app/models/feed_entity.rb new file mode 100644 index 00000000..458ca9d6 --- /dev/null +++ b/app/models/feed_entity.rb @@ -0,0 +1,13 @@ +# Data object representating a raw feed entity (like a blog post or RSS item). +# +class FeedEntity + attr_reader :feed, :uid, :content + + # @param uid: [String] unique entity identifier, like RSS item id + # or permalink URL + # @content [Object] arbitrary content representation + def initialize(uid:, content:) + @uid = uid + @content = content + end +end From 17ccc5f5dc02c477a9b71e7a651e49e5763420e8 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 15:00:30 +0200 Subject: [PATCH 09/20] Introduce common utilities --- app/services/application_logger.rb | 50 ++++++++++++++++++++++++++++++ app/services/logging.rb | 5 +++ app/services/stopwatch.rb | 16 ++++++++++ app/services/text_helpers.rb | 3 ++ 4 files changed, 74 insertions(+) create mode 100644 app/services/application_logger.rb create mode 100644 app/services/logging.rb create mode 100644 app/services/stopwatch.rb create mode 100644 app/services/text_helpers.rb diff --git a/app/services/application_logger.rb b/app/services/application_logger.rb new file mode 100644 index 00000000..f99eb67f --- /dev/null +++ b/app/services/application_logger.rb @@ -0,0 +1,50 @@ +class ApplicationLogger + CLEAR = "\e[0m" + BOLD = "\e[1m" + + BLACK = "\e[30m" + RED = "\e[31m" + GREEN = "\e[32m" + YELLOW = "\e[33m" + BLUE = "\e[34m" + MAGENTA = "\e[35m" + CYAN = "\e[36m" + WHITE = "\e[37m" + + attr_reader :logger + + def initialize(logger) + @logger = logger + end + + def debug(message) + logger.debug(format_message(message, WHITE)) + end + + def info(message) + logger.info(format_message(message, BLUE)) + end + + def warn(message) + logger.warn(format_message(message, YELLOW)) + end + + def error(message) + logger.error(format_message(message, RED)) + end + + def success(message) + logger.info(format_message(message, GREEN)) + end + + private + + def format_message(message, color) + return message unless colorize? + "#{color}#{message}#{CLEAR}" + end + + def colorize? + @colorize ||= ENV["NO_COLOR"].blank? + end +end diff --git a/app/services/logging.rb b/app/services/logging.rb new file mode 100644 index 00000000..b5541033 --- /dev/null +++ b/app/services/logging.rb @@ -0,0 +1,5 @@ +module Logging + def logger + @logger ||= ApplicationLogger.new(Rails.logger) + end +end diff --git a/app/services/stopwatch.rb b/app/services/stopwatch.rb new file mode 100644 index 00000000..dc618836 --- /dev/null +++ b/app/services/stopwatch.rb @@ -0,0 +1,16 @@ +class Stopwatch + attr_reader :start + + def self.measure(&block) + instance = new + block.call, instance.elapsed + end + + def initialize + @start = Time.current + end + + def elapsed + Time.current - start + end +end diff --git a/app/services/text_helpers.rb b/app/services/text_helpers.rb new file mode 100644 index 00000000..70f3e26a --- /dev/null +++ b/app/services/text_helpers.rb @@ -0,0 +1,3 @@ +class TextHelpers + extend ActionView::Helpers::TextHelper +end From 18f11dd5dec4595f65af65f2093bfbadfe224f81 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 15:01:39 +0200 Subject: [PATCH 10/20] Introduce boilerplate for the feeder workflow --- .reek.yml | 4 ++ app/loaders/base_loader.rb | 8 +--- app/models/feed.rb | 14 +++--- app/normalizers/base_normalizer.rb | 14 ++---- app/processors/base_processor.rb | 12 ++--- app/services/feed_importer.rb | 71 ++++++++++++++++++++++++++++++ app/services/feed_processor.rb | 21 +++++++++ app/services/feed_service.rb | 10 +++++ app/services/post_builder.rb | 11 +++++ app/services/publisher.rb | 26 +++++++++++ app/services/stopwatch.rb | 2 +- 11 files changed, 161 insertions(+), 32 deletions(-) create mode 100644 app/services/feed_importer.rb create mode 100644 app/services/feed_processor.rb create mode 100644 app/services/feed_service.rb create mode 100644 app/services/post_builder.rb create mode 100644 app/services/publisher.rb diff --git a/.reek.yml b/.reek.yml index 51944011..7577e984 100644 --- a/.reek.yml +++ b/.reek.yml @@ -20,5 +20,9 @@ detectors: accept: - "e" - "_" + UnusedParameters: + exclude: + - "BaseNormalizer#normalize" + - "BaseProcessor#process" UtilityFunction: public_methods_only: true diff --git a/app/loaders/base_loader.rb b/app/loaders/base_loader.rb index fc4e0b56..73ed5629 100644 --- a/app/loaders/base_loader.rb +++ b/app/loaders/base_loader.rb @@ -1,10 +1,4 @@ -class BaseLoader - attr_reader :feed - - def initialize(feed:) - @feed = feed - end - +class BaseLoader < FeedService # @return [FeedContent] def load raise AbstractMethodError diff --git a/app/models/feed.rb b/app/models/feed.rb index 16409bee..29a53e32 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,20 +1,20 @@ class Feed < ApplicationRecord def supported? !!(loader_class && processor_class && normalizer_class) - rescue NameError - false end def service_classes { - loader_class: loader_class rescue nil, - processor_class: processor_class rescue nil, - normalizer_class: normalizer_class rescue nil + loader_class: loader_class, + processor_class: processor_class, + normalizer_class: normalizer_class } end def loader_class ClassResolver.new(loader, suffix: "loader").resolve + rescue NameError + nil end def loader_instance @@ -23,6 +23,8 @@ def loader_instance def processor_class ClassResolver.new(processor, suffix: "processor").resolve + rescue NameError + nil end def processor_instance @@ -31,6 +33,8 @@ def processor_instance def normalizer_class ClassResolver.new(normalizer, suffix: "normalizer").resolve + rescue NameError + nil end def normalizer_instance diff --git a/app/normalizers/base_normalizer.rb b/app/normalizers/base_normalizer.rb index dfedabb9..f09cb78f 100644 --- a/app/normalizers/base_normalizer.rb +++ b/app/normalizers/base_normalizer.rb @@ -1,13 +1,7 @@ -class BaseNormalizer - attr_reader :feed, :entity - - def initialize(feed:, entity:) - @feed = feed - @entity = entity - end - - # @return [Hash] - def normalize +class BaseNormalizer < FeedService + # @param [FeedEntity] + # @return [Post] + def normalize(feed_entity:) raise AbstractMethodError end end diff --git a/app/processors/base_processor.rb b/app/processors/base_processor.rb index 66537d41..f5a50a1d 100644 --- a/app/processors/base_processor.rb +++ b/app/processors/base_processor.rb @@ -1,13 +1,7 @@ -class BaseProcessor - attr_reader :feed, :content - - def initialize(feed:, content:) - @feed = feed - @content = content - end - +class BaseProcessor < FeedService + # @param [FeedContent] # @return [Array] - def process + def process(feed_content:) raise AbstractMethodError end end diff --git a/app/services/feed_importer.rb b/app/services/feed_importer.rb new file mode 100644 index 00000000..07051fb6 --- /dev/null +++ b/app/services/feed_importer.rb @@ -0,0 +1,71 @@ +# Load content and generate posts for the specified feed. Raises an error +# in case feed processing is not possible. +# +class FeedImporter + include Logging + + attr_reader :feed + + # @param feed: [Feed] + def initialize(feed:) + @feed = feed + end + + # @raise [FeedConfigurationError] if the feed is missing related classes + def perform + log_info("importing feed ##{feed.id} (#{feed.name})") + ensure_services_resolved + feed_content = load_content(feed) + entities = process_feed_content(feed_content) + build_posts(entities) + end + + private + + def ensure_services_resolved + return if feed.supported? + track_feed_error(category: "configuration") + raise FeedConfigurationError + end + + # @return [FeedContent] + # @raise [StandardError] if processor execution is not possible + def load_content(feed) + feed.loader_instance.load + rescue StandardError => e + track_feed_error(error: e, category: "loader") + raise e + end + + # @return [Array] + # @raise [StandardError] if content processing is not possible + def process_feed_content(feed_content) + feed.processor_instance.process(feed_content) + rescue StandardError => e + track_feed_error(error: e, category: "processor", context: {content: feed_content}) + raise e + end + + def build_posts(entities) + entities.each do |entity| + PostBuilder.new(feed: feed, feed_entity: entity).build + end + end + + # :reek:UnusedParameters + def track_feed_error(category:, error: nil, context: {}) + ActiveRecord::Base.transaction do + feed.update!(errored_at: Time.current, errors_count: feed.errors_count.succ) + + # TBD: Add error details to the record + Error.create!(target: feed, category: category, context: context.merge(error_context_defaults)) + end + end + + def error_context_defaults + { + feed_supported: feed.supported?, + feed_service_classes: feed.service_classes + } + end +end diff --git a/app/services/feed_processor.rb b/app/services/feed_processor.rb new file mode 100644 index 00000000..1f8ea220 --- /dev/null +++ b/app/services/feed_processor.rb @@ -0,0 +1,21 @@ +# Import content for each feed in the specified scope, and publish new posts. +# +class FeedProcessor < FeedService + include Logging + + attr_reader :feeds + + # @param feeds: [Enumerable] feeds to process + def initialize(feeds:) + @feeds = feeds + end + + def perform + feeds.each do |feed| + FeedImporter.new(feed: feed).perform + Publisher.new(posts: feed.posts.pending).publish + rescue StandardError + log_error("feed processing interrupted: #{feed.name} (id: #{feed.id})") + end + end +end diff --git a/app/services/feed_service.rb b/app/services/feed_service.rb new file mode 100644 index 00000000..1e4c6169 --- /dev/null +++ b/app/services/feed_service.rb @@ -0,0 +1,10 @@ +# Basic abstraction for feed services. +# +class FeedService + attr_reader :feed + + # @param [Feed] + def initialize(feed) + @feed = feed + end +end diff --git a/app/services/post_builder.rb b/app/services/post_builder.rb new file mode 100644 index 00000000..91cce3b6 --- /dev/null +++ b/app/services/post_builder.rb @@ -0,0 +1,11 @@ +class PostBuilder + attr_reader :feed_entity + def initialize(feed_entity:) + @feed_entity = feed_entity + end + + # @return [Post] + def build + # TBD + end +end diff --git a/app/services/publisher.rb b/app/services/publisher.rb new file mode 100644 index 00000000..977d14f7 --- /dev/null +++ b/app/services/publisher.rb @@ -0,0 +1,26 @@ +class Publisher + include LoggingHelper + + def initialize(posts:) + @posts = posts + end + + def publish + logger.info("publishing #{TextHelpers.pluralize(pending_posts.count, "posts")}") + + pending_posts.each do |post| + publish_post(post) + end + end + + private + + def pending_posts + @pending_posts ||= posts.filter(&:pending?) + end + + # :reek:UnusedParameters + def publish_post(post) + # TBD + end +end diff --git a/app/services/stopwatch.rb b/app/services/stopwatch.rb index dc618836..4a044814 100644 --- a/app/services/stopwatch.rb +++ b/app/services/stopwatch.rb @@ -3,7 +3,7 @@ class Stopwatch def self.measure(&block) instance = new - block.call, instance.elapsed + [block.call, instance.elapsed] end def initialize From 9e0552215caa630bfe416dd3576153eb5134d568 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 17:46:36 +0200 Subject: [PATCH 11/20] Fix module name --- app/services/publisher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/publisher.rb b/app/services/publisher.rb index 977d14f7..a4b85636 100644 --- a/app/services/publisher.rb +++ b/app/services/publisher.rb @@ -1,5 +1,5 @@ class Publisher - include LoggingHelper + include Logging def initialize(posts:) @posts = posts From b0ea53aef5d79d59860be5a56f2cc499fd644134 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 17:46:58 +0200 Subject: [PATCH 12/20] Add error class name to the context --- app/services/feed_importer.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/services/feed_importer.rb b/app/services/feed_importer.rb index 07051fb6..88cbea75 100644 --- a/app/services/feed_importer.rb +++ b/app/services/feed_importer.rb @@ -52,20 +52,18 @@ def build_posts(entities) end end - # :reek:UnusedParameters def track_feed_error(category:, error: nil, context: {}) ActiveRecord::Base.transaction do feed.update!(errored_at: Time.current, errors_count: feed.errors_count.succ) - - # TBD: Add error details to the record - Error.create!(target: feed, category: category, context: context.merge(error_context_defaults)) + Error.create!(target: feed, category: category, context: context.merge(error_context(error))) end end - def error_context_defaults + def error_context(error) { feed_supported: feed.supported?, - feed_service_classes: feed.service_classes + feed_service_classes: feed.service_classes, + error: error.class } end end From c212fad35bff5e43cb7b37aeb0841bbbf1d8c3e4 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 20:50:34 +0200 Subject: [PATCH 13/20] Add `memo_wise` gem --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index 5c2ecb75..996d7c29 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ gem "aasm", "~> 5.5" gem "amazing_print" gem "bootsnap", require: false gem "jsbundling-rails" +gem "memo_wise", "~> 1.9" gem "pg", "~> 1.5" gem "propshaft" gem "pry", "~> 0.14" diff --git a/Gemfile.lock b/Gemfile.lock index 39290231..3cc5d726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -169,6 +169,7 @@ GEM marginalia (1.11.1) actionpack (>= 5.2) activerecord (>= 5.2) + memo_wise (1.9.0) method_source (1.1.0) mini_mime (1.1.5) minitest (5.24.1) @@ -385,6 +386,7 @@ DEPENDENCIES factory_bot_rails (~> 6.4) jsbundling-rails marginalia (~> 1.11) + memo_wise (~> 1.9) pg (~> 1.5) propshaft pry (~> 0.14) From 43c6e8338cf9e3070c31e46d93fc3fc2081b81b5 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 20:51:13 +0200 Subject: [PATCH 14/20] Make `ApplicationLogger` thread-safe and more efficient with blocks --- app/services/application_logger.rb | 49 +++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/app/services/application_logger.rb b/app/services/application_logger.rb index f99eb67f..079257f8 100644 --- a/app/services/application_logger.rb +++ b/app/services/application_logger.rb @@ -1,3 +1,12 @@ +# ApplicationLogger is a wrapper for Ruby loggers that adds color formatting to +# log messages. Color formatting can be enabled or disabled, and it respects the +# NO_COLOR environment variable by default. +# +# Usage: +# logger = ApplicationLogger.new +# logger.info "This is an info message" +# logger.info { "This line will be evaluated only if the logger" } +# class ApplicationLogger CLEAR = "\e[0m" BOLD = "\e[1m" @@ -13,38 +22,48 @@ class ApplicationLogger attr_reader :logger - def initialize(logger) + def initialize(logger: Rails.logger, colorize: ENV["NO_COLOR"].blank?) @logger = logger + @colorize = colorize end - def debug(message) - logger.debug(format_message(message, WHITE)) + def debug(message = nil, &) + log_formatted_message(:debug, message, WHITE, &) end - def info(message) - logger.info(format_message(message, BLUE)) + def info(message = nil, &) + log_formatted_message(:info, message, BLUE, &) end - def warn(message) - logger.warn(format_message(message, YELLOW)) + def warn(message = nil, &) + log_formatted_message(:warn, message, YELLOW, &) end - def error(message) - logger.error(format_message(message, RED)) + def error(message = nil, &) + log_formatted_message(:error, message, RED, &) end - def success(message) - logger.info(format_message(message, GREEN)) + def success(message = nil, &) + log_formatted_message(:info, message, GREEN, &) end - private + def log_formatted_message(level, message, color, &block) + if block_given? + logger.send(level, &-> { format_message(block.call, color) }) + else + logger.send(level, format_message(message, color)) + end + end def format_message(message, color) - return message unless colorize? - "#{color}#{message}#{CLEAR}" + if colorize? + "#{color}#{message}#{CLEAR}" + else + message + end end def colorize? - @colorize ||= ENV["NO_COLOR"].blank? + @colorize end end From a88d36e94c31a05ba85477777e31bb8a1a04fb6a Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 20:56:56 +0200 Subject: [PATCH 15/20] Annotate `PostBuilder` --- app/services/post_builder.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/services/post_builder.rb b/app/services/post_builder.rb index 91cce3b6..312dc672 100644 --- a/app/services/post_builder.rb +++ b/app/services/post_builder.rb @@ -1,11 +1,17 @@ +# Hydrate new Post instance from a FeedEntity. Always returns a valid Post, +# though it may not be suitable for publication depending on the content. +# class PostBuilder - attr_reader :feed_entity - def initialize(feed_entity:) + attr_reader :feed, :feed_entity + + def initialize(feed:, feed_entity:) + @feed = feed @feed_entity = feed_entity end # @return [Post] def build # TBD + Post.new(feed: feed) end end From 6eeb0d34bfcb7dd63443682cdb82b097192f4421 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 21:34:22 +0200 Subject: [PATCH 16/20] Introduce `ErrorReport` model and service --- app/models/error_report.rb | 3 ++ app/services/error_reporter.rb | 41 +++++++++++++++++++ .../20240817185724_create_error_reports.rb | 17 ++++++++ db/schema.rb | 20 ++++++++- 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 app/models/error_report.rb create mode 100644 app/services/error_reporter.rb create mode 100644 db/migrate/20240817185724_create_error_reports.rb diff --git a/app/models/error_report.rb b/app/models/error_report.rb new file mode 100644 index 00000000..6aec3ddf --- /dev/null +++ b/app/models/error_report.rb @@ -0,0 +1,3 @@ +class ErrorReport < ApplicationRecord + belongs_to :target, polymorphic: true, optional: true +end diff --git a/app/services/error_reporter.rb b/app/services/error_reporter.rb new file mode 100644 index 00000000..98ed067a --- /dev/null +++ b/app/services/error_reporter.rb @@ -0,0 +1,41 @@ +# Hidrate new instance of error report with +class ErrorReporter + # @param error: [Exception] + # @params category: [String] + # @param target: [ActiveRecord::Base] target record (optional) + # @param context: [Hash] JSON-friendly arbitrary hash object (optional) + # @param occured_at: [DateTime] error timestamp (optional) + def report(error:, category:, **options) + ErrorReport.create!( + category: category, + target: options[:target], + context: options[:context] || {}, + occured_at: options[:occured_at] || Time.current, + **error_details(error) + ) + end + + private + + def error_details(error) + { + error_class: error.class.name, + message: error.message, + **backtrace_dtails(error.backtrace || []) + } + end + + def backtrace_dtails(backtrace) + match = backtrace&.first&.match(/^(.+):(\d+):in/) + + if match + { + backtrace: backtrace, + file_name: match[1], + line_number: match[2].to_i + } + else + {backtrace: backtrace} + end + end +end diff --git a/db/migrate/20240817185724_create_error_reports.rb b/db/migrate/20240817185724_create_error_reports.rb new file mode 100644 index 00000000..89d283ed --- /dev/null +++ b/db/migrate/20240817185724_create_error_reports.rb @@ -0,0 +1,17 @@ +class CreateErrorReports < ActiveRecord::Migration[7.1] + def change + create_table :error_reports do |t| + t.references "target", polymorphic: true, index: true + t.string "category", index: true + t.string "error_class", default: "", null: false + t.string "file_name" + t.integer "line_number" + t.string "message", default: "", null: false + t.string "backtrace", default: [], null: false, array: true + t.jsonb "context", default: {}, null: false + t.datetime "occured_at", null: false, index: true + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 34cf1ff1..0d206c14 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,28 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_07_28_141124) do +ActiveRecord::Schema[7.1].define(version: 2024_08_17_185724) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "error_reports", force: :cascade do |t| + t.string "target_type" + t.bigint "target_id" + t.string "category" + t.string "error_class", default: "", null: false + t.string "file_name" + t.integer "line_number" + t.string "message", default: "", null: false + t.string "backtrace", default: [], null: false, array: true + t.jsonb "context", default: {}, null: false + t.datetime "occured_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["category"], name: "index_error_reports_on_category" + t.index ["occured_at"], name: "index_error_reports_on_occured_at" + t.index ["target_type", "target_id"], name: "index_error_reports_on_target" + end + create_table "errors", id: :serial, force: :cascade do |t| t.integer "status", default: 0, null: false t.string "exception", default: "", null: false From 22886d1d0ee5812f06c6539b34b9ea340a0b6f9c Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sat, 17 Aug 2024 21:59:27 +0200 Subject: [PATCH 17/20] Simplify errors handling --- app/models/feed.rb | 9 ++++-- app/services/feed_processor.rb | 4 +-- .../{feed_importer.rb => importer.rb} | 31 ++++++++++++------- 3 files changed, 28 insertions(+), 16 deletions(-) rename app/services/{feed_importer.rb => importer.rb} (66%) diff --git a/app/models/feed.rb b/app/models/feed.rb index 29a53e32..e4c43d20 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,6 +1,11 @@ class Feed < ApplicationRecord - def supported? - !!(loader_class && processor_class && normalizer_class) + def readable_id + [self.class.name.underscore, id, name].compact_blank.join("-") + end + + def ensure_supported! + return if loader_class && processor_class && normalizer_class + raise FeedConfigurationError end def service_classes diff --git a/app/services/feed_processor.rb b/app/services/feed_processor.rb index 1f8ea220..a97027dc 100644 --- a/app/services/feed_processor.rb +++ b/app/services/feed_processor.rb @@ -12,10 +12,10 @@ def initialize(feeds:) def perform feeds.each do |feed| - FeedImporter.new(feed: feed).perform + Importer.new(feed: feed).import Publisher.new(posts: feed.posts.pending).publish rescue StandardError - log_error("feed processing interrupted: #{feed.name} (id: #{feed.id})") + logger.error { "feed processing interrupted: #{feed.name} (id: #{feed.id})" } end end end diff --git a/app/services/feed_importer.rb b/app/services/importer.rb similarity index 66% rename from app/services/feed_importer.rb rename to app/services/importer.rb index 88cbea75..c5d858d7 100644 --- a/app/services/feed_importer.rb +++ b/app/services/importer.rb @@ -1,7 +1,7 @@ # Load content and generate posts for the specified feed. Raises an error # in case feed processing is not possible. # -class FeedImporter +class Importer include Logging attr_reader :feed @@ -11,9 +11,11 @@ def initialize(feed:) @feed = feed end + # Import feed content and persist new posts. Will raise an error if the feed + # processing workflow should be interrupted. # @raise [FeedConfigurationError] if the feed is missing related classes - def perform - log_info("importing feed ##{feed.id} (#{feed.name})") + def import + logger.info("importing #{feed.readable_id}") ensure_services_resolved feed_content = load_content(feed) entities = process_feed_content(feed_content) @@ -23,9 +25,9 @@ def perform private def ensure_services_resolved - return if feed.supported? - track_feed_error(category: "configuration") - raise FeedConfigurationError + feed.ensure_supported! + rescue StandardError => e + track_feed_error(error: e, category: "configuration") end # @return [FeedContent] @@ -33,7 +35,7 @@ def ensure_services_resolved def load_content(feed) feed.loader_instance.load rescue StandardError => e - track_feed_error(error: e, category: "loader") + track_feed_error(error: e, category: "loading") raise e end @@ -42,7 +44,7 @@ def load_content(feed) def process_feed_content(feed_content) feed.processor_instance.process(feed_content) rescue StandardError => e - track_feed_error(error: e, category: "processor", context: {content: feed_content}) + track_feed_error(error: e, category: "processing", context: {content: feed_content}) raise e end @@ -55,15 +57,20 @@ def build_posts(entities) def track_feed_error(category:, error: nil, context: {}) ActiveRecord::Base.transaction do feed.update!(errored_at: Time.current, errors_count: feed.errors_count.succ) - Error.create!(target: feed, category: category, context: context.merge(error_context(error))) + + ErrorReporter.report( + error: error, + target: feed, + category: category, + context: context.merge(error_context) + ) end end - def error_context(error) + def error_context { feed_supported: feed.supported?, - feed_service_classes: feed.service_classes, - error: error.class + feed_service_classes: feed.service_classes } end end From 0e166a081470fea1b32cffea1ceb9e0f035774de Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sun, 18 Aug 2024 02:04:05 +0200 Subject: [PATCH 18/20] Add more fields --- app/models/feed_content.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/feed_content.rb b/app/models/feed_content.rb index f3534d25..f37a71e6 100644 --- a/app/models/feed_content.rb +++ b/app/models/feed_content.rb @@ -4,9 +4,11 @@ # @see BaseLoader # class FeedContent - attr_reader :raw_content + attr_reader :raw_content, :imported_at, :import_duration - def initialize(raw_content) + def initialize(raw_content:, imported_at:, import_duration:) @raw_content = raw_content + @imported_at = imported_at + @import_duration = import_duration end end From a001e841a6f04fdda0cdd7b6f5cb0da1a7fece74 Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sun, 18 Aug 2024 02:05:30 +0200 Subject: [PATCH 19/20] Rubocop --- app/services/application_logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/application_logger.rb b/app/services/application_logger.rb index 079257f8..ece84880 100644 --- a/app/services/application_logger.rb +++ b/app/services/application_logger.rb @@ -48,7 +48,7 @@ def success(message = nil, &) end def log_formatted_message(level, message, color, &block) - if block_given? + if block logger.send(level, &-> { format_message(block.call, color) }) else logger.send(level, format_message(message, color)) From aa370433779e64b376e6a1386c1dbee91f0f3afd Mon Sep 17 00:00:00 2001 From: Alex Musayev Date: Sun, 18 Aug 2024 02:06:14 +0200 Subject: [PATCH 20/20] Reek --- app/models/feed.rb | 2 +- app/services/importer.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/feed.rb b/app/models/feed.rb index e4c43d20..21269828 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -3,7 +3,7 @@ def readable_id [self.class.name.underscore, id, name].compact_blank.join("-") end - def ensure_supported! + def ensure_supported return if loader_class && processor_class && normalizer_class raise FeedConfigurationError end diff --git a/app/services/importer.rb b/app/services/importer.rb index c5d858d7..53d908b6 100644 --- a/app/services/importer.rb +++ b/app/services/importer.rb @@ -25,7 +25,7 @@ def import private def ensure_services_resolved - feed.ensure_supported! + feed.ensure_supported rescue StandardError => e track_feed_error(error: e, category: "configuration") end