diff --git a/.reek.yml b/.reek.yml index 51944011..7577e984 100644 --- a/.reek.yml +++ b/.reek.yml @@ -20,5 +20,9 @@ detectors: accept: - "e" - "_" + UnusedParameters: + exclude: + - "BaseNormalizer#normalize" + - "BaseProcessor#process" UtilityFunction: public_methods_only: true diff --git a/Gemfile b/Gemfile index f2b2ab43..996d7c29 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,11 @@ source "https://rubygems.org" ruby "3.3.4" +gem "aasm", "~> 5.5" gem "amazing_print" gem "bootsnap", require: false gem "jsbundling-rails" +gem "memo_wise", "~> 1.9" gem "pg", "~> 1.5" gem "propshaft" gem "pry", "~> 0.14" @@ -12,6 +14,7 @@ gem "pry-byebug" gem "pry-rails", "~> 0.3.9" gem "puma", "~> 6.4" gem "rails", "~> 7.1.3", ">= 7.1.3.4" +gem "rexml", ">= 3.3.4" # No direct dependency; added to mitigate a CVE gem "stimulus-rails" gem "turbo-rails" gem "tzinfo-data" diff --git a/Gemfile.lock b/Gemfile.lock index 559e1bfd..3cc5d726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -87,7 +87,7 @@ GEM base64 (0.2.0) bigdecimal (3.1.8) bindex (0.8.1) - bootsnap (1.18.3) + bootsnap (1.18.4) msgpack (~> 1.2) brakeman (6.1.2) racc @@ -97,7 +97,7 @@ GEM thor (~> 1.0) byebug (11.1.3) coderay (1.1.3) - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) crack (1.0.0) bigdecimal @@ -145,14 +145,14 @@ GEM railties (>= 5.0.0) globalid (1.2.1) activesupport (>= 6.1) - hashdiff (1.1.0) + hashdiff (1.1.1) i18n (1.14.5) concurrent-ruby (~> 1.0) io-console (0.7.2) irb (1.14.0) rdoc (>= 4.0.0) reline (>= 0.4.2) - jsbundling-rails (1.3.0) + jsbundling-rails (1.3.1) railties (>= 6.0.0) json (2.7.2) language_server-protocol (3.17.0.3) @@ -169,6 +169,7 @@ GEM marginalia (1.11.1) actionpack (>= 5.2) activerecord (>= 5.2) + memo_wise (1.9.0) method_source (1.1.0) mini_mime (1.1.5) minitest (5.24.1) @@ -196,8 +197,8 @@ GEM racc (~> 1.4) nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) - parallel (1.25.1) - parser (3.3.4.0) + parallel (1.26.1) + parser (3.3.4.2) ast (~> 2.4.1) racc pg (1.5.7) @@ -219,7 +220,7 @@ GEM public_suffix (6.0.1) puma (6.4.2) nio4r (~> 2.0) - racc (1.8.0) + racc (1.8.1) rack (3.1.7) rack-session (2.0.0) rack (>= 3.0.0) @@ -269,7 +270,7 @@ GEM regexp_parser (2.9.2) reline (0.5.9) io-console (~> 0.5) - rexml (3.3.2) + rexml (3.3.4) strscan rspec-core (3.13.0) rspec-support (~> 3.13.0) @@ -299,7 +300,7 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.3) + rubocop-ast (1.32.0) parser (>= 3.3.1.0) rubocop-factory_bot (2.26.1) rubocop (~> 1.61) @@ -311,7 +312,7 @@ GEM rack (>= 1.1) rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rspec (3.0.3) + rubocop-rspec (3.0.4) rubocop (~> 1.61) rubocop-rspec_rails (2.30.0) rubocop (~> 1.61) @@ -364,7 +365,7 @@ GEM websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) yaml-lint (0.1.2) - zeitwerk (2.6.16) + zeitwerk (2.6.17) PLATFORMS aarch64-linux @@ -385,6 +386,7 @@ DEPENDENCIES factory_bot_rails (~> 6.4) jsbundling-rails marginalia (~> 1.11) + memo_wise (~> 1.9) pg (~> 1.5) propshaft pry (~> 0.14) @@ -393,6 +395,7 @@ DEPENDENCIES puma (~> 6.4) rails (~> 7.1.3, >= 7.1.3.4) reek (~> 6.3) + rexml (>= 3.3.4) rspec-rails (~> 6.1) rubocop rubocop-factory_bot diff --git a/app/errors/abstract_method_error.rb b/app/errors/abstract_method_error.rb new file mode 100644 index 00000000..3b71eca1 --- /dev/null +++ b/app/errors/abstract_method_error.rb @@ -0,0 +1,2 @@ +class AbstractMethodError < StandardError +end diff --git a/app/errors/feed_configuration_error.rb b/app/errors/feed_configuration_error.rb new file mode 100644 index 00000000..69c81de1 --- /dev/null +++ b/app/errors/feed_configuration_error.rb @@ -0,0 +1,2 @@ +class FeedConfigurationError < StandardError +end diff --git a/app/loaders/base_loader.rb b/app/loaders/base_loader.rb new file mode 100644 index 00000000..73ed5629 --- /dev/null +++ b/app/loaders/base_loader.rb @@ -0,0 +1,6 @@ +class BaseLoader < FeedService + # @return [FeedContent] + def load + raise AbstractMethodError + end +end diff --git a/app/models/error_report.rb b/app/models/error_report.rb new file mode 100644 index 00000000..6aec3ddf --- /dev/null +++ b/app/models/error_report.rb @@ -0,0 +1,3 @@ +class ErrorReport < ApplicationRecord + belongs_to :target, polymorphic: true, optional: true +end diff --git a/app/models/feed.rb b/app/models/feed.rb new file mode 100644 index 00000000..21269828 --- /dev/null +++ b/app/models/feed.rb @@ -0,0 +1,48 @@ +class Feed < ApplicationRecord + def readable_id + [self.class.name.underscore, id, name].compact_blank.join("-") + end + + def ensure_supported + return if loader_class && processor_class && normalizer_class + raise FeedConfigurationError + end + + def service_classes + { + loader_class: loader_class, + processor_class: processor_class, + normalizer_class: normalizer_class + } + end + + def loader_class + ClassResolver.new(loader, suffix: "loader").resolve + rescue NameError + nil + end + + def loader_instance + loader_class.new(self) + end + + def processor_class + ClassResolver.new(processor, suffix: "processor").resolve + rescue NameError + nil + end + + def processor_instance + processor_class.new(self) + end + + def normalizer_class + ClassResolver.new(normalizer, suffix: "normalizer").resolve + rescue NameError + nil + end + + def normalizer_instance + normalizer_class.new(self) + end +end diff --git a/app/models/feed_content.rb b/app/models/feed_content.rb new file mode 100644 index 00000000..f37a71e6 --- /dev/null +++ b/app/models/feed_content.rb @@ -0,0 +1,14 @@ +# Data object for raw feed content representation. All loaders return +# a FeedContent instance. +# +# @see BaseLoader +# +class FeedContent + attr_reader :raw_content, :imported_at, :import_duration + + def initialize(raw_content:, imported_at:, import_duration:) + @raw_content = raw_content + @imported_at = imported_at + @import_duration = import_duration + end +end diff --git a/app/models/feed_entity.rb b/app/models/feed_entity.rb new file mode 100644 index 00000000..458ca9d6 --- /dev/null +++ b/app/models/feed_entity.rb @@ -0,0 +1,13 @@ +# Data object representating a raw feed entity (like a blog post or RSS item). +# +class FeedEntity + attr_reader :feed, :uid, :content + + # @param uid: [String] unique entity identifier, like RSS item id + # or permalink URL + # @content [Object] arbitrary content representation + def initialize(uid:, content:) + @uid = uid + @content = content + end +end diff --git a/app/normalizers/base_normalizer.rb b/app/normalizers/base_normalizer.rb new file mode 100644 index 00000000..f09cb78f --- /dev/null +++ b/app/normalizers/base_normalizer.rb @@ -0,0 +1,7 @@ +class BaseNormalizer < FeedService + # @param [FeedEntity] + # @return [Post] + def normalize(feed_entity:) + raise AbstractMethodError + end +end diff --git a/app/processors/base_processor.rb b/app/processors/base_processor.rb new file mode 100644 index 00000000..f5a50a1d --- /dev/null +++ b/app/processors/base_processor.rb @@ -0,0 +1,7 @@ +class BaseProcessor < FeedService + # @param [FeedContent] + # @return [Array] + def process(feed_content:) + raise AbstractMethodError + end +end diff --git a/app/services/application_logger.rb b/app/services/application_logger.rb new file mode 100644 index 00000000..ece84880 --- /dev/null +++ b/app/services/application_logger.rb @@ -0,0 +1,69 @@ +# ApplicationLogger is a wrapper for Ruby loggers that adds color formatting to +# log messages. Color formatting can be enabled or disabled, and it respects the +# NO_COLOR environment variable by default. +# +# Usage: +# logger = ApplicationLogger.new +# logger.info "This is an info message" +# logger.info { "This line will be evaluated only if the logger" } +# +class ApplicationLogger + CLEAR = "\e[0m" + BOLD = "\e[1m" + + BLACK = "\e[30m" + RED = "\e[31m" + GREEN = "\e[32m" + YELLOW = "\e[33m" + BLUE = "\e[34m" + MAGENTA = "\e[35m" + CYAN = "\e[36m" + WHITE = "\e[37m" + + attr_reader :logger + + def initialize(logger: Rails.logger, colorize: ENV["NO_COLOR"].blank?) + @logger = logger + @colorize = colorize + end + + def debug(message = nil, &) + log_formatted_message(:debug, message, WHITE, &) + end + + def info(message = nil, &) + log_formatted_message(:info, message, BLUE, &) + end + + def warn(message = nil, &) + log_formatted_message(:warn, message, YELLOW, &) + end + + def error(message = nil, &) + log_formatted_message(:error, message, RED, &) + end + + def success(message = nil, &) + log_formatted_message(:info, message, GREEN, &) + end + + def log_formatted_message(level, message, color, &block) + if block + logger.send(level, &-> { format_message(block.call, color) }) + else + logger.send(level, format_message(message, color)) + end + end + + def format_message(message, color) + if colorize? + "#{color}#{message}#{CLEAR}" + else + message + end + end + + def colorize? + @colorize + end +end diff --git a/app/services/class_resolver.rb b/app/services/class_resolver.rb new file mode 100644 index 00000000..7f03948f --- /dev/null +++ b/app/services/class_resolver.rb @@ -0,0 +1,19 @@ +# Resolver a class from the class name and optional suffix. +# +class ClassResolver + attr_reader :class_name, :suffix + + # @param [String] a string representing the target class name with words + # separated by underscores + # @param suffix: [String] optional suffix for the class name + def initialize(class_name, suffix: nil) + @class_name = class_name + @suffix = suffix + end + + # @return [Class] + # @raise [NameError] if the target class is missing + def resolve + [class_name, suffix].join("_").classify.constantize + end +end diff --git a/app/services/error_reporter.rb b/app/services/error_reporter.rb new file mode 100644 index 00000000..98ed067a --- /dev/null +++ b/app/services/error_reporter.rb @@ -0,0 +1,41 @@ +# Hidrate new instance of error report with +class ErrorReporter + # @param error: [Exception] + # @params category: [String] + # @param target: [ActiveRecord::Base] target record (optional) + # @param context: [Hash] JSON-friendly arbitrary hash object (optional) + # @param occured_at: [DateTime] error timestamp (optional) + def report(error:, category:, **options) + ErrorReport.create!( + category: category, + target: options[:target], + context: options[:context] || {}, + occured_at: options[:occured_at] || Time.current, + **error_details(error) + ) + end + + private + + def error_details(error) + { + error_class: error.class.name, + message: error.message, + **backtrace_dtails(error.backtrace || []) + } + end + + def backtrace_dtails(backtrace) + match = backtrace&.first&.match(/^(.+):(\d+):in/) + + if match + { + backtrace: backtrace, + file_name: match[1], + line_number: match[2].to_i + } + else + {backtrace: backtrace} + end + end +end diff --git a/app/services/feed_processor.rb b/app/services/feed_processor.rb new file mode 100644 index 00000000..a97027dc --- /dev/null +++ b/app/services/feed_processor.rb @@ -0,0 +1,21 @@ +# Import content for each feed in the specified scope, and publish new posts. +# +class FeedProcessor < FeedService + include Logging + + attr_reader :feeds + + # @param feeds: [Enumerable] feeds to process + def initialize(feeds:) + @feeds = feeds + end + + def perform + feeds.each do |feed| + Importer.new(feed: feed).import + Publisher.new(posts: feed.posts.pending).publish + rescue StandardError + logger.error { "feed processing interrupted: #{feed.name} (id: #{feed.id})" } + end + end +end diff --git a/app/services/feed_service.rb b/app/services/feed_service.rb new file mode 100644 index 00000000..1e4c6169 --- /dev/null +++ b/app/services/feed_service.rb @@ -0,0 +1,10 @@ +# Basic abstraction for feed services. +# +class FeedService + attr_reader :feed + + # @param [Feed] + def initialize(feed) + @feed = feed + end +end diff --git a/app/services/importer.rb b/app/services/importer.rb new file mode 100644 index 00000000..53d908b6 --- /dev/null +++ b/app/services/importer.rb @@ -0,0 +1,76 @@ +# Load content and generate posts for the specified feed. Raises an error +# in case feed processing is not possible. +# +class Importer + include Logging + + attr_reader :feed + + # @param feed: [Feed] + def initialize(feed:) + @feed = feed + end + + # Import feed content and persist new posts. Will raise an error if the feed + # processing workflow should be interrupted. + # @raise [FeedConfigurationError] if the feed is missing related classes + def import + logger.info("importing #{feed.readable_id}") + ensure_services_resolved + feed_content = load_content(feed) + entities = process_feed_content(feed_content) + build_posts(entities) + end + + private + + def ensure_services_resolved + feed.ensure_supported + rescue StandardError => e + track_feed_error(error: e, category: "configuration") + end + + # @return [FeedContent] + # @raise [StandardError] if processor execution is not possible + def load_content(feed) + feed.loader_instance.load + rescue StandardError => e + track_feed_error(error: e, category: "loading") + raise e + end + + # @return [Array] + # @raise [StandardError] if content processing is not possible + def process_feed_content(feed_content) + feed.processor_instance.process(feed_content) + rescue StandardError => e + track_feed_error(error: e, category: "processing", context: {content: feed_content}) + raise e + end + + def build_posts(entities) + entities.each do |entity| + PostBuilder.new(feed: feed, feed_entity: entity).build + end + end + + def track_feed_error(category:, error: nil, context: {}) + ActiveRecord::Base.transaction do + feed.update!(errored_at: Time.current, errors_count: feed.errors_count.succ) + + ErrorReporter.report( + error: error, + target: feed, + category: category, + context: context.merge(error_context) + ) + end + end + + def error_context + { + feed_supported: feed.supported?, + feed_service_classes: feed.service_classes + } + end +end diff --git a/app/services/logging.rb b/app/services/logging.rb new file mode 100644 index 00000000..b5541033 --- /dev/null +++ b/app/services/logging.rb @@ -0,0 +1,5 @@ +module Logging + def logger + @logger ||= ApplicationLogger.new(Rails.logger) + end +end diff --git a/app/services/post_builder.rb b/app/services/post_builder.rb new file mode 100644 index 00000000..312dc672 --- /dev/null +++ b/app/services/post_builder.rb @@ -0,0 +1,17 @@ +# Hydrate new Post instance from a FeedEntity. Always returns a valid Post, +# though it may not be suitable for publication depending on the content. +# +class PostBuilder + attr_reader :feed, :feed_entity + + def initialize(feed:, feed_entity:) + @feed = feed + @feed_entity = feed_entity + end + + # @return [Post] + def build + # TBD + Post.new(feed: feed) + end +end diff --git a/app/services/publisher.rb b/app/services/publisher.rb new file mode 100644 index 00000000..a4b85636 --- /dev/null +++ b/app/services/publisher.rb @@ -0,0 +1,26 @@ +class Publisher + include Logging + + def initialize(posts:) + @posts = posts + end + + def publish + logger.info("publishing #{TextHelpers.pluralize(pending_posts.count, "posts")}") + + pending_posts.each do |post| + publish_post(post) + end + end + + private + + def pending_posts + @pending_posts ||= posts.filter(&:pending?) + end + + # :reek:UnusedParameters + def publish_post(post) + # TBD + end +end diff --git a/app/services/stopwatch.rb b/app/services/stopwatch.rb new file mode 100644 index 00000000..4a044814 --- /dev/null +++ b/app/services/stopwatch.rb @@ -0,0 +1,16 @@ +class Stopwatch + attr_reader :start + + def self.measure(&block) + instance = new + [block.call, instance.elapsed] + end + + def initialize + @start = Time.current + end + + def elapsed + Time.current - start + end +end diff --git a/app/services/text_helpers.rb b/app/services/text_helpers.rb new file mode 100644 index 00000000..70f3e26a --- /dev/null +++ b/app/services/text_helpers.rb @@ -0,0 +1,3 @@ +class TextHelpers + extend ActionView::Helpers::TextHelper +end diff --git a/config/application.rb b/config/application.rb index 360fe3d7..ff60ccfa 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,17 +1,18 @@ require_relative "boot" require "rails" -# Pick the frameworks you want: -require "active_model/railtie" -require "active_job/railtie" + +# Picking only the required gems +# SEE: https://github.com/rails/rails/blob/main/railties/lib/rails/all.rb require "active_record/railtie" # require "active_storage/engine" require "action_controller/railtie" +require "action_view/railtie" # require "action_mailer/railtie" +require "active_job/railtie" +# require "action_cable/engine" # require "action_mailbox/engine" # require "action_text/engine" -require "action_view/railtie" -# require "action_cable/engine" require "rails/test_unit/railtie" # Require the gems listed in Gemfile, including any gems @@ -38,5 +39,18 @@ class Application < Rails::Application # Don't generate system test files. config.generators.system_tests = nil + + # + # Manually added to the originally generated configuration + # + + config.hosts << "feeder.local" + config.hosts << "frf.im" + config.hosts << "localhost" + + # Use Redis cache store on all environments (see docker-compose.yml) + config.cache_store = :redis_cache_store, {url: ENV.fetch("REDIS_URL")} + + config.ssl_options = {hsts: {subdomains: true}} end end diff --git a/db/migrate/20240817185724_create_error_reports.rb b/db/migrate/20240817185724_create_error_reports.rb new file mode 100644 index 00000000..89d283ed --- /dev/null +++ b/db/migrate/20240817185724_create_error_reports.rb @@ -0,0 +1,17 @@ +class CreateErrorReports < ActiveRecord::Migration[7.1] + def change + create_table :error_reports do |t| + t.references "target", polymorphic: true, index: true + t.string "category", index: true + t.string "error_class", default: "", null: false + t.string "file_name" + t.integer "line_number" + t.string "message", default: "", null: false + t.string "backtrace", default: [], null: false, array: true + t.jsonb "context", default: {}, null: false + t.datetime "occured_at", null: false, index: true + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 34cf1ff1..0d206c14 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,28 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_07_28_141124) do +ActiveRecord::Schema[7.1].define(version: 2024_08_17_185724) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "error_reports", force: :cascade do |t| + t.string "target_type" + t.bigint "target_id" + t.string "category" + t.string "error_class", default: "", null: false + t.string "file_name" + t.integer "line_number" + t.string "message", default: "", null: false + t.string "backtrace", default: [], null: false, array: true + t.jsonb "context", default: {}, null: false + t.datetime "occured_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["category"], name: "index_error_reports_on_category" + t.index ["occured_at"], name: "index_error_reports_on_occured_at" + t.index ["target_type", "target_id"], name: "index_error_reports_on_target" + end + create_table "errors", id: :serial, force: :cascade do |t| t.integer "status", default: 0, null: false t.string "exception", default: "", null: false diff --git a/spec/services/class_resolver_spec.rb b/spec/services/class_resolver_spec.rb new file mode 100644 index 00000000..0f7d87a9 --- /dev/null +++ b/spec/services/class_resolver_spec.rb @@ -0,0 +1,36 @@ +require "rails_helper" + +RSpec.describe ClassResolver do + describe "initialization" do + it "sets expected parameters" do + resolver = described_class.new("test", suffix: "suffix") + expect(resolver.class_name).to eq("test") + expect(resolver.suffix).to eq("suffix") + end + + it "allows suffix to be nil" do + resolver = described_class.new("test") + expect(resolver.class_name).to eq("test") + expect(resolver.suffix).to be_nil + end + end + + describe "#resolve" do + it "resolves class by name" do + stub_const("TestService", Class.new) + resolver = described_class.new("test_service") + expect(resolver.resolve).to eq(TestService) + end + + it "resolves class with suffix" do + stub_const("TestService", Class.new) + resolver = described_class.new("test", suffix: "service") + expect(resolver.resolve).to eq(TestService) + end + + it "raises a NameError for missing class" do + resolver = described_class.new("non_existent") + expect { resolver.resolve }.to raise_error(NameError) + end + end +end