Fix last scraped date for FA

See #113 (comment)

app/jobs/scrape_artist_url_job.rb

@@ -16,10 +16,10 @@ def perform(artist_url) # rubocop:disable Metrics/CyclomaticComplexity
 
       artist_url.update(scraper_status: artist_url.scraper_status.merge(scraper.class.state => scraper.state_value))
 
-      stop_marker = artist_url.last_scraped_at
-      break if stop_marker.present? && submissions.any? { |submission| submission.created_at.before? stop_marker }
+      break if submissions.any? { |submission| artist_url.scraper_stop_marker&.after?(submission.created_at) }
     end
-    artist_url.last_scraped_at = scraper.cutoff_timestamp
+    artist_url.last_scraped_at = artist_url.scraper_status["started_at"]
+    artist_url.scraper_stop_marker = scraper.new_stop_marker
     artist_url.scraper_status = {}
     artist_url.save
   end

app/logical/scraper/base.rb

@@ -49,7 +49,7 @@ def jumpstart(value)
     # Which date should already be considered scraped? Normally this is good to set to
     # when the scrape started but some sites may exhibit a delay with indexing, resulting
     # in images being missed. See https://github.com/Earlopain/reverser/issues/113
-    def cutoff_timestamp
+    def new_stop_marker
       @artist_url.scraper_status["started_at"]
     end
 

app/logical/scraper/furaffinity.rb

@@ -56,7 +56,7 @@ def fetch_api_identifier
       url_identifier
     end
 
-    def cutoff_timestamp
+    def new_stop_marker
       query = SubmissionFile.joins(artist_submission: :artist_url).where(artist_submission: { artist_url: @artist_url })
       query.order(created_at_on_site: :desc).pick(:created_at_on_site)
     end

db/migrate/20240128115103_add_artist_url_scraper_stop_marker.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class AddArtistUrlScraperStopMarker < ActiveRecord::Migration[7.1]
+  def change
+    add_column :artist_urls, :scraper_stop_marker, :datetime
+    reversible do |dir|
+      dir.up do
+        execute("update artist_urls set scraper_stop_marker = last_scraped_at")
+      end
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_01_12_091112) do
+ActiveRecord::Schema[7.1].define(version: 2024_01_28_115103) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -67,6 +67,7 @@
     t.text "api_identifier"
     t.integer "site_type", null: false
     t.jsonb "scraper_status", default: {}, null: false
+    t.datetime "scraper_stop_marker"
     t.index "site_type, lower(url_identifier)", name: "index_artist_urls_on_site_and_url_identifier", unique: true
     t.index ["artist_id"], name: "index_artist_urls_on_artist_id"
     t.index ["site_type", "api_identifier"], name: "index_site_type_on_api_identifier", unique: true