LLM scraper

Gemfile

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gem 'rails', '7.0.8.1'
@@ -14,7 +15,6 @@ gem 'bootstrap-tab-history-rails'
 gem 'country_select'
 gem 'devise'
 gem 'devise_invitable'
-gem 'sitemap_generator'
 gem 'eventbrite_sdk'
 gem 'font-awesome-sass', '~> 4.7.0' # Prefer V4 icon styles
 gem 'friendly_id'
@@ -33,23 +33,23 @@ gem 'jquery-turbolinks'
 gem 'kt-paperclip'
 gem 'linkeddata'
 gem 'money-rails'
-gem 'omniauth-rails_csrf_protection'
 gem 'omniauth_openid_connect'
+gem 'omniauth-rails_csrf_protection'
 gem 'pg'
 gem 'private_address_check'
 gem 'public_activity'
 gem 'pundit'
 gem 'rack-cors', require: 'rack/cors'
-gem 'rails-i18n'
 gem 'rails_admin'
+gem 'rails-i18n'
 gem 'recaptcha', require: 'recaptcha/rails'
 gem 'redcarpet'
 gem 'redis'
 gem 'rest-client'
 gem 'reverse_markdown'
 gem 'rss'
-gem 'sass-rails'
 gem 'sassc-rails'
+gem 'sass-rails'
 gem 'sentry-rails'
 gem 'sentry-ruby'
 gem 'sentry-sidekiq'
@@ -58,6 +58,7 @@ gem 'sidekiq-status'
 gem 'simple_calendar', '~> 2.4'
 gem 'simple_form'
 gem 'simple_token_authentication'
+gem 'sitemap_generator'
 gem 'sitemap-parser'
 gem 'slim'
 gem 'sunspot_rails', github: 'sunspot/sunspot', branch: 'master' # Contains Ruby 3 fixes that are not released
@@ -103,3 +104,5 @@ group :test do
   gem 'vcr'
   gem 'webmock'
 end
+
+gem 'ruby-openai'

Gemfile.lock

@@ -195,15 +195,17 @@ GEM
     erubi (1.12.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
+    event_stream_parser (1.0.0)
     eventbrite_sdk (3.6.0)
       rest-client (~> 2.0)
     execjs (2.8.1)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-follow_redirects (0.3.0)
       faraday (>= 1, < 3)
-    faraday-net_http (3.1.0)
-      net-http
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.0.2)
     ffi (1.15.5)
     font-awesome-sass (4.7.0)
       sass (>= 3.2)
@@ -366,6 +368,7 @@ GEM
     msgpack (1.7.2)
     multi_json (1.15.0)
     multi_xml (0.6.0)
+    multipart-post (2.4.0)
     nested_form (0.3.2)
     net-http (0.4.1)
       uri
@@ -600,6 +603,10 @@ GEM
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
+    ruby-openai (6.3.1)
+      event_stream_parser (>= 0.3.0, < 2.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
     safely_block (0.4.0)
     sass (3.7.4)
@@ -831,6 +838,7 @@ DEPENDENCIES
   reverse_markdown
   rss
   rubocop
+  ruby-openai
   sass-rails
   sassc-rails
   sentry-rails

app/controllers/events_controller.rb

@@ -235,6 +235,7 @@ def event_params
                                   { host_institutions: [] }, :capacity, :contact, :recognition, :learning_objectives,
                                   :prerequisites, :tech_requirements, :cost_basis, :cost_value, :cost_currency,
                                   external_resources_attributes: %i[id url title _destroy], material_ids: [],
+                                  llm_interaction_attributes: %i[id scrape_or_process model prompt input output needs_processing _destroy],
                                   locked_fields: [])
   end
 

app/models/event.rb

@@ -109,6 +109,8 @@ class Event < ApplicationRecord
   enum presence: { onsite: 0, online: 1, hybrid: 2 }
 
   belongs_to :user
+  has_one :llm_interaction, inverse_of: :event, dependent: :destroy
+  accepts_nested_attributes_for :llm_interaction, allow_destroy: true
   has_one :edit_suggestion, as: :suggestible, dependent: :destroy
   has_one :link_monitor, as: :lcheck, dependent: :destroy
   has_many :collection_items, as: :resource

app/models/llm_interaction.rb

@@ -0,0 +1,9 @@
+class LlmInteraction < ApplicationRecord
+  belongs_to :event
+  validates :scrape_or_process, presence: true
+  validates :model, presence: true
+  validates :prompt, presence: true
+  validates :input, presence: true
+  validates :output, presence: true
+  # validates :needs_processing, presence: true
+end

config/application.rb

@@ -21,7 +21,7 @@ class Application < Rails::Application
     config.middleware.insert_before 0, Rack::Cors do
       allow do
         origins '*'
-        resource '*', headers: :any, methods: [:get, :post, :options]
+        resource '*', headers: :any, methods: %i[get post options]
       end
     end
 
@@ -38,7 +38,7 @@ class Application < Rails::Application
       ActiveSupport::HashWithIndifferentAccess, BigDecimal
     ]
 
-    config.exceptions_app = self.routes
+    config.exceptions_app = routes
   end
 
   tess_config = Rails.configuration.tess.with_indifferent_access
@@ -64,9 +64,7 @@ def self.merge_config(default_config, config, current_path = '')
         puts "Setting '#{current_path}#{key}' not configured, using defaults" if Rails.env.development?
         config[key] = value
       end
-      if value.is_a?(Hash) && config[key].is_a?(Hash)
-        merge_config(value, config[key], current_path + "#{key}: ")
-      end
+      merge_config(value, config[key], current_path + "#{key}: ") if value.is_a?(Hash) && config[key].is_a?(Hash)
     end
   end
 
@@ -83,6 +81,7 @@ def redis_url
 
     def ingestion
       return @ingestion if @ingestion
+
       config_file = File.join(Rails.root, 'config', 'ingestion.yml')
       @ingestion = YAML.safe_load(File.read(config_file)).deep_symbolize_keys! if File.exist?(config_file)
     end

config/secrets.example.yml

@@ -38,6 +38,8 @@ external_api_keys: &external_api_keys
   fairsharing:
     username:
     password:
+  gpt_api_key:
+  willma_api_key:
 
 #Internal config
 development:

config/tess.example.yml

@@ -123,6 +123,9 @@ default: &default
     - CC-BY-NC-SA-4.0
     - CC-BY-ND-4.0
     - CC-BY-SA-4.0
+  llm_scraper:
+    model:
+    model_version:
 
 development:
   <<: *default

db/migrate/20240220144246_add_llm_check.rb

@@ -0,0 +1,17 @@
+class AddLlmCheck < ActiveRecord::Migration[7.0]
+  def change
+    create_table :llm_interactions do |t|
+      t.belongs_to :event, foreign_key: true
+      t.datetime :created_at
+      t.datetime :updated_at
+      t.string :scrape_or_process
+      t.string :model
+      t.string :prompt
+      t.string :input
+      t.string :output
+      t.boolean :needs_processing, default: false
+    end
+    add_reference :events, :llm_interaction, foreign_key: true
+    add_column :events, :open_science, :string, array: true, default: []
+  end
+end