diff --git a/app/controllers/settings/api_services_controller.rb b/app/controllers/settings/api_services_controller.rb index ce6a3831..5ac8251a 100644 --- a/app/controllers/settings/api_services_controller.rb +++ b/app/controllers/settings/api_services_controller.rb @@ -30,6 +30,16 @@ def update end end + def test + @api_service = Current.user.api_services.find_by(id: params[:api_service_id]) + @answer = @api_service.test_api_service(params[:url], params[:token]) + + respond_to do |format| + format.html { redirect_to settings_api_services_path, notice: "Tested: #{@answer}", status: :see_other } + format.turbo_stream + end + end + def destroy @api_service.deleted! redirect_to settings_api_services_path, notice: "Deleted", status: :see_other diff --git a/app/controllers/settings/language_models_controller.rb b/app/controllers/settings/language_models_controller.rb index 5c99396f..70f6d3bf 100644 --- a/app/controllers/settings/language_models_controller.rb +++ b/app/controllers/settings/language_models_controller.rb @@ -34,6 +34,16 @@ def update end end + def test + @language_model = Current.user.language_models.find_by(id: params[:language_model_id]) + @answer = @language_model.test(params[:model]) + + respond_to do |format| + format.html { redirect_to settings_language_models_path, notice: "Tested: #{@answer}", status: :see_other } + format.turbo_stream + end + end + def destroy @language_model.deleted! redirect_to settings_language_models_path, notice: "Deleted", status: :see_other diff --git a/app/javascript/stimulus/testing_controller.js b/app/javascript/stimulus/testing_controller.js new file mode 100644 index 00000000..c9fc27de --- /dev/null +++ b/app/javascript/stimulus/testing_controller.js @@ -0,0 +1,25 @@ +import { Controller } from "@hotwired/stimulus" + +export default class extends Controller { + static targets = ["model", "test", "url","token"] + + update_link_language_model(event) { + const link = event.currentTarget; + const href = link.href.split('?')[0]; + link.href = href + "?model=" + this.modelTarget.value + } + + update_link_api_service(event) { + const link = event.currentTarget; + const href = link.href.split('?')[0]; + link.href = href + "?url=" + this.urlTarget.value + "&token=" + this.tokenTarget.value + console.log("Link: ", link) + } + + disable_test_link() { + const link = this.testTarget; + link.addEventListener('click', function(event) { + event.preventDefault(); + }); + } +} diff --git a/app/models/api_service.rb b/app/models/api_service.rb index 9ad738fd..17798d05 100644 --- a/app/models/api_service.rb +++ b/app/models/api_service.rb @@ -39,6 +39,10 @@ def effective_token token.presence || default_llm_key end + def test_api_service(url = nil, token = nil) + ai_backend.test_api_service(self, url, token) + end + private def default_llm_key diff --git a/app/models/language_model.rb b/app/models/language_model.rb index a0e48e8d..ffd44bb0 100644 --- a/app/models/language_model.rb +++ b/app/models/language_model.rb @@ -31,6 +31,10 @@ def supports_tools? api_service.name != "Groq" # TODO: Remove this short circuit once I can debug tool use with Groq end + def test(api_name = nil) + ai_backend.test_language_model(self, api_name) + end + private def populate_position diff --git a/app/services/ai_backend.rb b/app/services/ai_backend.rb index d4204f08..5f14342b 100644 --- a/app/services/ai_backend.rb +++ b/app/services/ai_backend.rb @@ -48,6 +48,27 @@ def stream_next_conversation_message(&chunk_handler) end end + def self.test_language_model(language_model, api_name = nil) + api_name ||= language_model.api_name + url = language_model.api_service.url + token = language_model.api_service.effective_token + return "Error: API key (token) is blank" if language_model.api_service.requires_token? && token.blank? + + test_execute(url, token, api_name) + end + + def self.test_api_service(api_service, url = nil, token = nil) + url ||= api_service.url + token ||= api_service.effective_token + language_model = LanguageModel.where(best: true, api_service: api_service).first + api_name = language_model.api_name unless language_model.nil? + + return "Error: API key (token) is blank" if api_service.requires_token? && token.blank? + return "Error: API name is blank. Define a best Language Model for this API service." if api_name.blank? + + test_execute(url, token, api_name) + end + private def client_method_name diff --git a/app/services/ai_backend/anthropic.rb b/app/services/ai_backend/anthropic.rb index f8a4d5d9..638df9ea 100644 --- a/app/services/ai_backend/anthropic.rb +++ b/app/services/ai_backend/anthropic.rb @@ -14,6 +14,26 @@ def self.client end end + def self.test_execute(url, token, api_name) + Rails.logger.info "Connecting to Anthropic API server at #{url} with access token of length #{token.to_s.length}" + client = ::Anthropic::Client.new( + uri_base: url, + access_token: token + ) + + Rails.logger.info "Testing using model #{api_name}" + client.messages( + model: api_name, + messages: [ + { "role": "user", "content": "Hello!" } + ], + system: "You are a helpful assistant.", + parameters: { max_tokens: 1000 } + ).dig("content", 0, "text") + rescue => e + "Error: #{e.message}" + end + def initialize(user, assistant, conversation = nil, message = nil) super(user, assistant, conversation, message) begin diff --git a/app/services/ai_backend/gemini.rb b/app/services/ai_backend/gemini.rb index f89447cf..d125fc68 100644 --- a/app/services/ai_backend/gemini.rb +++ b/app/services/ai_backend/gemini.rb @@ -15,6 +15,27 @@ def self.client end end + def self.test_execute(url, token, api_name) + Rails.logger.info "Connecting to Gemini API server at #{url} with access token of length #{token.to_s.length}" + client = ::Gemini.new( + credentials: { + service: "generative-language-api", + api_key: token, + version: "v1beta" + }, + options: { + model: api_name, + server_sent_events: true + } + ) + + client.generate_content({ + contents: { role: "user", parts: { text: "Hello!" }} + }).dig("candidates", 0, "content", "parts", 0, "text") + rescue ::Faraday::Error => e + "Error: #{e.message}" + end + def initialize(user, assistant, conversation = nil, message = nil) super(user, assistant, conversation, message) begin diff --git a/app/services/ai_backend/open_ai.rb b/app/services/ai_backend/open_ai.rb index aa8bd600..f40c730f 100644 --- a/app/services/ai_backend/open_ai.rb +++ b/app/services/ai_backend/open_ai.rb @@ -14,6 +14,29 @@ def self.client end end + def self.test_execute(url, token, api_name) + if Rails.env.test? + client = ::TestClient::OpenAI.new( + access_token: token, + uri_base: url + ) + response = client.send(:chat, ** {parameters: {model: api_name, messages: [{ role: "user", content: "Hello!" }]}}) + else + Rails.logger.info "Connecting to OpenAI API server at #{url} with access token of length #{token.to_s.length}" + client = ::OpenAI::Client.new( + access_token: token, + uri_base: url + ) + + Rails.logger.info "Testing using model #{api_name}" + response = client.chat(parameters: {model: api_name, messages: [{ role: "user", content: "Hello!" }]}) + end + + response.dig("choices", 0, "message", "content") + rescue ::Faraday::Error => e + "Error: #{e.message}" + end + def initialize(user, assistant, conversation = nil, message = nil) super(user, assistant, conversation, message) begin diff --git a/app/views/settings/api_services/_form.html.erb b/app/views/settings/api_services/_form.html.erb index a6c9d073..30e5e8b9 100644 --- a/app/views/settings/api_services/_form.html.erb +++ b/app/views/settings/api_services/_form.html.erb @@ -1,4 +1,4 @@ -<%= form_with(model: [:settings, api_service], class: "contents") do |form| %> +<%= form_with(model: [:settings, api_service], class: "contents", data: {controller: "testing"}) do |form| %> <% if api_service.errors.any? %>

<%= pluralize(api_service.errors.count, "error") %> prohibited this API service from being saved:

@@ -21,6 +21,7 @@ <%= form.select :driver, APIService.drivers.keys, {}, + data: { action: "change->testing#disable_test_link" }, class: %| block border border-gray-200 outline-none @@ -32,9 +33,22 @@ %>
-
+
<%= form.label :url %> - <%= form.text_field :url, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %> + + <% if api_service.persisted? %> + <% link_url = settings_api_services_path + "/" + api_service.id.to_s + "/test" %> + <%= link_to "Test", link_url, + data: { testing_target: "test", turbo_stream: true, action: "click->testing#update_link_api_service" }, + class: "underline text-blue-600" + %> + <% end %> + + + <%= form.text_field :url, + data: { testing_target: "url" }, + class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" + %>
@@ -167,7 +181,16 @@ %>. - <%= form.text_field :token, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black", autocomplete: "off" %> + <%= form.text_field :token, + class: %| + block shadow + rounded-md border border-gray-200 outline-none + px-3 py-2 mt-2 w-full dark:text-black + |, + autocomplete: "off", + id: "input_token", + data: { testing_target: "token" } + %>
<%= form.submit "Save", diff --git a/app/views/settings/api_services/test.turbo_stream.erb b/app/views/settings/api_services/test.turbo_stream.erb new file mode 100644 index 00000000..14155ff6 --- /dev/null +++ b/app/views/settings/api_services/test.turbo_stream.erb @@ -0,0 +1,11 @@ + + + diff --git a/app/views/settings/language_models/_form.html.erb b/app/views/settings/language_models/_form.html.erb index d8ee22f2..934961a7 100644 --- a/app/views/settings/language_models/_form.html.erb +++ b/app/views/settings/language_models/_form.html.erb @@ -1,4 +1,4 @@ -<%= form_with(model: [:settings, language_model], class: "contents") do |form| %> +<%= form_with(model: [:settings, language_model], class: "contents", data: { controller: "testing" }) do |form| %> <% if language_model.errors.any? %>

<%= pluralize(language_model.errors.count, "error") %> prohibited this language model from being saved:

@@ -16,9 +16,22 @@ <%= form.text_field :name, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %>
-
+
<%= form.label :api_name %> - <%= form.text_field :api_name, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %> + + <% if language_model.persisted? %> + <% link_url = settings_language_models_path + "/" + language_model.id.to_s + "/test" %> + <%= link_to "Test", link_url, + data: { testing_target: "test", turbo_stream: true, action: "click->testing#update_link_language_model" }, + class: "underline text-blue-600" + %> + + <% end %> + + <%= form.text_field :api_name, + data: { testing_target: "model" }, + class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" + %> As specified in the API docs
@@ -28,6 +41,7 @@ <%= form.select :api_service_id, Current.user.api_services.ordered.pluck(:name, :id).reverse, {}, + data: { action: "change->testing#disable_test_link" }, class: %| block border border-gray-200 outline-none diff --git a/app/views/settings/language_models/test.turbo_stream.erb b/app/views/settings/language_models/test.turbo_stream.erb new file mode 100644 index 00000000..14155ff6 --- /dev/null +++ b/app/views/settings/language_models/test.turbo_stream.erb @@ -0,0 +1,11 @@ + + + diff --git a/config/routes.rb b/config/routes.rb index 73386abb..5f5a2fb3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -16,8 +16,12 @@ namespace :settings do resources :assistants, except: [:index, :show] resource :person, only: [:edit, :update] - resources :language_models - resources :api_services, except: [:show] + resources :language_models do + get :test, to: "language_models#test" + end + resources :api_services, except: [:show] do + get :test, to: "api_services#test" + end resources :memories, only: [:index, :destroy] do delete :destroy, to: "memories#destroy_all", on: :collection end diff --git a/test/controllers/settings/api_services_controller_test.rb b/test/controllers/settings/api_services_controller_test.rb index 2e42e25f..7bb44c58 100644 --- a/test/controllers/settings/api_services_controller_test.rb +++ b/test/controllers/settings/api_services_controller_test.rb @@ -108,4 +108,12 @@ class Settings::APIServicesControllerTest < ActionDispatch::IntegrationTest get edit_settings_api_service_url(api_services(:keith_other_service)) assert_select "button#instructions.hidden" end + + test "test should return success and a message" do + TestClient::OpenAI.stub :text, "Success." do + get settings_api_service_test_url(format: :turbo_stream, api_service_id: api_services(:keith_openai_service).id, url: "https://api.openai.com/v1", token: "abc-secret123") + assert_response :success + assert_contains_text "div#test_result", "Success." + end + end end diff --git a/test/controllers/settings/language_models_controller_test.rb b/test/controllers/settings/language_models_controller_test.rb index 1accfe13..f35b42a5 100644 --- a/test/controllers/settings/language_models_controller_test.rb +++ b/test/controllers/settings/language_models_controller_test.rb @@ -199,4 +199,12 @@ class Settings::LanguageModelsControllerTest < ActionDispatch::IntegrationTest assert_select 'input[name="language_model[supports_system_message]"][checked="checked"]', false end end + + test "test should return success and a message" do + TestClient::OpenAI.stub :text, "Success." do + get settings_language_model_test_url(format: :turbo_stream, language_model_id: language_models(:guanaco).id, model: "gpt-4o") + assert_response :success + assert_contains_text "div#test_result", "Success." + end + end end