From 080245fa608565c4525e895ea902aa0e9e7a1d71 Mon Sep 17 00:00:00 2001 From: Justin Vallelonga Date: Thu, 19 Sep 2024 00:35:26 -0600 Subject: [PATCH] message cost calculations and display (#507) --- app/helpers/application_helper.rb | 4 ++ app/models/conversation.rb | 4 ++ app/models/message.rb | 10 ++++ .../conversations/_conversation.html.erb | 26 ++++++-- .../settings/api_services/_form.html.erb | 2 +- .../initializers/filter_parameter_logging.rb | 2 +- test/fixtures/language_models.yml | 60 ++++++++++++++++++- test/models/conversation/billable_test.rb | 38 ++++++++---- test/models/message_test.rb | 16 +++++ 9 files changed, 143 insertions(+), 19 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 0d2d2aed5..1e2f0d02e 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -84,4 +84,8 @@ def viewport_tag(content) def n_a_if_blank(value, n_a = "Not Available") value.blank? ? n_a : value.to_s end + + def to_dollars(cents, precision: 2) + number_to_currency(cents / 100.0, precision: precision) + end end diff --git a/app/models/conversation.rb b/app/models/conversation.rb index 7dabd9712..61106f911 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -57,6 +57,10 @@ def self.grouped_by_increasing_time_interval_for_user(user) .delete_if { |_, v| v.empty? } end + def total_cost + input_token_total_cost + output_token_total_cost + end + private def set_title_async diff --git a/app/models/message.rb b/app/models/message.rb index c694200b2..6fc2faadf 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -25,6 +25,8 @@ class Message < ApplicationRecord after_create :start_assistant_reply, if: :user? after_create :set_last_assistant_message, if: :assistant? after_save :update_assistant_on_conversation, if: -> { assistant.present? && conversation.present? } + before_save :update_input_token_cost, if: :input_token_count_changed? + before_save :update_output_token_cost, if: :output_token_count_changed? scope :ordered, -> { latest_version_for_conversation } @@ -77,4 +79,12 @@ def update_assistant_on_conversation return if conversation.assistant == assistant conversation.update!(assistant: assistant) end + + def update_input_token_cost + self.input_token_cost = assistant.language_model.input_token_cost_cents * input_token_count + end + + def update_output_token_cost + self.output_token_cost = assistant.language_model.output_token_cost_cents * output_token_count + end end diff --git a/app/views/conversations/_conversation.html.erb b/app/views/conversations/_conversation.html.erb index 0818de16a..5764862f8 100644 --- a/app/views/conversations/_conversation.html.erb +++ b/app/views/conversations/_conversation.html.erb @@ -80,16 +80,30 @@ diff --git a/app/views/settings/api_services/_form.html.erb b/app/views/settings/api_services/_form.html.erb index 3ead171ca..974871197 100644 --- a/app/views/settings/api_services/_form.html.erb +++ b/app/views/settings/api_services/_form.html.erb @@ -135,7 +135,7 @@ %>).
  • Click "Create API Key", name it something like "<%= Setting.product_name %>" and paste it below.
    • - <%= form.text_field :token, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black" %> + <%= form.text_field :token, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 mt-2 w-full dark:text-black", autocomplete: "off" %> <%= form.submit "Save", diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 7ca15d15c..40d6af22e 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -26,5 +26,5 @@ def filter_regexp(filter_key, exemptions = []) :certificate, :otp, :ssn, - filter_regexp(:token, [:token_count, :token_cost]), + filter_regexp(:token, [:token_count, :token_cost, :token_total_count, :token_total_cost]), ] diff --git a/test/fixtures/language_models.yml b/test/fixtures/language_models.yml index e771f236d..ac21c32f1 100644 --- a/test/fixtures/language_models.yml +++ b/test/fixtures/language_models.yml @@ -16,6 +16,8 @@ pacos: api_name: pacos-imagine supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 # Rob's Language Models gpt_4_turbo: @@ -26,6 +28,8 @@ gpt_4_turbo: api_service: rob_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_turbo_2024_04_09: position: 2 @@ -35,6 +39,8 @@ gpt_4_turbo_2024_04_09: api_service: rob_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_turbo_preview: position: 3 @@ -44,6 +50,8 @@ gpt_4_turbo_preview: api_service: rob_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_0125_preview: position: 4 @@ -53,6 +61,8 @@ gpt_4_0125_preview: api_service: rob_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_3_5_turbo_0125: position: 5 @@ -62,6 +72,8 @@ gpt_3_5_turbo_0125: api_service: rob_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_3_5_turbo_1106: position: 6 @@ -71,6 +83,8 @@ gpt_3_5_turbo_1106: api_service: rob_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_3_5_turbo_instruct: position: 7 @@ -80,6 +94,8 @@ gpt_3_5_turbo_instruct: api_service: rob_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_2_0: position: 8 @@ -89,6 +105,8 @@ claude_2_0: api_service: rob_anthropic_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_instant_1_2: position: 9 @@ -98,6 +116,8 @@ claude_instant_1_2: api_service: rob_anthropic_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 rob_gpt: position: 10 @@ -107,6 +127,8 @@ rob_gpt: api_service: rob_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 # Keith's Language Models gpt_best: @@ -117,6 +139,8 @@ gpt_best: api_service: keith_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_best: position: 2 @@ -126,6 +150,8 @@ claude_best: api_name: claude-best supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4o: position: 3 @@ -135,7 +161,9 @@ gpt_4o: api_service: keith_openai_service supports_images: true supports_tools: true - + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 + gpt_4o_2024_05_13: position: 4 user: keith @@ -144,6 +172,8 @@ gpt_4o_2024_05_13: api_service: keith_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_1106_preview: position: 9 @@ -153,6 +183,8 @@ gpt_4_1106_preview: api_service: keith_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_vision_preview: position: 10 @@ -162,6 +194,8 @@ gpt_4_vision_preview: api_service: keith_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_1106_vision_preview: position: 11 @@ -171,6 +205,8 @@ gpt_4_1106_vision_preview: api_service: keith_openai_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4: position: 12 @@ -180,6 +216,8 @@ gpt_4: api_service: keith_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_4_0613: position: 13 @@ -189,6 +227,8 @@ gpt_4_0613: api_service: keith_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_3_5_turbo: position: 14 @@ -198,6 +238,8 @@ gpt_3_5_turbo: api_service: keith_openai_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 gpt_3_5_turbo_16k_0613: position: 15 @@ -207,6 +249,8 @@ gpt_3_5_turbo_16k_0613: api_service: keith_openai_service supports_images: false supports_tools: false + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_3_5_sonnet_20240620: position: 16 @@ -216,6 +260,8 @@ claude_3_5_sonnet_20240620: api_service: keith_anthropic_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_3_opus_20240229: position: 17 @@ -225,6 +271,8 @@ claude_3_opus_20240229: api_service: keith_anthropic_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_3_sonnet_20240229: position: 18 @@ -234,6 +282,8 @@ claude_3_sonnet_20240229: api_service: keith_anthropic_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_3_haiku_20240307: position: 19 @@ -243,6 +293,8 @@ claude_3_haiku_20240307: api_service: keith_anthropic_service supports_images: true supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 claude_2_1: position: 20 @@ -252,6 +304,8 @@ claude_2_1: api_service: keith_anthropic_service supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 camel: position: 21 @@ -261,6 +315,8 @@ camel: api_name: camel supports_images: false supports_tools: true + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 guanaco: position: 22 @@ -270,3 +326,5 @@ guanaco: api_name: guanaco:large supports_images: true supports_tools: false + input_token_cost_cents: 0.0001 + output_token_cost_cents: 0.0001 diff --git a/test/models/conversation/billable_test.rb b/test/models/conversation/billable_test.rb index 35ad60035..d9137ec21 100644 --- a/test/models/conversation/billable_test.rb +++ b/test/models/conversation/billable_test.rb @@ -72,8 +72,8 @@ class Conversation::BillableTest < ActiveSupport::TestCase content_text: "What is your name?", input_token_count: 1, output_token_count: 2, - input_token_cost: 0.1, - output_token_cost: 0.2 + # input_token_cost: 0.1, + # output_token_cost: 0.2 ) adjustments = message.attributes.symbolize_keys.slice(:input_token_cost, :output_token_cost, :input_token_count, :output_token_count) @@ -91,13 +91,31 @@ class Conversation::BillableTest < ActiveSupport::TestCase end test "updating values updates the rollups" do - assert_difference "conversations(:debugging).reload.input_token_total_count", 3 do - assert_difference "conversations(:debugging).reload.input_token_total_cost", 0.03 do - messages(:filter_map).update!( - input_token_count: 8, - input_token_cost: 0.08, - ) - end - end + conversation_og = conversations(:debugging).attributes.symbolize_keys.slice( + :input_token_total_cost, + :output_token_total_cost, + :input_token_total_count, + :output_token_total_count + ) + message_og = messages(:filter_map).attributes.symbolize_keys.slice(:input_token_cost, :input_token_count, :output_token_cost, :output_token_count) + + messages(:filter_map).update!( + input_token_count: 8, + output_token_count: 90, + ) + + message_adjusted = messages(:filter_map).reload.attributes.symbolize_keys.slice(:input_token_cost, :input_token_count, :output_token_cost, :output_token_count) + + conversation_adjusted = conversations(:debugging).reload.attributes.symbolize_keys.slice( + :input_token_total_cost, + :output_token_total_cost, + :input_token_total_count, + :output_token_total_count + ) + + assert_equal conversation_adjusted[:input_token_total_cost], conversation_og[:input_token_total_cost] + (message_adjusted[:input_token_cost] - message_og[:input_token_cost]) + assert_equal conversation_adjusted[:output_token_total_cost], conversation_og[:output_token_total_cost] + (message_adjusted[:output_token_cost] - message_og[:output_token_cost]) + assert_equal conversation_adjusted[:input_token_total_count], conversation_og[:input_token_total_count] + (message_adjusted[:input_token_count] - message_og[:input_token_count]) + assert_equal conversation_adjusted[:output_token_total_count], conversation_og[:output_token_total_count] + (message_adjusted[:output_token_count] - message_og[:output_token_count]) end end diff --git a/test/models/message_test.rb b/test/models/message_test.rb index 6c21a7b05..fd27ed03f 100644 --- a/test/models/message_test.rb +++ b/test/models/message_test.rb @@ -241,4 +241,20 @@ class MessageTest < ActiveSupport::TestCase assert_nil memory.reload.message end + + test "modifying input_token_count updates input_token_cost" do + message = messages(:hear_me) + message.update!(input_token_count: 5, output_token_cost: 1) + assert_equal 5 * message.assistant.language_model.input_token_cost_cents, message.input_token_cost + # make sure output_token_cost is not changed + assert_equal 1, message.output_token_cost + end + + test "modifying output_token_count updates output_token_cost" do + message = messages(:hear_me) + message.update!(output_token_count: 5, input_token_cost: 1) + assert_equal 5 * message.assistant.language_model.output_token_cost_cents, message.output_token_cost + # make sure input_token_cost is not changed + assert_equal 1, message.input_token_cost + end end