Skip to content

Commit

Permalink
Add system tests for Claude
Browse files Browse the repository at this point in the history
This creates system tests for a conversation with Claude using
structured answers.

It also adds a new system spec helper to allow a scenario where
Rails configuration is set to use Claude as an answer strategy.
  • Loading branch information
chaecramb committed Feb 5, 2025
1 parent 98145e8 commit f90e696
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 0 deletions.
6 changes: 6 additions & 0 deletions spec/support/system_spec_helpers.rb
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
module SystemSpecHelpers
def given_i_am_using_the_claude_structured_answer_strategy
allow(Rails.configuration)
.to receive(:answer_strategy)
.and_return("claude_structured_answer")
end

def given_i_have_confirmed_i_understand_chat_risks
visit onboarding_limitations_path

Expand Down
80 changes: 80 additions & 0 deletions spec/system/conversation_with_claude_structured_answer_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
RSpec.describe "Conversation with Claude with a structured answer" do
scenario do
given_i_am_using_the_claude_structured_answer_strategy
and_i_am_a_signed_in_early_access_user
and_i_have_confirmed_i_understand_chat_risks

when_i_visit_the_conversation_page
and_i_enter_a_question
then_i_see_the_answer_is_pending

when_the_first_answer_is_generated
and_i_click_on_the_check_answer_button
then_i_see_my_question_on_the_page
and_i_can_see_the_first_answer

when_i_enter_a_second_question
then_i_see_the_answer_is_pending

when_the_second_answer_is_generated
and_i_click_on_the_check_answer_button
then_i_see_my_second_question_on_the_page
and_i_can_see_the_second_answer
end

def when_i_visit_the_conversation_page
visit show_conversation_path
end

def and_i_enter_a_question
@first_question = "How much tax should I be paying?"
fill_in "Message", with: @first_question
click_on "Send"
end

def then_i_see_the_answer_is_pending
expect(page).to have_content("GOV.UK Chat is generating an answer")
end

def when_the_first_answer_is_generated
@first_answer = "Lots of tax."
stub_bedrock_converse(
bedrock_claude_structured_answer_response(@first_question, @first_answer),
)
execute_queued_sidekiq_jobs
end

def and_i_click_on_the_check_answer_button
click_on "Check if an answer has been generated"
end

def then_i_see_my_question_on_the_page
expect(page).to have_content(@first_question)
end

def and_i_can_see_the_first_answer
expect(page).to have_content(@first_answer)
end

def when_i_enter_a_second_question
@second_question = "Are you sure?"
fill_in "Message", with: @second_question
click_on "Send"
end

def when_the_second_answer_is_generated
@second_answer = "Even more tax."
stub_bedrock_converse(
bedrock_claude_structured_answer_response(@second_question, @second_answer),
)
execute_queued_sidekiq_jobs
end

def then_i_see_my_second_question_on_the_page
expect(page).to have_content(@second_question)
end

def and_i_can_see_the_second_answer
expect(page).to have_content(@second_answer)
end
end

0 comments on commit f90e696

Please sign in to comment.