From 923a29a4c878a2011f5c0d5d4dbcbf01c6a31c7b Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Mon, 18 Dec 2023 19:16:12 -0500 Subject: [PATCH 01/11] debugging --- lib/inferno/apps/web/controllers/test_runs/create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/inferno/apps/web/controllers/test_runs/create.rb b/lib/inferno/apps/web/controllers/test_runs/create.rb index 113fec3d0..2d3bc8dac 100644 --- a/lib/inferno/apps/web/controllers/test_runs/create.rb +++ b/lib/inferno/apps/web/controllers/test_runs/create.rb @@ -56,8 +56,8 @@ def handle(req, res) test_session.suite_options ) + binding.pry test_run = repo.create(create_params(req.params).merge(status: 'queued')) - res.body = serialize(test_run, suite_options: test_session.suite_options) persist_inputs(req.params, test_run) From 28e750bd647aca2d10937926e703aa396cfb3ce5 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Wed, 20 Dec 2023 13:00:16 -0500 Subject: [PATCH 02/11] debugging --- lib/inferno/apps/web/controllers/test_runs/create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/inferno/apps/web/controllers/test_runs/create.rb b/lib/inferno/apps/web/controllers/test_runs/create.rb index 2d3bc8dac..113fec3d0 100644 --- a/lib/inferno/apps/web/controllers/test_runs/create.rb +++ b/lib/inferno/apps/web/controllers/test_runs/create.rb @@ -56,8 +56,8 @@ def handle(req, res) test_session.suite_options ) - binding.pry test_run = repo.create(create_params(req.params).merge(status: 'queued')) + res.body = serialize(test_run, suite_options: test_session.suite_options) persist_inputs(req.params, test_run) From 8d2a7bcd6ec568f5746cac3ec5d168a1a4da87b4 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 21 Dec 2023 18:09:56 -0500 Subject: [PATCH 03/11] debugging server side --- Gemfile | 2 +- dev_suites/dev_infrastructure_test/empty_group.rb | 11 +++++++++++ .../dev_infrastructure_test/external_inner_group.rb | 1 + .../dev_infrastructure_test/external_outer_group.rb | 1 + .../dev_infrastructure_test/infrastructure_test.rb | 2 ++ lib/inferno/apps/web/controllers/test_runs/show.rb | 3 +++ lib/inferno/test_runner.rb | 2 ++ 7 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 dev_suites/dev_infrastructure_test/empty_group.rb diff --git a/Gemfile b/Gemfile index 4593ed536..824a968f7 100644 --- a/Gemfile +++ b/Gemfile @@ -36,4 +36,4 @@ group :test do gem 'simplecov-cobertura' gem 'webmock' gem 'factory_bot' -end \ No newline at end of file +end diff --git a/dev_suites/dev_infrastructure_test/empty_group.rb b/dev_suites/dev_infrastructure_test/empty_group.rb new file mode 100644 index 000000000..6ea47f1be --- /dev/null +++ b/dev_suites/dev_infrastructure_test/empty_group.rb @@ -0,0 +1,11 @@ +module InfrastructureTest + class EmptyGroup < Inferno::TestGroup + id 'empty_group' + title 'Empty group title' + + fhir_client :empty_group do + url 'EMPTY_GROUP' + end + + end +end diff --git a/dev_suites/dev_infrastructure_test/external_inner_group.rb b/dev_suites/dev_infrastructure_test/external_inner_group.rb index 58f6a32c1..cf789fc83 100644 --- a/dev_suites/dev_infrastructure_test/external_inner_group.rb +++ b/dev_suites/dev_infrastructure_test/external_inner_group.rb @@ -3,6 +3,7 @@ module InfrastructureTest class ExternalInnerGroup < Inferno::TestGroup id 'external_inner_group' + title 'External Inner Group' input :external_inner_group_input, type: 'text' output :external_inner_group_output diff --git a/dev_suites/dev_infrastructure_test/external_outer_group.rb b/dev_suites/dev_infrastructure_test/external_outer_group.rb index ad2681e9a..467280f97 100644 --- a/dev_suites/dev_infrastructure_test/external_outer_group.rb +++ b/dev_suites/dev_infrastructure_test/external_outer_group.rb @@ -3,6 +3,7 @@ module InfrastructureTest class ExternalOuterGroup < Inferno::TestGroup id 'external_outer_group' + title 'External Outer Group' input :external_outer_group_input output :external_outer_group_output diff --git a/dev_suites/dev_infrastructure_test/infrastructure_test.rb b/dev_suites/dev_infrastructure_test/infrastructure_test.rb index ee8d62f07..6a71bc00f 100644 --- a/dev_suites/dev_infrastructure_test/infrastructure_test.rb +++ b/dev_suites/dev_infrastructure_test/infrastructure_test.rb @@ -2,6 +2,7 @@ require_relative 'failing_optional_group' require_relative 'passing_optional_group' require_relative 'mixed_optional_group' +require_relative 'empty_group' module InfrastructureTest class Suite < Inferno::TestSuite @@ -112,5 +113,6 @@ def inline_test1_helper group from: 'failing_optional_group' group from: 'mixed_optional_group', exclude_optional: true group from: 'external_outer_group' + group from: 'empty_group' end end diff --git a/lib/inferno/apps/web/controllers/test_runs/show.rb b/lib/inferno/apps/web/controllers/test_runs/show.rb index df7e33c32..29fb9cb8b 100644 --- a/lib/inferno/apps/web/controllers/test_runs/show.rb +++ b/lib/inferno/apps/web/controllers/test_runs/show.rb @@ -20,6 +20,9 @@ def handle(req, res) repo.results_for_test_run(test_run.id) end end + + ## XXX + # binding.pry test_session = test_sessions_repo.find(test_run.test_session_id) res.body = serialize(test_run, suite_options: test_session.suite_options) diff --git a/lib/inferno/test_runner.rb b/lib/inferno/test_runner.rb index 6b7e9c922..41a4c12ad 100644 --- a/lib/inferno/test_runner.rb +++ b/lib/inferno/test_runner.rb @@ -132,6 +132,8 @@ def run_group(group, scratch) results.flatten! + ## XXX when group has no children this raises Sequel::ValidationError + # binding.pry group_result = persist_result(group.reference_hash.merge(result: roll_up_result(results), input_json: JSON.generate(group_inputs_with_values))) From a42bb3ef4afa7f352bf4783a2d4510dd0265d6cf Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 21 Dec 2023 18:18:52 -0500 Subject: [PATCH 04/11] add titles to infrastructure test suite external groups --- dev_suites/dev_infrastructure_test/infrastructure_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_suites/dev_infrastructure_test/infrastructure_test.rb b/dev_suites/dev_infrastructure_test/infrastructure_test.rb index 6a71bc00f..379ecb697 100644 --- a/dev_suites/dev_infrastructure_test/infrastructure_test.rb +++ b/dev_suites/dev_infrastructure_test/infrastructure_test.rb @@ -113,6 +113,6 @@ def inline_test1_helper group from: 'failing_optional_group' group from: 'mixed_optional_group', exclude_optional: true group from: 'external_outer_group' - group from: 'empty_group' + # group from: 'empty_group' end end From 27dd0cf2d15ed00b73383121ce6a7015fad3ddf6 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 21 Dec 2023 18:41:18 -0500 Subject: [PATCH 05/11] add rspec tests --- .../infrastructure_test.rb | 2 +- spec/inferno/dsl/test_creation_spec.rb | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dev_suites/dev_infrastructure_test/infrastructure_test.rb b/dev_suites/dev_infrastructure_test/infrastructure_test.rb index 379ecb697..cc4340d9b 100644 --- a/dev_suites/dev_infrastructure_test/infrastructure_test.rb +++ b/dev_suites/dev_infrastructure_test/infrastructure_test.rb @@ -112,7 +112,7 @@ def inline_test1_helper group from: 'passing_optional_group' group from: 'failing_optional_group' group from: 'mixed_optional_group', exclude_optional: true + group from: 'empty_group' group from: 'external_outer_group' - # group from: 'empty_group' end end diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index 069b77875..ffc3f2bad 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -33,7 +33,7 @@ end it 'contains the correct groups' do - expect(suite.groups.length).to eq(5) + expect(suite.groups.length).to eq(6) expect(suite.groups.first).to eq(outer_inline_group) end @@ -434,6 +434,20 @@ end end end + + describe 'empty_group' do + let(:empty_group) { Inferno::Repositories::TestGroups.new.find('empty_group') } + + it 'contains zero tests' do + expect(empty_group.tests.length).to eq(0) + end + + it 'results in error' do + result = runner.run(:empty_group) + + expect(result.result).to eq('error') + end + end end end From 6f04693e08afcf0f554f83727a7c561a3bd5c600 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 21 Dec 2023 19:46:09 -0500 Subject: [PATCH 06/11] fix server side error for empty group --- dev_suites/dev_infrastructure_test/empty_group.rb | 2 +- lib/inferno/test_runner.rb | 14 ++++++++++++++ spec/inferno/dsl/test_creation_spec.rb | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dev_suites/dev_infrastructure_test/empty_group.rb b/dev_suites/dev_infrastructure_test/empty_group.rb index 6ea47f1be..5462950c6 100644 --- a/dev_suites/dev_infrastructure_test/empty_group.rb +++ b/dev_suites/dev_infrastructure_test/empty_group.rb @@ -1,7 +1,7 @@ module InfrastructureTest class EmptyGroup < Inferno::TestGroup id 'empty_group' - title 'Empty group title' + title 'Empty Group' fhir_client :empty_group do url 'EMPTY_GROUP' diff --git a/lib/inferno/test_runner.rb b/lib/inferno/test_runner.rb index 41a4c12ad..b82974927 100644 --- a/lib/inferno/test_runner.rb +++ b/lib/inferno/test_runner.rb @@ -123,6 +123,16 @@ def run_group(group, scratch) } end + if group.children.empty? + group_result = persist_result( + group.reference_hash.merge(result: 'omit', + result_message: 'No tests defined', + input_json: JSON.generate(group_inputs_with_values), + output_json: '[]')) + update_parent_result(group.parent) + return group_result + end + results = [] group.children(test_session.suite_options).each do |child| result = run(child, scratch) @@ -202,6 +212,10 @@ def save_outputs(runnable_instance) end def persist_result(params) + + ## XXX + # binding.pry + result = results_repo.create( params.merge(test_run_id: test_run.id, test_session_id: test_session.id) ) diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index ffc3f2bad..f2716c4c1 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -62,7 +62,7 @@ results = results_repo.current_results_for_test_session(test_session.id) - expect(results.length).to eq(16) + expect(results.length).to eq(17) required_results = results.reject(&:optional?) non_passing_results = required_results.reject { |result| result.result == 'pass' } @@ -443,7 +443,7 @@ end it 'results in error' do - result = runner.run(:empty_group) + result = runner.run(empty_group) expect(result.result).to eq('error') end From ca96db1cda01f5f69eace398adad9dba7ddb9014 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Fri, 22 Dec 2023 19:37:13 -0500 Subject: [PATCH 07/11] debugging rspec tests --- lib/inferno/test_runner.rb | 16 +++++----------- spec/inferno/dsl/test_creation_spec.rb | 11 +++++++---- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/inferno/test_runner.rb b/lib/inferno/test_runner.rb index b82974927..d1ce75410 100644 --- a/lib/inferno/test_runner.rb +++ b/lib/inferno/test_runner.rb @@ -124,11 +124,10 @@ def run_group(group, scratch) end if group.children.empty? - group_result = persist_result( - group.reference_hash.merge(result: 'omit', - result_message: 'No tests defined', - input_json: JSON.generate(group_inputs_with_values), - output_json: '[]')) + group_result = persist_result(group.reference_hash.merge(result: 'omit', + result_message: 'No tests defined', + input_json: JSON.generate(group_inputs_with_values), + output_json: '[]')) update_parent_result(group.parent) return group_result end @@ -142,8 +141,6 @@ def run_group(group, scratch) results.flatten! - ## XXX when group has no children this raises Sequel::ValidationError - # binding.pry group_result = persist_result(group.reference_hash.merge(result: roll_up_result(results), input_json: JSON.generate(group_inputs_with_values))) @@ -212,10 +209,7 @@ def save_outputs(runnable_instance) end def persist_result(params) - - ## XXX - # binding.pry - + binding.pry result = results_repo.create( params.merge(test_run_id: test_run.id, test_session_id: test_session.id) ) diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index f2716c4c1..6176ade44 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -66,8 +66,10 @@ required_results = results.reject(&:optional?) non_passing_results = required_results.reject { |result| result.result == 'pass' } + bad_results = non_passing_results.reject { |result| result.test_group.id == 'infra_test-empty_group' && + result.result == 'omit' } - expect(non_passing_results).to be_empty, non_passing_results.map { |r| + expect(bad_results).to be_empty, bad_results.map { |r| "#{r.runnable.title}: #{r.result_message}" }.join("\n") end @@ -436,16 +438,17 @@ end describe 'empty_group' do - let(:empty_group) { Inferno::Repositories::TestGroups.new.find('empty_group') } + let(:empty_group) { Inferno::Repositories::TestGroups.new.find('infra_test-empty_group') } it 'contains zero tests' do expect(empty_group.tests.length).to eq(0) end - it 'results in error' do + it 'results in omit' do result = runner.run(empty_group) - expect(result.result).to eq('error') + expect(result.result).to eq('omit') + expect(result.message).to eq('No tests defined') end end end From 21436bf951533b47a4f376a18edb45361f54e2f3 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 4 Jan 2024 11:17:24 -0500 Subject: [PATCH 08/11] clean comments --- lib/inferno/apps/web/controllers/test_runs/show.rb | 3 --- lib/inferno/test_runner.rb | 1 - 2 files changed, 4 deletions(-) diff --git a/lib/inferno/apps/web/controllers/test_runs/show.rb b/lib/inferno/apps/web/controllers/test_runs/show.rb index 29fb9cb8b..d9c8a4688 100644 --- a/lib/inferno/apps/web/controllers/test_runs/show.rb +++ b/lib/inferno/apps/web/controllers/test_runs/show.rb @@ -21,9 +21,6 @@ def handle(req, res) end end - ## XXX - # binding.pry - test_session = test_sessions_repo.find(test_run.test_session_id) res.body = serialize(test_run, suite_options: test_session.suite_options) end diff --git a/lib/inferno/test_runner.rb b/lib/inferno/test_runner.rb index d1ce75410..ff3751ae5 100644 --- a/lib/inferno/test_runner.rb +++ b/lib/inferno/test_runner.rb @@ -209,7 +209,6 @@ def save_outputs(runnable_instance) end def persist_result(params) - binding.pry result = results_repo.create( params.merge(test_run_id: test_run.id, test_session_id: test_session.id) ) From 9f247e2257f1f7255896043265dbf410a0c54ace Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Thu, 4 Jan 2024 11:36:28 -0500 Subject: [PATCH 09/11] rubocop conformance --- dev_suites/dev_infrastructure_test/empty_group.rb | 1 - lib/inferno/apps/web/controllers/test_runs/show.rb | 2 +- spec/inferno/dsl/test_creation_spec.rb | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev_suites/dev_infrastructure_test/empty_group.rb b/dev_suites/dev_infrastructure_test/empty_group.rb index 5462950c6..9261e591e 100644 --- a/dev_suites/dev_infrastructure_test/empty_group.rb +++ b/dev_suites/dev_infrastructure_test/empty_group.rb @@ -6,6 +6,5 @@ class EmptyGroup < Inferno::TestGroup fhir_client :empty_group do url 'EMPTY_GROUP' end - end end diff --git a/lib/inferno/apps/web/controllers/test_runs/show.rb b/lib/inferno/apps/web/controllers/test_runs/show.rb index d9c8a4688..df7e33c32 100644 --- a/lib/inferno/apps/web/controllers/test_runs/show.rb +++ b/lib/inferno/apps/web/controllers/test_runs/show.rb @@ -20,7 +20,7 @@ def handle(req, res) repo.results_for_test_run(test_run.id) end end - + test_session = test_sessions_repo.find(test_run.test_session_id) res.body = serialize(test_run, suite_options: test_session.suite_options) end diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index 6176ade44..28b9a4c32 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -66,8 +66,9 @@ required_results = results.reject(&:optional?) non_passing_results = required_results.reject { |result| result.result == 'pass' } - bad_results = non_passing_results.reject { |result| result.test_group.id == 'infra_test-empty_group' && - result.result == 'omit' } + bad_results = non_passing_results.reject do |result| + result.test_group.id == 'infra_test-empty_group' && result.result == 'omit' + end expect(bad_results).to be_empty, bad_results.map { |r| "#{r.runnable.title}: #{r.result_message}" From e03eb8de6c4dd9a74f9f7115a0fadbf6d5555a1b Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Mon, 8 Jan 2024 11:01:22 -0500 Subject: [PATCH 10/11] fix rspec tests --- spec/inferno/dsl/test_creation_spec.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index 28b9a4c32..4a9a5791d 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -439,7 +439,14 @@ end describe 'empty_group' do - let(:empty_group) { Inferno::Repositories::TestGroups.new.find('infra_test-empty_group') } + let(:empty_group) { InfrastructureTest::EmptyGroup } + let(:test_run) do + repo_create( + :test_run, + runnable: { test_group_id: empty_group.id }, + test_session: + ) + end it 'contains zero tests' do expect(empty_group.tests.length).to eq(0) @@ -449,7 +456,7 @@ result = runner.run(empty_group) expect(result.result).to eq('omit') - expect(result.message).to eq('No tests defined') + expect(result.result_message).to eq('No tests defined') end end end From 426a0713a063be87b8941523a025c4ed9a80d2d7 Mon Sep 17 00:00:00 2001 From: Shaumik-Ashraf Date: Wed, 17 Jan 2024 17:07:25 -0500 Subject: [PATCH 11/11] move empty group result test to test_runner_spec.rb --- spec/inferno/dsl/test_creation_spec.rb | 7 ------- spec/inferno/test_runner_spec.rb | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/spec/inferno/dsl/test_creation_spec.rb b/spec/inferno/dsl/test_creation_spec.rb index 4a9a5791d..8e2ca541f 100644 --- a/spec/inferno/dsl/test_creation_spec.rb +++ b/spec/inferno/dsl/test_creation_spec.rb @@ -451,13 +451,6 @@ it 'contains zero tests' do expect(empty_group.tests.length).to eq(0) end - - it 'results in omit' do - result = runner.run(empty_group) - - expect(result.result).to eq('omit') - expect(result.result_message).to eq('No tests defined') - end end end end diff --git a/spec/inferno/test_runner_spec.rb b/spec/inferno/test_runner_spec.rb index b25d413b2..abbb4b663 100644 --- a/spec/inferno/test_runner_spec.rb +++ b/spec/inferno/test_runner_spec.rb @@ -165,4 +165,23 @@ def error_results_message(error_results) expect(results).to all(eq('pass')) end end + + describe 'when running an empty group' do + let(:test_session) { repo_create(:test_session, test_suite_id: 'infra_test') } + let(:empty_group) { InfrastructureTest::EmptyGroup } + let(:test_run) do + repo_create( + :test_run, + runnable: { test_group_id: empty_group.id }, + test_session: + ) + end + + it 'results in omit' do + result = runner.run(empty_group) + + expect(result.result).to eq('omit') + expect(result.result_message).to eq('No tests defined') + end + end end