FI-2763 Export in workflow (#16)

* Add export routes * Parse export response as json * Update endpoints and postman collection * Update wait test prompt * Change URL in request field of -poll-status payload * fix postman documentation error and slight renaming of requests * clarify postman requests * updated requester patient id in postman collection --------- Co-authored-by: Karl Naden <[email protected]>
inferno-framework · Jul 3, 2024 · 429d8ea · 429d8ea
1 parent 1240fbf
commit 429d8ea
Show file tree

Hide file tree

Showing 10 changed files with 190 additions and 87 deletions.
diff --git a/.env.development b/.env.development
@@ -2,3 +2,4 @@ INFERNO_HOST=http://localhost:4567
 FHIR_RESOURCE_VALIDATOR_URL=http://localhost/hl7validatorapi
 REDIS_URL=redis://localhost:6379/0
 FHIR_REFERENCE_SERVER=http://localhost:8080/reference-server/r4
+HOST_HEADER=localhost:8080
diff --git a/.env.production b/.env.production
@@ -1,4 +1,5 @@
 INFERNO_HOST=http://localhost
 REDIS_URL=redis://redis:6379/0
-FHIR_RESOURCE_VALIDATOR_URL=http://hl7_validator_service:3500
+VALIDATOR_URL=http://validator_service:4567
 FHIR_REFERENCE_SERVER=http://host.docker.internal:8080/reference-server/r4
+HOST_HEADER=host.docker.internal:8080
diff --git a/PDEX.postman_collection.json b/PDEX.postman_collection.json
diff --git a/lib/davinci_pdex_test_kit/mock_server.rb b/lib/davinci_pdex_test_kit/mock_server.rb
@@ -17,7 +17,7 @@ def server_proxy
       @server_proxy ||= Faraday.new(
           url: ENV.fetch('FHIR_REFERENCE_SERVER'),
           params: {},
-          headers: {'Content-Type' => 'application/json', 'Authorization' => 'Bearer SAMPLE_TOKEN'},
+          headers: {'Content-Type' => 'application/json', 'Authorization' => 'Bearer SAMPLE_TOKEN', 'Host' => ENV.fetch('HOST_HEADER')},
         )
     end
 
@@ -60,16 +60,38 @@ def everything_response(request, test = nil, test_result = nil)
       request.response_body = replace_bundle_urls(FHIR.from_contents(response.body)).to_json
     end
 
-    # def export_response(request, test = nil, test_result = nil)
-    #   response = server_proxy.get do |req|
-    #     req.url 'Group/pdex-Group/$export' #TODO: change from static response
-    #     req.headers['Prefer'] = 'respond-async'
-    #     req.headers['Accept'] = 'application/fhir+json'
-    #   end
-    #   request.status = response.status
-    #   request.response_headers = response.env.response_headers
-    #   request.response_body = response.body
-    # end
+    def export_response(request, test = nil, test_result = nil)
+      headers_as_hash = request.request_headers.map { |header| {"#{header.name}": header.value}}.reduce({}) { |reduced, curr| reduced.merge(curr)}
+      response = server_proxy.get do |req|
+        req.url 'Group/pdex-Group/$export' #TODO: change from static response
+        req.headers = headers_as_hash.merge(server_proxy.headers)
+      end
+      request.status = response.status
+      request.response_headers = response.env.response_headers
+      request.response_header("content-location").value.gsub!(/(.*)\?/, "#{new_link}/$export-poll-status?")
+      request.response_body = response.body
+    end
+
+    def export_status_response(request, test = nil, test_result = nil)
+      headers_as_hash = request.request_headers.map { |header| {"#{header.name}": header.value}}.reduce({}) { |reduced, curr| reduced.merge(curr)}
+      response = server_proxy.get do |req|
+        req.url '$export-poll-status'
+        req.params = request.query_parameters
+        req.headers = headers_as_hash.merge(server_proxy.headers)
+      end
+      request.status = response.status
+      request.response_headers = response.env.response_headers
+      request.response_body = response.status.to_i == 200 ? replace_export_urls(JSON.parse(response.body)).to_json : response.body
+      request.response_header("content-length").value = request.response_body.length
+    end
+
+    def binary_read_response(request, test = nil, test_result = nil)
+      binary_id = request.url.split('/').last
+      response = server_proxy.get('Binary/'+binary_id)
+      request.status = response.status
+      request.response_headers = response.headers
+      request.response_body = response.body
+    end
 
     def member_match_response(request, test = nil, test_result = nil)
       #remove token from request as well
@@ -170,7 +192,7 @@ def mock_operation_outcome_resource
 
     def replace_bundle_urls(bundle)
       reference_server_base = ENV.fetch('FHIR_REFERENCE_SERVER')
-      bundle.link.map! {|link| {relation: link.relation, url: link.url.gsub(reference_server_base, new_link)}}
+      bundle&.link.map! {|link| {relation: link.relation, url: link.url.gsub(reference_server_base, new_link)}}
       bundle&.entry&.map! do |bundled_resource| 
         {fullUrl: bundled_resource.fullUrl.gsub(reference_server_base, new_link),
          resource: bundled_resource.resource,
@@ -180,8 +202,15 @@ def replace_bundle_urls(bundle)
       bundle
     end
 
+    def replace_export_urls(export_status_output)
+      reference_server_base = ENV.fetch('FHIR_REFERENCE_SERVER')
+      export_status_output['output'].map! { |binary| {type: binary["type"], url: binary["url"].gsub(reference_server_base, new_link)} }
+      export_status_output['request'] = new_link + '/Patient/$export'
+      export_status_output
+    end
+
     def new_link
-      "#{Inferno::Application['base_url']}/custom/pdex_payer_client/fhir"
+      "#{Inferno::Application['base_url']}\/custom\/pdex_payer_client\/fhir"
     end
 
     # @private

diff --git a/lib/davinci_pdex_test_kit/pdex_payer_client/client_validation_test.rb b/lib/davinci_pdex_test_kit/pdex_payer_client/client_validation_test.rb
@@ -14,54 +14,13 @@ def previous_clinical_data_request_resources
       end
     end
 
-    # def collect_export_resources
-    #   export_payload = collect_export_payload
-    #   if export_payload
-    #     ndjson_list = JSON.parse(export_payload)
-    #     request_resources = ndjson_list['output'].map do |resource_binary|
-    #       retrieved_resources = Faraday.new(
-    #         url: resource_binary['url'],
-    #         headers: {'Content-Type' => 'application/json',
-    #                   'Authorization' => 'Bearer SAMPLE_TOKEN'}
-    #         ).get
-    #       connect_bundle(retrieved_resources.env.response_body) 
-    #     end
-    #     puts request_resources
-    #     request_resources.flatten
-    #   else
-    #     nil
-    #   end
-    # end
-
-    # def connect_bundle(export_binary)
-    #   export_binary.split(/(?<=}\n)(?={)/).map { |str| FHIR.from_contents(str)}
-    # end
-
-    # def collect_export_payload
-    #   url = export_request&.response_header('content-location')&.value
-    #   attempts = 0
-    #   return nil if url.nil?
-    #   while attempts < 5
-    #     request_attempt = Faraday.new(
-    #       url: url,
-    #       headers: {'Content-Type' => 'application/json',
-    #                 'Authorization' => 'Bearer SAMPLE_TOKEN',
-    #                 'Prefer' => 'respond-async',
-    #                 'Accept' => 'application/fhir+json'}
-    #       ).get
-    #     if request_attempt.status != 200
-    #       attempts += 1
-    #       sleep(2)
-    #     else
-    #       return request_attempt.env.response_body
-    #     end
-    #   end
-    #   return nil
-    # end
+    def connect_bundle(export_binary)
+      export_binary.split(/(?<=}\n)(?={)/).map { |str| FHIR.from_contents(str)}
+    end
 
-    # def export_resources
-    #   @export_resources ||= collect_export_resources 
-    # end
+    def export_resources
+      @export_resources ||= (load_tagged_requests(BINARY_TAG).map { |binary_read| binary_read.response_body.split("\n") }.flatten).map { |resource_in_binary| FHIR.from_contents(resource_in_binary)}
+    end
 
     def previous_clinical_data_requests
       @previous_clinical_data_requests ||= load_tagged_requests(SUBMIT_TAG) + [everything_request].compact
@@ -75,9 +34,13 @@ def member_match_request
       @member_match_request ||= load_tagged_requests(MEMBER_MATCH_TAG).first
     end
 
-    # def export_request
-    #   @export_request ||= load_tagged_requests(EXPORT_TAG).first
-    # end
+    def export_request
+      @export_request ||= load_tagged_requests(EXPORT_TAG).first
+    end
+
+    def export_status_request
+      @export_status_request ||= load_tagged_requests(EXPORT_STATUS_TAG).first
+    end
 
     # def patient_id_from_match_request
     #   @patient_id_from_match_request ||= member_match_request ? "999" : nil #TODO: Change from static response

diff --git a/...ci_pdex_test_kit/pdex_payer_client/clinical_data_request_tests/initial_scratch_storing.rb b/...ci_pdex_test_kit/pdex_payer_client/clinical_data_request_tests/initial_scratch_storing.rb
@@ -17,15 +17,13 @@ class PDexClientScratchStorage < Inferno::Test
           scratch[resource.resourceType.to_sym] |= [resource]
         end
       end
-      # if export_resources
-      #   info "Attempted an $export request"
-      #   export_resources.each do |resource|
-      #     scratch[resource.resourceType.to_sym] ||= []
-      #     scratch[resource.resourceType.to_sym] |= [resource]
-      #   end
-      # elsif export_request
-      #   info "Found an $export request, but no resources found.  It may not have had enough time to build"
-      # end
+      if !export_resources.empty?
+        info "Attempted an $export request"
+        export_resources.each do |resource|
+          scratch[resource.resourceType.to_sym] ||= []
+          scratch[resource.resourceType.to_sym] |= [resource]
+        end
+      end
       if everything_request
         info "Attempted an $everything request"
       end

diff --git a/lib/davinci_pdex_test_kit/pdex_payer_client/clinical_data_request_tests/initial_wait_test.rb b/lib/davinci_pdex_test_kit/pdex_payer_client/clinical_data_request_tests/initial_wait_test.rb
@@ -19,6 +19,8 @@ class PDexClientSubmitMustSupportTest < Inferno::Test
           Submit PDex requests via at least one of the following methods:
           * Single Resource API: `#{submit_url}`, with `:endpoint` replaced with the endpoint you want to reach
           * $everything method: `#{everything_url}`, with `:patient` replaced with the patient you are matching
+          * $export method: `#{export_url}`, see workflow process at the [Bulk Data IG](https://hl7.org/fhir/uv/bulkdata/STU2/)
+          * $export-poll-status: `#{export_status_url}`, then continue to make reads from the binaries if a payload is delivered
           
           and [click here](#{resume_clinical_data_url}?token=#{access_token}) when done.
         )

diff --git a/lib/davinci_pdex_test_kit/pdex_payer_client_suite.rb b/lib/davinci_pdex_test_kit/pdex_payer_client_suite.rb
@@ -88,21 +88,31 @@ def self.test_resumes?(test)
                             resumes: method(:test_resumes?) do |request|
         PDexPayerClientSuite.extract_bearer_token(request)
       end
-  
+
       record_response_route :get, SUBMIT_PATH, SUBMIT_TAG, method(:claim_response),
                             resumes: method(:test_resumes?) do |request|
         PDexPayerClientSuite.extract_bearer_token(request)
       end
 
+      record_response_route :get, BINARY_PATH, BINARY_TAG, method(:binary_read_response),
+                            resumes: method(:test_resumes?) do |request|
+        PDexPayerClientSuite.extract_bearer_token(request)
+      end
+
       record_response_route :get, EVERYTHING_PATH, EVERYTHING_TAG, method(:everything_response),
                             resumes: method(:test_resumes?) do |request|
         PDexPayerClientSuite.extract_bearer_token(request)
       end
 
-      # record_response_route :get, EXPORT_PATH, EXPORT_TAG, method(:export_response),
-      #                       resumes: method(:test_resumes?) do |request|
-      #   PDexPayerClientSuite.extract_bearer_token(request)
-      # end
+      record_response_route :get, EXPORT_PATH, EXPORT_TAG, method(:export_response),
+                            resumes: method(:test_resumes?) do |request|
+        PDexPayerClientSuite.extract_bearer_token(request)
+      end
+
+      record_response_route :get, EXPORT_STATUS_PATH, EXPORT_STATUS_TAG, method(:export_status_response),
+                            resumes: method(:test_resumes?) do |request|
+        PDexPayerClientSuite.extract_bearer_token(request)
+      end
 
       record_response_route :post, MEMBER_MATCH_PATH, MEMBER_MATCH_TAG, method(:member_match_response),
                             resumes: method(:test_resumes?) do |request|

diff --git a/lib/davinci_pdex_test_kit/tags.rb b/lib/davinci_pdex_test_kit/tags.rb
@@ -3,7 +3,9 @@
 module DaVinciPDexTestKit
   AUTH_TAG = 'pdex_auth'
   SUBMIT_TAG = 'pdex_submit'
-  # EXPORT_TAG = 'pdex_export'
+  BINARY_TAG = 'pdex_binary'
+  EXPORT_TAG = 'pdex_export'
+  EXPORT_STATUS_TAG = 'pdex_export_status'
   EVERYTHING_TAG = 'pdex_everything'
   MEMBER_MATCH_TAG = 'pdex_member_match'
 end
diff --git a/lib/davinci_pdex_test_kit/urls.rb b/lib/davinci_pdex_test_kit/urls.rb
@@ -2,10 +2,12 @@ module DaVinciPDexTestKit
   TOKEN_PATH = '/mock_auth/token'
   PATIENT_PATH = '/fhir/Patient'
   SUBMIT_PATH = '/fhir/:endpoint'
+  BINARY_PATH = '/fhir/Binary/:id'
   METADATA_PATH = '/fhir/metadata'
   EVERYTHING_PATH = '/fhir/Patient/:patient/$everything'
   MEMBER_MATCH_PATH = '/fhir/Patient/$member-match'
-  # EXPORT_PATH = '/fhir/Patient/$export'
+  EXPORT_PATH = '/fhir/Patient/$export'
+  EXPORT_STATUS_PATH = '/fhir/$export-poll-status'
   BASE_FHIR_PATH = '/fhir'
   RESUME_PASS_PATH = '/resume_pass'
   RESUME_CLINICAL_DATA_PATH = '/resume_clinical_data'
@@ -32,6 +34,10 @@ def submit_url
       @submit_url ||= base_url + SUBMIT_PATH
     end
 
+    def binary_url
+      @binary_url ||= base_url + BINARY_PATH
+    end
+
     def metadata_url
       @metadata_url ||= base_url + METADATA_PATH
     end
@@ -44,9 +50,13 @@ def member_match_url
       @member_match_url ||= base_url + MEMBER_MATCH_PATH
     end
 
-    # def export_url
-    #   @export_url ||= base_url + EXPORT_PATH
-    # end
+    def export_url
+      @export_url ||= base_url + EXPORT_PATH
+    end
+
+    def export_status_url
+      @export_status_url ||= base_url + EXPORT_STATUS_PATH
+    end
 
     def resume_pass_url
       @resume_pass_url ||= base_url + RESUME_PASS_PATH