From 8cc0597c2b9ee534b15384ac85bee13ead18dba0 Mon Sep 17 00:00:00 2001 From: bamader <49412165+bamader@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:18:43 -0400 Subject: [PATCH 1/2] Allow overwriting inserts (#91) --- query-connector/src/app/database-service.ts | 53 +++++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/query-connector/src/app/database-service.ts b/query-connector/src/app/database-service.ts index 610cc634f..02f2ce308 100644 --- a/query-connector/src/app/database-service.ts +++ b/query-connector/src/app/database-service.ts @@ -270,8 +270,25 @@ function generateValueSetSqlPromise(vs: ValueSet) { const valueSetOid = vs.valueSetId; const valueSetUniqueId = `${valueSetOid}_${vs.valueSetVersion}`; - const insertValueSetSql = - "INSERT INTO valuesets VALUES($1,$2,$3,$4,$5,$6) RETURNING id;"; + + // In the event a duplicate value set by OID + Version is entered, simply + // update the existing one to have the new set of information + // ValueSets are already uniquely identified by OID + V so this just allows + // us to proceed with DB creation in the event a duplicate VS from another + // group is pulled and loaded + const insertValueSetSql = ` + INSERT INTO valuesets + VALUES($1,$2,$3,$4,$5,$6) + ON CONFLICT(id) + DO UPDATE SET + id = EXCLUDED.id, + oid = EXCLUDED.oid, + version = EXCLUDED.version, + name = EXCLUDED.name, + author = EXCLUDED.author, + type = EXCLUDED.type + RETURNING id; + `; const valuesArray = [ valueSetUniqueId, valueSetOid, @@ -294,7 +311,22 @@ function generateConceptSqlPromises(vs: ValueSet) { const insertConceptsSqlArray = vs.concepts.map((concept) => { const systemPrefix = stripProtocolAndTLDFromSystemUrl(vs.system); const conceptUniqueId = `${systemPrefix}_${concept.code}`; - const insertConceptSql = `INSERT INTO concepts VALUES($1,$2,$3,$4,$5,$6) RETURNING id;`; + + // Duplicate value set insertion is likely to percolate to the concept level + // Apply the same logic of overwriting if unique keys are the same + const insertConceptSql = ` + INSERT INTO concepts + VALUES($1,$2,$3,$4,$5,$6) + ON CONFLICT(id) + DO UPDATE SET + id = EXCLUDED.id, + code = EXCLUDED.code, + code_system = EXCLUDED.code_system, + display = EXCLUDED.display, + gem_formatted_code = EXCLUDED.gem_formatted_code, + version = EXCLUDED.version + RETURNING id; + `; const conceptInsertPromise = dbClient.query(insertConceptSql, [ conceptUniqueId, concept.code, @@ -316,7 +348,20 @@ function generateValuesetConceptJoinSqlPromises(vs: ValueSet) { const insertConceptsSqlArray = vs.concepts.map((concept) => { const systemPrefix = stripProtocolAndTLDFromSystemUrl(vs.system); const conceptUniqueId = `${systemPrefix}_${concept.code}`; - const insertJoinSql = `INSERT INTO valueset_to_concept VALUES($1,$2, $3) RETURNING valueset_id, concept_id;`; + + // Last place to make an overwriting upsert adjustment + // Even if the duplicate entries have the same data, PG will attempt to + // insert another row, so just make that upsert the relationship + const insertJoinSql = ` + INSERT INTO valueset_to_concept + VALUES($1,$2,$3) + ON CONFLICT(id) + DO UPDATE SET + id = EXCLUDED.id, + valueset_id = EXCLUDED.valueset_id, + concept_id = EXCLUDED.concept_id + RETURNING valueset_id, concept_id; + `; const conceptInsertPromise = dbClient.query(insertJoinSql, [ `${valueSetUniqueId}_${conceptUniqueId}`, valueSetUniqueId, From 9950139429910880844fedc681de5acaa2c714a0 Mon Sep 17 00:00:00 2001 From: Marcelle <53578688+m-goggins@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:32:14 -0400 Subject: [PATCH 2/2] Set up local HAPI server for e2e tests (#90) Co-authored-by: bamader Co-authored-by: bamader <49412165+bamader@users.noreply.github.com> --- .github/workflows/ci.yaml | 20 +++++++- query-connector/docker-compose-dev.yaml | 16 ++++++ query-connector/docker-compose-e2e.yaml | 50 +++++++++++++++++++ query-connector/e2e/alternate_queries.spec.ts | 18 +++++++ query-connector/e2e/customize_query.spec.ts | 6 +++ query-connector/e2e/query_workflow.spec.ts | 12 +++++ query-connector/package.json | 2 +- query-connector/playwright-setup.ts | 25 +++------- ...{post_request.sh => post_e2e_data_hapi.sh} | 6 +-- query-connector/src/app/constants.ts | 1 + query-connector/src/app/fhir-servers.ts | 12 ++++- 11 files changed, 142 insertions(+), 26 deletions(-) create mode 100644 query-connector/docker-compose-e2e.yaml rename query-connector/{post_request.sh => post_e2e_data_hapi.sh} (84%) mode change 100755 => 100644 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 39dbf3233..475a4f41b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,7 +47,7 @@ jobs: run: npm test end-to-end-tests: - timeout-minutes: 10 + timeout-minutes: 15 runs-on: ubuntu-latest steps: - name: Checkout code @@ -67,13 +67,29 @@ jobs: run: docker compose build --no-cache - name: Run Query Connector working-directory: ./query-connector - run: docker compose up -d + run: docker compose -f ./docker-compose-e2e.yaml up -d - name: Poll until Query Connector is ready run: | until curl -s http://localhost:3000/tefca-viewer; do echo "Waiting for Query Connector to be ready before running Playwright..." sleep 5 done + - name: Poll until HAPI server is ready + run: | + until response=$(curl -v -s -w "HTTP_STATUS:%{http_code}" http://localhost:8080/fhir/Patient); do + # Extract status code from the response + http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F: '{print $2}') + echo "Waiting for HAPI server to be ready..." + echo "Response code: $http_status" + echo "Full response: $response" + sleep 5 + done + - name: Poll until FHIR server has data + run: | + until curl -s http://localhost:8080/fhir/Patient | jq '.entry | length > 0' | grep -q 'true'; do + echo "Waiting for FHIR server to have data..." + sleep 5 + done - name: Playwright Tests working-directory: ./query-connector run: npx playwright test e2e --reporter=list --config playwright.config.ts diff --git a/query-connector/docker-compose-dev.yaml b/query-connector/docker-compose-dev.yaml index 5b50552de..4a0d4df4f 100644 --- a/query-connector/docker-compose-dev.yaml +++ b/query-connector/docker-compose-dev.yaml @@ -29,3 +29,19 @@ services: depends_on: db: condition: service_started + + # HAPI FHIR Server for running e2e tests + hapi-fhir-server: + image: "hapiproject/hapi:latest" + ports: + - "8080:8080" + data-loader: + build: + context: . + dockerfile: Dockerfile.dev + volumes: + - "./src/app/tests/assets/GoldenSickPatient.json:/etc/GoldenSickPatient.json" + - "./post_e2e_data_hapi.sh:/post_e2e_data_hapi.sh" + command: ["sh", "post_e2e_data_hapi.sh"] + depends_on: + - hapi-fhir-server diff --git a/query-connector/docker-compose-e2e.yaml b/query-connector/docker-compose-e2e.yaml new file mode 100644 index 000000000..6cb47cd32 --- /dev/null +++ b/query-connector/docker-compose-e2e.yaml @@ -0,0 +1,50 @@ +services: + # PostgreSQL DB for custom query and value set storage + db: + image: "postgres:alpine" + ports: + - "5432:5432" + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=pw + - POSTGRES_DB=tefca_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 5s + retries: 20 + + # Next.js app with Flyway + tefca-viewer: + platform: linux/amd64 + build: + context: . + dockerfile: Dockerfile + tty: true + ports: + - "3000:3000" + environment: + - NODE_ENV=production + - DATABASE_URL=postgres://postgres:pw@db:5432/tefca_db + - NEXT_PUBLIC_HAPI_FHIR_URL=http://hapi-fhir-server:8080 + # Note: you must have a local .env file with the ERSD_API_KEY set to a key + # obtained from the ERSD API at https://ersd.aimsplatform.org/#/api-keys + depends_on: + db: + condition: service_healthy + # HAPI FHIR Server for running e2e tests + hapi-fhir-server: + image: "hapiproject/hapi:latest" + ports: + - "8080:8080" + # Loads synthetic data into hapi-fhir-server for e2e tests + data-loader: + build: + context: . + dockerfile: Dockerfile.dev + volumes: + - "./src/app/tests/assets/GoldenSickPatient.json:/etc/GoldenSickPatient.json" + - "./post_e2e_data_hapi.sh:/post_e2e_data_hapi.sh" + command: ["sh", "post_e2e_data_hapi.sh"] + depends_on: + - hapi-fhir-server diff --git a/query-connector/e2e/alternate_queries.spec.ts b/query-connector/e2e/alternate_queries.spec.ts index 2f0b3c055..2f07aef13 100644 --- a/query-connector/e2e/alternate_queries.spec.ts +++ b/query-connector/e2e/alternate_queries.spec.ts @@ -20,6 +20,12 @@ test.describe("alternate queries with the Query Connector", () => { await page.getByLabel("Last Name").clear(); await page.getByLabel("Medical Record Number").clear(); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + // Among verification, make sure phone number is right await page.getByRole("button", { name: "Search for patient" }).click(); await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 }); @@ -46,6 +52,12 @@ test.describe("alternate queries with the Query Connector", () => { test("cancer query with generalized function", async ({ page }) => { await page.getByRole("button", { name: "Go to the demo" }).click(); await page.getByRole("button", { name: "Fill fields" }).click(); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + await page.getByRole("button", { name: "Search for patient" }).click(); await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 }); @@ -68,6 +80,12 @@ test.describe("alternate queries with the Query Connector", () => { }) => { await page.getByRole("button", { name: "Go to the demo" }).click(); await page.getByRole("button", { name: "Fill fields" }).click(); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + await page.getByRole("button", { name: "Search for patient" }).click(); await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 }); await page.getByRole("link", { name: "Select patient" }).click(); diff --git a/query-connector/e2e/customize_query.spec.ts b/query-connector/e2e/customize_query.spec.ts index cbe16de1e..b935c528b 100644 --- a/query-connector/e2e/customize_query.spec.ts +++ b/query-connector/e2e/customize_query.spec.ts @@ -23,6 +23,12 @@ test.describe("querying with the Query Connector", () => { ).toBeVisible(); await page.getByRole("button", { name: "Fill fields" }).click(); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + await page.getByRole("button", { name: "Search for patient" }).click(); await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 }); diff --git a/query-connector/e2e/query_workflow.spec.ts b/query-connector/e2e/query_workflow.spec.ts index 5a000478d..2437a4530 100644 --- a/query-connector/e2e/query_workflow.spec.ts +++ b/query-connector/e2e/query_workflow.spec.ts @@ -20,6 +20,12 @@ test.describe("querying with the Query Connector", () => { await page.getByRole("button", { name: "Fill fields" }).click(); await page.getByLabel("First Name").fill("Shouldnt"); await page.getByLabel("Last Name").fill("Findanyone"); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + await page.getByRole("button", { name: "Search for patient" }).click(); // Better luck next time, user! @@ -48,6 +54,12 @@ test.describe("querying with the Query Connector", () => { ).toBeVisible(); await page.getByRole("button", { name: "Fill fields" }).click(); + // Select FHIR server from drop down + await page.getByRole("button", { name: "Advanced" }).click(); + await page + .getByLabel("FHIR Server (QHIN)") + .selectOption("Local e2e HAPI Server: Direct"); + await page.getByRole("button", { name: "Search for patient" }).click(); await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 }); diff --git a/query-connector/package.json b/query-connector/package.json index 0ecf09926..bebf764e5 100644 --- a/query-connector/package.json +++ b/query-connector/package.json @@ -16,7 +16,7 @@ "test:unit": "jest --testPathPattern=tests/unit", "test:unit:watch": "jest --watch", "test:integration": "jest --testPathPattern=tests/integration", - "test:playwright": "docker compose build --no-cache && docker compose up -d && npx playwright test --reporter=list", + "test:playwright": "docker compose build --no-cache && docker compose -f ./docker-compose-e2e.yaml up -d && npx playwright test --reporter=list", "test:playwright:local": "dotenv -e ./.env -- npx playwright test --ui", "cypress:open": "cypress open", "cypress:run": "cypress run" diff --git a/query-connector/playwright-setup.ts b/query-connector/playwright-setup.ts index 1592a4335..06d9c6c5b 100644 --- a/query-connector/playwright-setup.ts +++ b/query-connector/playwright-setup.ts @@ -1,45 +1,32 @@ -/** - * - */ - export const TEST_URL = process.env.TEST_ENV ?? "http://localhost:3000/tefca-viewer"; - /** * */ async function globalSetup() { const maxRetries = 300; // Maximum number of retries - const delay = 1000; // Delay between retries in milliseconds + const delay = 5000; // Delay between retries in milliseconds + // Check TEST_URL for (let attempts = 0; attempts < maxRetries; attempts++) { try { - const response = await fetch(TEST_URL); // Fetch the TEST_URL + const response = await fetch(TEST_URL); if (response.status === 200) { console.log(`Connected to ${TEST_URL} successfully.`); - return; // Exit the function if the webpage loads successfully + break; // Proceed to the FHIR server check } else { console.log( - `Failed to connect to ${TEST_URL}, status: ${response.status}. Retrying...`, + `Failed to connect to ${TEST_URL}, status: ${response.status}. Error: ${response.text} Retrying...`, ); - // Wait before the next attempt await new Promise((resolve) => setTimeout(resolve, delay)); } } catch (error) { console.log( - `Fetch failed for ${TEST_URL}: ${ - (error as Error).message - }. Retrying...`, + `Fetch failed for ${TEST_URL}: ${(error as Error).message}. Retrying...`, ); await new Promise((resolve) => setTimeout(resolve, delay)); } - // Wait before the next attempt - await new Promise((resolve) => setTimeout(resolve, delay)); } - - throw new Error( - `Unable to connect to ${TEST_URL} after ${maxRetries} attempts.`, - ); } export default globalSetup; diff --git a/query-connector/post_request.sh b/query-connector/post_e2e_data_hapi.sh old mode 100755 new mode 100644 similarity index 84% rename from query-connector/post_request.sh rename to query-connector/post_e2e_data_hapi.sh index 28c21a2ae..9e55259f8 --- a/query-connector/post_request.sh +++ b/query-connector/post_e2e_data_hapi.sh @@ -1,8 +1,8 @@ #!/bin/bash # URL to check -URL="http://tefca-fhir-server:8080/fhir/" -TEST_URL="http://tefca-fhir-server:8080/fhir/metadata" +URL="http://hapi-fhir-server:8080/fhir/" +TEST_URL="http://hapi-fhir-server:8080/fhir/metadata" # Maximum number of attempts (120 seconds / 5 seconds = 24 attempts) MAX_ATTEMPTS=24 @@ -20,7 +20,7 @@ while [ $attempt -lt $MAX_ATTEMPTS ]; do echo "Server is healthy!" # POST Bundle of synthetic data to spun up server - curl -X POST -H "Content-Type: application/json" -d @/etc/BundleHAPIServer.json $URL + curl -X POST -H "Content-Type: application/json" -d @/etc/GoldenSickPatient.json $URL exit 0 fi diff --git a/query-connector/src/app/constants.ts b/query-connector/src/app/constants.ts index da12272b0..8cdba8404 100644 --- a/query-connector/src/app/constants.ts +++ b/query-connector/src/app/constants.ts @@ -81,6 +81,7 @@ export const FhirServers = [ "JMC Meld: Direct", "JMC Meld: eHealthExchange", "Public HAPI: Direct", + "Local e2e HAPI Server: Direct", "OpenEpic: eHealthExchange", "CernerHelios: eHealthExchange", "OPHDST Meld: Direct", diff --git a/query-connector/src/app/fhir-servers.ts b/query-connector/src/app/fhir-servers.ts index 990079938..5d28ca68a 100644 --- a/query-connector/src/app/fhir-servers.ts +++ b/query-connector/src/app/fhir-servers.ts @@ -28,6 +28,10 @@ export const fhirServers: Record = { hostname: "https://hapi.fhir.org/baseR4", init: {} as RequestInit, }, + "Local e2e HAPI Server: Direct": { + hostname: "http://hapi-fhir-server:8080/fhir", + init: {} as RequestInit, + }, "OpenEpic: eHealthExchange": configureEHX("OpenEpic"), "CernerHelios: eHealthExchange": configureEHX("CernerHelios"), "OPHDST Meld: Direct": { @@ -85,7 +89,13 @@ class FHIRClient { } async get(path: string): Promise { - return fetch(this.hostname + path, this.init); + try { + console.log("FHIR Server: ", this.hostname); + return fetch(this.hostname + path, this.init); + } catch (error) { + console.error(error); + throw error; + } } async getBatch(paths: Array): Promise> {