Merge branch 'main' into rob/68-determine-how-to-store-category-infor…

…mation-for-conditions

.github/workflows/ci.yaml

@@ -47,7 +47,7 @@ jobs:
         run: npm test
 
   end-to-end-tests:
-    timeout-minutes: 10
+    timeout-minutes: 15
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -67,13 +67,29 @@ jobs:
         run: docker compose build --no-cache
       - name: Run Query Connector
         working-directory: ./query-connector
-        run: docker compose up -d
+        run: docker compose -f ./docker-compose-e2e.yaml up -d
       - name: Poll until Query Connector is ready
         run: |
           until curl -s http://localhost:3000/tefca-viewer; do
             echo "Waiting for Query Connector to be ready before running Playwright..."
             sleep 5
           done
+      - name: Poll until HAPI server is ready
+        run: |
+          until response=$(curl -v -s -w "HTTP_STATUS:%{http_code}" http://localhost:8080/fhir/Patient); do
+            # Extract status code from the response
+            http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F: '{print $2}')
+            echo "Waiting for HAPI server to be ready..."
+            echo "Response code: $http_status"
+            echo "Full response: $response"
+            sleep 5
+          done
+      - name: Poll until FHIR server has data
+        run: |
+          until curl -s http://localhost:8080/fhir/Patient | jq '.entry | length > 0' | grep -q 'true'; do
+            echo "Waiting for FHIR server to have data..."
+            sleep 5
+          done
       - name: Playwright Tests
         working-directory: ./query-connector
         run: npx playwright test e2e --reporter=list --config playwright.config.ts

query-connector/docker-compose-dev.yaml

@@ -29,3 +29,19 @@ services:
     depends_on:
       db:
         condition: service_started
+
+  # HAPI FHIR Server for running e2e tests
+  hapi-fhir-server:
+    image: "hapiproject/hapi:latest"
+    ports:
+      - "8080:8080"
+  data-loader:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    volumes:
+      - "./src/app/tests/assets/GoldenSickPatient.json:/etc/GoldenSickPatient.json"
+      - "./post_e2e_data_hapi.sh:/post_e2e_data_hapi.sh"
+    command: ["sh", "post_e2e_data_hapi.sh"]
+    depends_on:
+      - hapi-fhir-server

query-connector/docker-compose-e2e.yaml

@@ -0,0 +1,50 @@
+services:
+  # PostgreSQL DB for custom query and value set storage
+  db:
+    image: "postgres:alpine"
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=pw
+      - POSTGRES_DB=tefca_db
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 2s
+      timeout: 5s
+      retries: 20
+
+  # Next.js app with Flyway
+  tefca-viewer:
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: Dockerfile
+    tty: true
+    ports:
+      - "3000:3000"
+    environment:
+      - NODE_ENV=production
+      - DATABASE_URL=postgres://postgres:pw@db:5432/tefca_db
+      - NEXT_PUBLIC_HAPI_FHIR_URL=http://hapi-fhir-server:8080
+      # Note: you must have a local .env file with the ERSD_API_KEY set to a key
+      # obtained from the ERSD API at https://ersd.aimsplatform.org/#/api-keys
+    depends_on:
+      db:
+        condition: service_healthy
+  # HAPI FHIR Server for running e2e tests
+  hapi-fhir-server:
+    image: "hapiproject/hapi:latest"
+    ports:
+      - "8080:8080"
+  # Loads synthetic data into hapi-fhir-server for e2e tests
+  data-loader:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    volumes:
+      - "./src/app/tests/assets/GoldenSickPatient.json:/etc/GoldenSickPatient.json"
+      - "./post_e2e_data_hapi.sh:/post_e2e_data_hapi.sh"
+    command: ["sh", "post_e2e_data_hapi.sh"]
+    depends_on:
+      - hapi-fhir-server

query-connector/e2e/alternate_queries.spec.ts

@@ -20,6 +20,12 @@ test.describe("alternate queries with the Query Connector", () => {
     await page.getByLabel("Last Name").clear();
     await page.getByLabel("Medical Record Number").clear();
 
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     // Among verification, make sure phone number is right
     await page.getByRole("button", { name: "Search for patient" }).click();
     await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 });
@@ -46,6 +52,12 @@ test.describe("alternate queries with the Query Connector", () => {
   test("cancer query with generalized function", async ({ page }) => {
     await page.getByRole("button", { name: "Go to the demo" }).click();
     await page.getByRole("button", { name: "Fill fields" }).click();
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     await page.getByRole("button", { name: "Search for patient" }).click();
     await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 });
 
@@ -68,6 +80,12 @@ test.describe("alternate queries with the Query Connector", () => {
   }) => {
     await page.getByRole("button", { name: "Go to the demo" }).click();
     await page.getByRole("button", { name: "Fill fields" }).click();
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     await page.getByRole("button", { name: "Search for patient" }).click();
     await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 });
     await page.getByRole("link", { name: "Select patient" }).click();

query-connector/e2e/customize_query.spec.ts

@@ -23,6 +23,12 @@ test.describe("querying with the Query Connector", () => {
     ).toBeVisible();
 
     await page.getByRole("button", { name: "Fill fields" }).click();
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     await page.getByRole("button", { name: "Search for patient" }).click();
     await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 });
 

query-connector/e2e/query_workflow.spec.ts

@@ -20,6 +20,12 @@ test.describe("querying with the Query Connector", () => {
     await page.getByRole("button", { name: "Fill fields" }).click();
     await page.getByLabel("First Name").fill("Shouldnt");
     await page.getByLabel("Last Name").fill("Findanyone");
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     await page.getByRole("button", { name: "Search for patient" }).click();
 
     // Better luck next time, user!
@@ -48,6 +54,12 @@ test.describe("querying with the Query Connector", () => {
     ).toBeVisible();
 
     await page.getByRole("button", { name: "Fill fields" }).click();
+    // Select FHIR server from drop down
+    await page.getByRole("button", { name: "Advanced" }).click();
+    await page
+      .getByLabel("FHIR Server (QHIN)")
+      .selectOption("Local e2e HAPI Server: Direct");
+
     await page.getByRole("button", { name: "Search for patient" }).click();
     await expect(page.getByText("Loading")).toHaveCount(0, { timeout: 10000 });
 

query-connector/package.json

@@ -16,7 +16,7 @@
     "test:unit": "jest --testPathPattern=tests/unit",
     "test:unit:watch": "jest --watch",
     "test:integration": "jest --testPathPattern=tests/integration",
-    "test:playwright": "docker compose build --no-cache && docker compose up -d && npx playwright test --reporter=list",
+    "test:playwright": "docker compose build --no-cache && docker compose -f ./docker-compose-e2e.yaml up -d && npx playwright test --reporter=list",
     "test:playwright:local": "dotenv -e ./.env -- npx playwright test --ui",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run"

query-connector/playwright-setup.ts

@@ -1,45 +1,32 @@
-/**
- *
- */
-
 export const TEST_URL =
   process.env.TEST_ENV ?? "http://localhost:3000/tefca-viewer";
-
 /**
  *
  */
 async function globalSetup() {
   const maxRetries = 300; // Maximum number of retries
-  const delay = 1000; // Delay between retries in milliseconds
+  const delay = 5000; // Delay between retries in milliseconds
 
+  // Check TEST_URL
   for (let attempts = 0; attempts < maxRetries; attempts++) {
     try {
-      const response = await fetch(TEST_URL); // Fetch the TEST_URL
+      const response = await fetch(TEST_URL);
       if (response.status === 200) {
         console.log(`Connected to ${TEST_URL} successfully.`);
-        return; // Exit the function if the webpage loads successfully
+        break; // Proceed to the FHIR server check
       } else {
         console.log(
-          `Failed to connect to ${TEST_URL}, status: ${response.status}. Retrying...`,
+          `Failed to connect to ${TEST_URL}, status: ${response.status}. Error: ${response.text} Retrying...`,
         );
-        // Wait before the next attempt
         await new Promise((resolve) => setTimeout(resolve, delay));
       }
     } catch (error) {
       console.log(
-        `Fetch failed for ${TEST_URL}: ${
-          (error as Error).message
-        }. Retrying...`,
+        `Fetch failed for ${TEST_URL}: ${(error as Error).message}. Retrying...`,
       );
       await new Promise((resolve) => setTimeout(resolve, delay));
     }
-    // Wait before the next attempt
-    await new Promise((resolve) => setTimeout(resolve, delay));
   }
-
-  throw new Error(
-    `Unable to connect to ${TEST_URL} after ${maxRetries} attempts.`,
-  );
 }
 
 export default globalSetup;

query-connector/post_request.sh → query-connector/post_e2e_data_hapi.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 # URL to check
-URL="http://tefca-fhir-server:8080/fhir/"
-TEST_URL="http://tefca-fhir-server:8080/fhir/metadata"
+URL="http://hapi-fhir-server:8080/fhir/"
+TEST_URL="http://hapi-fhir-server:8080/fhir/metadata"
 
 # Maximum number of attempts (120 seconds / 5 seconds = 24 attempts)
 MAX_ATTEMPTS=24
@@ -20,7 +20,7 @@ while [ $attempt -lt $MAX_ATTEMPTS ]; do
         echo "Server is healthy!"
 
         # POST Bundle of synthetic data to spun up server
-        curl -X POST -H "Content-Type: application/json" -d @/etc/BundleHAPIServer.json $URL
+        curl -X POST -H "Content-Type: application/json" -d @/etc/GoldenSickPatient.json $URL
 
         exit 0
     fi

query-connector/src/app/constants.ts

@@ -81,6 +81,7 @@ export const FhirServers = [
   "JMC Meld: Direct",
   "JMC Meld: eHealthExchange",
   "Public HAPI: Direct",
+  "Local e2e HAPI Server: Direct",
   "OpenEpic: eHealthExchange",
   "CernerHelios: eHealthExchange",
   "OPHDST Meld: Direct",

query-connector/src/app/database-service.ts

@@ -270,8 +270,25 @@ function generateValueSetSqlPromise(vs: ValueSet) {
   const valueSetOid = vs.valueSetId;
 
   const valueSetUniqueId = `${valueSetOid}_${vs.valueSetVersion}`;
-  const insertValueSetSql =
-    "INSERT INTO valuesets VALUES($1,$2,$3,$4,$5,$6) RETURNING id;";
+
+  // In the event a duplicate value set by OID + Version is entered, simply
+  // update the existing one to have the new set of information
+  // ValueSets are already uniquely identified by OID + V so this just allows
+  // us to proceed with DB creation in the event a duplicate VS from another
+  // group is pulled and loaded
+  const insertValueSetSql = `
+  INSERT INTO valuesets
+    VALUES($1,$2,$3,$4,$5,$6)
+    ON CONFLICT(id)
+    DO UPDATE SET
+      id = EXCLUDED.id,
+      oid = EXCLUDED.oid,
+      version = EXCLUDED.version,
+      name = EXCLUDED.name,
+      author = EXCLUDED.author,
+      type = EXCLUDED.type
+    RETURNING id;
+  `;
   const valuesArray = [
     valueSetUniqueId,
     valueSetOid,
@@ -294,7 +311,22 @@ function generateConceptSqlPromises(vs: ValueSet) {
   const insertConceptsSqlArray = vs.concepts.map((concept) => {
     const systemPrefix = stripProtocolAndTLDFromSystemUrl(vs.system);
     const conceptUniqueId = `${systemPrefix}_${concept.code}`;
-    const insertConceptSql = `INSERT INTO concepts VALUES($1,$2,$3,$4,$5,$6) RETURNING id;`;
+
+    // Duplicate value set insertion is likely to percolate to the concept level
+    // Apply the same logic of overwriting if unique keys are the same
+    const insertConceptSql = `
+    INSERT INTO concepts
+      VALUES($1,$2,$3,$4,$5,$6)
+      ON CONFLICT(id)
+      DO UPDATE SET
+        id = EXCLUDED.id,
+        code = EXCLUDED.code,
+        code_system = EXCLUDED.code_system,
+        display = EXCLUDED.display,
+        gem_formatted_code = EXCLUDED.gem_formatted_code,
+        version = EXCLUDED.version
+      RETURNING id;
+    `;
     const conceptInsertPromise = dbClient.query(insertConceptSql, [
       conceptUniqueId,
       concept.code,
@@ -316,7 +348,20 @@ function generateValuesetConceptJoinSqlPromises(vs: ValueSet) {
   const insertConceptsSqlArray = vs.concepts.map((concept) => {
     const systemPrefix = stripProtocolAndTLDFromSystemUrl(vs.system);
     const conceptUniqueId = `${systemPrefix}_${concept.code}`;
-    const insertJoinSql = `INSERT INTO valueset_to_concept VALUES($1,$2, $3) RETURNING valueset_id, concept_id;`;
+
+    // Last place to make an overwriting upsert adjustment
+    // Even if the duplicate entries have the same data, PG will attempt to
+    // insert another row, so just make that upsert the relationship
+    const insertJoinSql = `
+    INSERT INTO valueset_to_concept
+    VALUES($1,$2,$3)
+    ON CONFLICT(id)
+    DO UPDATE SET
+      id = EXCLUDED.id,
+      valueset_id = EXCLUDED.valueset_id,
+      concept_id = EXCLUDED.concept_id
+    RETURNING valueset_id, concept_id;
+    `;
     const conceptInsertPromise = dbClient.query(insertJoinSql, [
       `${valueSetUniqueId}_${conceptUniqueId}`,
       valueSetUniqueId,

query-connector/src/app/fhir-servers.ts

@@ -28,6 +28,10 @@ export const fhirServers: Record<FHIR_SERVERS, FHIR_SERVER_CONFIG> = {
     hostname: "https://hapi.fhir.org/baseR4",
     init: {} as RequestInit,
   },
+  "Local e2e HAPI Server: Direct": {
+    hostname: "http://hapi-fhir-server:8080/fhir",
+    init: {} as RequestInit,
+  },
   "OpenEpic: eHealthExchange": configureEHX("OpenEpic"),
   "CernerHelios: eHealthExchange": configureEHX("CernerHelios"),
   "OPHDST Meld: Direct": {
@@ -85,7 +89,13 @@ class FHIRClient {
   }
 
   async get(path: string): Promise<Response> {
-    return fetch(this.hostname + path, this.init);
+    try {
+      console.log("FHIR Server: ", this.hostname);
+      return fetch(this.hostname + path, this.init);
+    } catch (error) {
+      console.error(error);
+      throw error;
+    }
   }
 
   async getBatch(paths: Array<string>): Promise<Array<Response>> {