fix: display all tables related to treatment plan (#2832)

* test: point to PR for fhir converter

* feat: pipe through meditech plan of treatment data, refactor component to be more generic

* [pre-commit.ci] auto fixes from pre-commit hooks

* feat: make treatment plan tables even more generic

* fix: cleanup

* fix: cleanup

* fix: make ecr id more stable

* test: update ids in converstion tests

* fix: simplify content selector

* fix: account for whitespace, more html-ish formats

* fix: sanitize the html

* fix: appease react key warnings

* fix: appease lighthouse

* fix: appease build

* fix: handle tables with no headers

* fix: use released fhir-converter

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

containers/ecr-viewer/package.json

@@ -58,7 +58,8 @@
     "next-auth": "^4.24.7",
     "pg-promise": "^11.6.0",
     "react": "^18",
-    "react-dom": "^18"
+    "react-dom": "^18",
+    "sanitize-html": "^2.13.1"
   },
   "devDependencies": {
     "@next/eslint-plugin-next": "^14.2.5",
@@ -72,6 +73,7 @@
     "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
+    "@types/sanitize-html": "^2.13.0",
     "@typescript-eslint/eslint-plugin": "^6.17.0",
     "@typescript-eslint/parser": "^6.17.0",
     "autoprefixer": "^10.4.19",

containers/ecr-viewer/src/app/services/formatService.tsx

@@ -1,6 +1,7 @@
 import React from "react";
 import { ToolTipElement } from "@/app/view-data/components/ToolTipElement";
 import { ContactPoint } from "fhir/r4";
+import sanitizeHtml from "sanitize-html";
 
 interface Metadata {
   [key: string]: string;
@@ -330,29 +331,62 @@ export const formatString = (input: string): string => {
  */
 export function formatTablesToJSON(htmlString: string): TableJson[] {
   const parser = new DOMParser();
+  // We purposefully don't sanitize here to remain close to the original format while
+  // looking for specific patterns. The data is sanitized as it's pulled out.
   const doc = parser.parseFromString(htmlString, "text/html");
   const jsonArray: any[] = [];
-  const liArray = doc.querySelectorAll("li");
+
+  // <li>{name}<table/></li> OR <list><item>{name}<table /></item></list>
+  const liArray = doc.querySelectorAll("li, list > item");
   if (liArray.length > 0) {
     liArray.forEach((li) => {
       const tables: any[] = [];
       const resultId = getDataId(li);
       const firstChildNode = getFirstNonCommentChild(li);
       const resultName = firstChildNode
-        ? (firstChildNode.textContent?.trim() ?? "")
+        ? getElementContent(firstChildNode)
         : "";
       li.querySelectorAll("table").forEach((table) => {
         tables.push(processTable(table));
       });
       jsonArray.push({ resultId, resultName, tables });
     });
-  } else {
+
+    return jsonArray;
+  }
+
+  // <table><caption>{name}</caption></table>
+  const tableWithCaptionArray = doc.querySelectorAll("table:has(caption)");
+  if (tableWithCaptionArray.length > 0) {
     doc.querySelectorAll("table").forEach((table) => {
-      const resultName = table.caption?.textContent;
+      const resultName = getElementContent(table.caption as Node);
       const resultId = getDataId(table) ?? undefined;
       jsonArray.push({ resultId, resultName, tables: [processTable(table)] });
     });
+
+    return jsonArray;
   }
+
+  // <content>{name}</content><br/><table/>
+  const contentArray = doc.querySelectorAll("content");
+  if (contentArray.length > 0) {
+    contentArray.forEach((content) => {
+      const resultName = getElementContent(content);
+      const tables: any[] = [];
+      let sibling = content.nextElementSibling;
+
+      while (sibling !== null && sibling.tagName.toLowerCase() !== "content") {
+        if (sibling.tagName.toLowerCase() === "table") {
+          tables.push(processTable(sibling));
+        }
+        sibling = sibling.nextElementSibling;
+      }
+      if (tables.length > 0) jsonArray.push({ resultName, tables });
+    });
+
+    return jsonArray;
+  }
+
   return jsonArray;
 }
 
@@ -363,7 +397,9 @@ export function formatTablesToJSON(htmlString: string): TableJson[] {
  * @param li - The HTML element to search for the first non-comment child node.
  * @returns - The first non-comment child node, or `null` if none is found.
  */
-export function getFirstNonCommentChild(li: HTMLElement): ChildNode | null {
+export function getFirstNonCommentChild(
+  li: HTMLElement | Element,
+): ChildNode | null {
   for (let i = 0; i < li.childNodes.length; i++) {
     const node = li.childNodes[i];
     if (node.nodeType !== Node.COMMENT_NODE) {
@@ -379,7 +415,7 @@ export function getFirstNonCommentChild(li: HTMLElement): ChildNode | null {
  * @param elem - The element to search for the `data-id`.
  * @returns  - The extracted `data-id` value if found, otherwise `null`.
  */
-export function getDataId(elem: HTMLLIElement | HTMLTableElement) {
+export function getDataId(elem: HTMLLIElement | HTMLTableElement | Element) {
   if (elem.getAttribute("data-id")) return elem.getAttribute("data-id");
   for (let i = 0; i < elem.childNodes.length; i++) {
     const node = elem.childNodes[i];
@@ -406,18 +442,23 @@ function processTable(table: Element): TableRow[] {
   const jsonArray: any[] = [];
   const rows = table.querySelectorAll("tr");
   const keys: string[] = [];
+  let hasHeaders = false;
 
-  rows[0].querySelectorAll("th").forEach((header) => {
-    keys.push(header.textContent?.trim() ?? "");
-  });
+  const headers = rows[0].querySelectorAll("th");
+  if (headers.length > 0) {
+    hasHeaders = true;
+    headers.forEach((header) => {
+      keys.push(getElementContent(header));
+    });
+  }
 
   rows.forEach((row, rowIndex) => {
     // Skip the first row as it contains headers
-    if (rowIndex === 0) return;
+    if (hasHeaders && rowIndex === 0) return;
 
     const obj: TableRow = {};
     row.querySelectorAll("td").forEach((cell, cellIndex) => {
-      const key = keys[cellIndex];
+      const key = hasHeaders ? keys[cellIndex] : "Unknown Header";
 
       const metaData: Metadata = {};
       const attributes = cell.attributes || [];
@@ -429,7 +470,7 @@ function processTable(table: Element): TableRow[] {
         }
       }
       obj[key] = {
-        value: cell.textContent?.trim() ?? "",
+        value: getElementContent(cell),
         metadata: metaData,
       };
     });
@@ -439,6 +480,10 @@ function processTable(table: Element): TableRow[] {
   return jsonArray;
 }
 
+function getElementContent(el: Node): string {
+  return sanitizeHtml(el.textContent?.trim() ?? "");
+}
+
 /**
  * Extracts and concatenates all sequences of numbers and periods from each string in the input array,
  * excluding any leading and trailing periods in the first matched sequence of each string.