From a47d5d32d60693b80eb753cea7950efdd273ac24 Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 11:31:15 -0600 Subject: [PATCH 01/10] exporter.ndjson.export = true --- src/main/resources/synthea.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/synthea.properties b/src/main/resources/synthea.properties index 843845d50..39cc12a18 100644 --- a/src/main/resources/synthea.properties +++ b/src/main/resources/synthea.properties @@ -22,6 +22,7 @@ exporter.fhir.us_core_version = 5.0.1 exporter.fhir.transaction_bundle = true # using bulk_data=true will ignore exporter.pretty_print exporter.fhir.bulk_data = false +exporter.fhir.personal_data = true # included_ and excluded_resources list out the resource types to include/exclude in the csv exporters. # only one of these may be set at a time, if both are set then both will be ignored. # if neither is set, then all resource types will be included. @@ -38,6 +39,7 @@ exporter.practitioner.fhir_dstu2.export = false exporter.encoding = UTF-8 exporter.json.export = false exporter.json.include_module_history = false +exporter.ndjson.export = true exporter.csv.export = false # if exporter.csv.append_mode = true, then each run will add new data to any existing CSVs. if false, each run will clear out the files and start fresh exporter.csv.append_mode = false From 7b84a8204a54cb9b2a329db013bb9a95c8ce9d1a Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 12:03:27 -0600 Subject: [PATCH 02/10] config get exporter.ndjson.export --- src/main/java/org/mitre/synthea/export/Exporter.java | 9 +++++++++ src/main/resources/synthea.properties | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index ca3e9f885..68f432278 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -255,6 +255,15 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); writeNewFile(outFilePath, json); } + + if (Config.getAsBoolean("exporter.ndjson.export")) { + // String json = JSONExporter.export(person); + // File outDirectory = getOutputFolder("json", person); + // Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); + // writeNewFile(outFilePath, json); + } + + if (Config.getAsBoolean("exporter.csv.export")) { try { CSVExporter.getInstance().export(person, stopTime); diff --git a/src/main/resources/synthea.properties b/src/main/resources/synthea.properties index 39cc12a18..dfbecad1d 100644 --- a/src/main/resources/synthea.properties +++ b/src/main/resources/synthea.properties @@ -39,7 +39,7 @@ exporter.practitioner.fhir_dstu2.export = false exporter.encoding = UTF-8 exporter.json.export = false exporter.json.include_module_history = false -exporter.ndjson.export = true +exporter.ndjson.export = false exporter.csv.export = false # if exporter.csv.append_mode = true, then each run will add new data to any existing CSVs. if false, each run will clear out the files and start fresh exporter.csv.append_mode = false From 8efd854bf2d5b1efe96812f540ce576dffb802ec Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 13:20:01 -0600 Subject: [PATCH 03/10] endToEnd() --- src/test/java/org/mitre/synthea/identity/EntityManagerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/mitre/synthea/identity/EntityManagerTest.java b/src/test/java/org/mitre/synthea/identity/EntityManagerTest.java index db22b27e4..553d7f0f7 100644 --- a/src/test/java/org/mitre/synthea/identity/EntityManagerTest.java +++ b/src/test/java/org/mitre/synthea/identity/EntityManagerTest.java @@ -71,6 +71,7 @@ public void endToEnd() throws Exception { TestHelper.exportOff(); Config.set("exporter.fhir.export", "true"); Config.set("exporter.json.export", "true"); + Config.set("exporter.ndjson.export", "true"); Generator.DEFAULT_STATE = "North Carolina"; Path baseDirectory = tempFolder.newFolder().toPath(); Config.set("exporter.baseDirectory", baseDirectory.toString()); From eff18c7b2d8dde9d18e0ceaad6b9408bc02fe55d Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 14:17:56 -0600 Subject: [PATCH 04/10] endToEnd() --- src/test/java/org/mitre/synthea/TestHelper.java | 1 + src/test/java/org/mitre/synthea/export/JSONExporterTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/test/java/org/mitre/synthea/TestHelper.java b/src/test/java/org/mitre/synthea/TestHelper.java index 6ef564a99..fe88c0134 100644 --- a/src/test/java/org/mitre/synthea/TestHelper.java +++ b/src/test/java/org/mitre/synthea/TestHelper.java @@ -125,6 +125,7 @@ public static void exportOff() { Config.set("exporter.practitioner.fhir_stu3.export", "false"); Config.set("exporter.practitioner.fhir_dstu2.export", "false"); Config.set("exporter.json.export", "false"); + Config.set("exporter.ndjson.export", "false"); Config.set("exporter.csv.export", "false"); Config.set("exporter.cpcds.export", "false"); Config.set("exporter.bfd.export", "false"); diff --git a/src/test/java/org/mitre/synthea/export/JSONExporterTest.java b/src/test/java/org/mitre/synthea/export/JSONExporterTest.java index 2d8f77864..dbec2f859 100644 --- a/src/test/java/org/mitre/synthea/export/JSONExporterTest.java +++ b/src/test/java/org/mitre/synthea/export/JSONExporterTest.java @@ -30,6 +30,7 @@ public void export() throws Exception { List validationErrors = new ArrayList<>(); TestHelper.exportOff(); Config.set("exporter.json.export", "true"); + Config.set("exporter.ndjson.export", "true"); boolean moduleExport = person.randBoolean(); Config.set("exporter.json.include_module_history", String.valueOf(moduleExport)); String personJson = JSONExporter.export(person); From 3d148068fcd956d8c157593a6f2f881e0185f574 Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 14:31:53 -0600 Subject: [PATCH 05/10] exporter.ndjson.export writeNewFile --- src/main/java/org/mitre/synthea/export/Exporter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index 68f432278..6eced6388 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -257,10 +257,10 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime } if (Config.getAsBoolean("exporter.ndjson.export")) { - // String json = JSONExporter.export(person); - // File outDirectory = getOutputFolder("json", person); - // Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); - // writeNewFile(outFilePath, json); + String ndjson = JSONExporter.export(person); + File outDirectory = getOutputFolder("ndjson", person); + Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "ndjson")); + writeNewFile(outFilePath, ndjson); } From 7c802c98bf5c761ca011e0db3dabc9c164a2e966 Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 14:56:39 -0600 Subject: [PATCH 06/10] exporter.fhir.personal_data --- src/main/java/org/mitre/synthea/export/Exporter.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index 6eced6388..a66046667 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -236,6 +236,17 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime String entryJson = parser.encodeResourceToString(entry.getResource()); appendToFile(outFilePath, entryJson); } + // } else if (Config.getAsBoolean("exporter.fhir.longitudinal_data")) { + } else if (Config.getAsBoolean("exporter.fhir.personal_data")) { + org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); + //IParser parser = FhirR4.getContext().newNdJsonParser().setPrettyPrint(false); + IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + String filename = entry.getResource().getResourceType().toString() + ".ndjson"; + Path outFilePath = outDirectory.toPath().resolve(filename); + String entryJson = parser.encodeResourceToString(entry.getResource()); + appendToFile(outFilePath, entryJson); + } } else { String bundleJson = FhirR4.convertToFHIRJson(person, stopTime); Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); From 43111dc71038c0602b1ec47b88da4b36ad625dd1 Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 15:11:26 -0600 Subject: [PATCH 07/10] disable contents of exporter.fhir.personal_data --- .../java/org/mitre/synthea/export/Exporter.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index a66046667..3b7cc0ec5 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -238,14 +238,14 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime } // } else if (Config.getAsBoolean("exporter.fhir.longitudinal_data")) { } else if (Config.getAsBoolean("exporter.fhir.personal_data")) { - org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); - //IParser parser = FhirR4.getContext().newNdJsonParser().setPrettyPrint(false); - IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); - for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { - String filename = entry.getResource().getResourceType().toString() + ".ndjson"; - Path outFilePath = outDirectory.toPath().resolve(filename); - String entryJson = parser.encodeResourceToString(entry.getResource()); - appendToFile(outFilePath, entryJson); + // org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); + // //IParser parser = FhirR4.getContext().newNdJsonParser().setPrettyPrint(false); + // IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); + // for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + // String filename = entry.getResource().getResourceType().toString() + ".ndjson"; + // Path outFilePath = outDirectory.toPath().resolve(filename); + // String entryJson = parser.encodeResourceToString(entry.getResource()); + // appendToFile(outFilePath, entryJson); } } else { String bundleJson = FhirR4.convertToFHIRJson(person, stopTime); From 82affdffec62abc3073c9af253ca5c4949b210a4 Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 15:15:29 -0600 Subject: [PATCH 08/10] disable contents of exporter.fhir.personal_data --- src/main/java/org/mitre/synthea/export/Exporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index 3b7cc0ec5..969e8020a 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -246,7 +246,7 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime // Path outFilePath = outDirectory.toPath().resolve(filename); // String entryJson = parser.encodeResourceToString(entry.getResource()); // appendToFile(outFilePath, entryJson); - } + // } } else { String bundleJson = FhirR4.convertToFHIRJson(person, stopTime); Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); From 60037cea4a993184b13cf9f7710808bc9d6c299d Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 15:33:10 -0600 Subject: [PATCH 09/10] re-enable contents of exporter.fhir.personal_data --- .../java/org/mitre/synthea/export/Exporter.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index 969e8020a..b089b355e 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -238,15 +238,14 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime } // } else if (Config.getAsBoolean("exporter.fhir.longitudinal_data")) { } else if (Config.getAsBoolean("exporter.fhir.personal_data")) { - // org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); - // //IParser parser = FhirR4.getContext().newNdJsonParser().setPrettyPrint(false); - // IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); - // for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { - // String filename = entry.getResource().getResourceType().toString() + ".ndjson"; - // Path outFilePath = outDirectory.toPath().resolve(filename); - // String entryJson = parser.encodeResourceToString(entry.getResource()); - // appendToFile(outFilePath, entryJson); - // } + org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); + IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + String filename = entry.getResource().getResourceType().toString() + ".ndjson"; + Path outFilePath = outDirectory.toPath().resolve(filename); + String entryJson = parser.encodeResourceToString(entry.getResource()); + appendToFile(outFilePath, entryJson); + } } else { String bundleJson = FhirR4.convertToFHIRJson(person, stopTime); Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json")); From abbb425001cafea2afbfe20365a6ab299cbe857f Mon Sep 17 00:00:00 2001 From: Abbie Watson Date: Tue, 15 Aug 2023 16:22:57 -0600 Subject: [PATCH 10/10] disable exporter.fhir.personal_data --- .../org/mitre/synthea/export/Exporter.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/mitre/synthea/export/Exporter.java b/src/main/java/org/mitre/synthea/export/Exporter.java index b089b355e..5699bbbef 100644 --- a/src/main/java/org/mitre/synthea/export/Exporter.java +++ b/src/main/java/org/mitre/synthea/export/Exporter.java @@ -236,16 +236,15 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime String entryJson = parser.encodeResourceToString(entry.getResource()); appendToFile(outFilePath, entryJson); } - // } else if (Config.getAsBoolean("exporter.fhir.longitudinal_data")) { - } else if (Config.getAsBoolean("exporter.fhir.personal_data")) { - org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); - IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); - for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { - String filename = entry.getResource().getResourceType().toString() + ".ndjson"; - Path outFilePath = outDirectory.toPath().resolve(filename); - String entryJson = parser.encodeResourceToString(entry.getResource()); - appendToFile(outFilePath, entryJson); - } + // } else if (Config.getAsBoolean("exporter.fhir.personal_data")) { + // org.hl7.fhir.r4.model.Bundle bundle = FhirR4.convertToFHIR(person, stopTime); + // IParser parser = FhirR4.getContext().newJsonParser().setPrettyPrint(false); + // for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + // String filename = entry.getResource().getResourceType().toString() + ".ndjson"; + // Path outFilePath = outDirectory.toPath().resolve(filename); + // String entryJson = parser.encodeResourceToString(entry.getResource()); + // appendToFile(outFilePath, entryJson); + // } } else { String bundleJson = FhirR4.convertToFHIRJson(person, stopTime); Path outFilePath = outDirectory.toPath().resolve(filename(person, fileTag, "json"));