Merge pull request #1357 from synthetichealth/dental

Dental Modules

.gitignore

@@ -6,6 +6,7 @@
 /nbproject/**
 /build/**
 bin/**
+lib/custom/*.jar
 
 /.gradle/
 /bin/
@@ -20,6 +21,7 @@ bin/**
 src/main/resources/version.txt
 
 # don't commit the code maps
+src/main/resources/export/cdt_code_map.json
 src/main/resources/export/condition_code_map.json
 src/main/resources/export/medication_code_map.json
 src/main/resources/export/drg_code_map.json

build.gradle

@@ -92,6 +92,8 @@ dependencies {
   implementation 'info.cqframework:elm-jackson:2.1.0'
 
   implementation 'com.squareup.okhttp3:okhttp:4.10.0'
+  implementation 'org.reflections:reflections:0.10.2'
+  implementation 'net.lingala.zip4j:zip4j:2.11.5'
 
   // Java 9 no longer includes these APIs by default
   implementation 'javax.xml.bind:jaxb-api:2.4.0-b180830.0359'

src/main/java/Graphviz.java

@@ -1,5 +1,6 @@
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.stream.JsonReader;
@@ -8,6 +9,7 @@
 import guru.nidi.graphviz.attribute.Label;
 import guru.nidi.graphviz.attribute.Records;
 import guru.nidi.graphviz.attribute.Style;
+import guru.nidi.graphviz.engine.Engine;
 import guru.nidi.graphviz.engine.Format;
 import guru.nidi.graphviz.engine.GraphvizException;
 import guru.nidi.graphviz.model.Factory;
@@ -24,12 +26,10 @@
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
-import java.util.Calendar;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.TimeZone;
 
 import org.mitre.synthea.engine.Module;
 import org.mitre.synthea.export.Exporter;
@@ -47,18 +47,20 @@ public class Graphviz {
   public static void main(String[] args) throws URISyntaxException, IOException {
     File folder = Exporter.getOutputFolder("graphviz", null);
 
-    Path inputPath = null;
+    List<Path> inputPaths = new ArrayList<Path>();
     if (args != null && args.length > 0) {
       File file = new File(args[0]);
-      inputPath = file.toPath();
+      inputPaths.add(file.toPath());
     } else {
-      inputPath = Module.getModulesPath();
+      inputPaths = Module.getModulePaths();
     }
 
     System.out.println("Rendering graphs to `" + folder.getAbsolutePath() + "`...");
 
     long start = System.currentTimeMillis();
-    generateJsonModuleGraphs(inputPath, folder);
+    for (Path path : inputPaths) {
+      generateJsonModuleGraphs(path, folder);
+    }
 
     System.out.println("Completed in " + (System.currentTimeMillis() - start) + " ms.");
   }
@@ -276,8 +278,16 @@ private static void generateJsonModuleGraph(JsonObject module, File outputFolder
 
     File outputFile = outputFolder.toPath().resolve(relativePath + ".png").toFile();
     outputFile.mkdirs();
-    guru.nidi.graphviz.engine.Graphviz.fromGraph(g)
+
+    try {
+      guru.nidi.graphviz.engine.Graphviz.fromGraph(g).engine(Engine.DOT)
+      .render(Format.PNG).toFile(outputFile);
+    } catch (guru.nidi.graphviz.engine.GraphvizException gve) {
+      if (gve.getMessage().contains("Command took too long to execute")) {
+        guru.nidi.graphviz.engine.Graphviz.fromGraph(g).engine(Engine.FDP)
         .render(Format.PNG).toFile(outputFile);
+      }
+    }
   }
 
   private static String getStateDescription(JsonObject state) {
@@ -319,7 +329,13 @@ private static String getStateDescription(JsonObject state) {
         }
         break;
       case "SetAttribute":
-        String v = state.has("value") ? state.get("value").getAsString() : null;
+        String v = "null";
+        if (state.has("value")) {
+          JsonElement e = state.get("value");
+          if (e != null && e != JsonNull.INSTANCE) {
+            v = e.getAsString();
+          }
+        }
         details.append("Set ").append(state.get("attribute").getAsString()).append(" = ").append(v);
         break;
       case "Symptom":

src/main/java/org/mitre/synthea/engine/Distribution.java

@@ -11,7 +11,7 @@
  */
 public class Distribution implements Serializable {
   public enum Kind {
-    EXACT, GAUSSIAN, UNIFORM, EXPONENTIAL
+    EXACT, GAUSSIAN, UNIFORM, EXPONENTIAL, TRIANGULAR
   }
 
   public Kind kind;
@@ -50,8 +50,23 @@ public double generate(Person person) {
         break;
       case EXPONENTIAL:
         double average = this.parameters.get("mean");
-        double lambda = (1.0d / average);
-        value = 1.0d + Math.log(1.0d - person.rand()) / (-1.0d * lambda);
+        double lambda = (-1.0d / average);
+        value = 1.0d + Math.log(1.0d - person.rand()) / lambda;
+        break;
+      case TRIANGULAR:
+        /* Pick a single value based on a triangular distribution. See:
+         * https://en.wikipedia.org/wiki/Triangular_distribution
+         */
+        double min = this.parameters.get("min");
+        double mode = this.parameters.get("mode");
+        double max = this.parameters.get("max");
+        double f = (mode - min) / (max - min);
+        double rand = person.rand();
+        if (rand < f) {
+          value = min + Math.sqrt(rand * (max - min) * (mode - min));
+        } else {
+          value = max - Math.sqrt((1 - rand) * (max - min) * (max - mode));
+        }
         break;
       default:
         value = -1;
@@ -81,6 +96,10 @@ public boolean validate() {
             && this.parameters.containsKey("standardDeviation");
       case EXPONENTIAL:
         return this.parameters.containsKey("mean");
+      case TRIANGULAR:
+        return this.parameters.containsKey("min")
+            && this.parameters.containsKey("mode")
+            && this.parameters.containsKey("max");
       default:
         return false;
     }

src/main/java/org/mitre/synthea/engine/Module.java

@@ -15,13 +15,15 @@
 import java.io.Serializable;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Enumeration;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -79,8 +81,10 @@ private static Map<String, ModuleSupplier> loadModules() {
     Properties moduleOverrides = getModuleOverrides();
 
     try {
-      Path modulesPath = getModulesPath();
-      submoduleCount = walkModuleTree(modulesPath, retVal, moduleOverrides, false);
+      List<Path> modulePaths = getModulePaths();
+      for (Path path : modulePaths) {
+        submoduleCount += walkModuleTree(path, retVal, moduleOverrides, false);
+      }
     } catch (Exception e) {
       e.printStackTrace();
     }
@@ -93,16 +97,21 @@ private static Map<String, ModuleSupplier> loadModules() {
   }
 
   /**
-   * The path to the modules directory, ensuring the right file system support is loaded if
+   * Get the paths to the modules directories, ensuring the right file system support is loaded if
    * we are running from a jar file.
    * @return the path
    * @throws URISyntaxException if something goes wrong
    * @throws IOException if something goes wrong
    */
-  public static Path getModulesPath() throws URISyntaxException, IOException {
-    URI modulesURI = Module.class.getClassLoader().getResource("modules").toURI();
-    Utilities.enableReadingURIFromJar(modulesURI);
-    return Paths.get(modulesURI);
+  public static List<Path> getModulePaths() throws URISyntaxException, IOException {
+    List<Path> paths = new ArrayList<Path>();
+    Enumeration<URL> moduleURLs = Module.class.getClassLoader().getResources("modules");
+    while (moduleURLs.hasMoreElements()) {
+      URI uri = moduleURLs.nextElement().toURI();
+      Utilities.enableReadingURIFromJar(uri);
+      paths.add(Paths.get(uri));
+    }
+    return paths;
   }
 
   private static Properties getModuleOverrides() {

src/main/java/org/mitre/synthea/engine/State.java

@@ -945,6 +945,12 @@ public boolean process(Person person, long time) {
                 break;
               }
             }
+          } else if (module.getStateNames().contains(reason)) {
+            State state = module.getState(reason);
+            if (state != null && state instanceof ConditionOnset) {
+              ConditionOnset condition = (ConditionOnset) state;
+              encounter.reason = condition.codes.get(0);
+            }
           }
         }
 
@@ -1419,6 +1425,12 @@ private void applyFeatures(Person person, Medication medication) {
               medication.reasons.addAll(entry.codes);
             }
           }
+        } else if (module.getStateNames().contains(reason)) {
+          State state = module.getState(reason);
+          if (state != null && state instanceof ConditionOnset) {
+            ConditionOnset condition = (ConditionOnset) state;
+            medication.reasons.addAll(condition.codes);
+          }
         }
       }
       medication.prescriptionDetails = prescription;
@@ -1513,6 +1525,12 @@ public boolean process(Person person, long time) {
               careplan.mergeReasonList(entry.codes);
             }
           }
+        } else if (module.getStateNames().contains(reason)) {
+          State state = module.getState(reason);
+          if (state != null && state instanceof ConditionOnset) {
+            ConditionOnset condition = (ConditionOnset) state;
+            careplan.reasons.addAll(condition.codes);
+          }
         }
       }
       if (shouldAssignAttribute()) {
@@ -1621,6 +1639,12 @@ public void processOnce(Person person, long time) {
               procedure.reasons.addAll(entry.codes);
             }
           }
+        } else if (module.getStateNames().contains(reason)) {
+          State state = module.getState(reason);
+          if (state != null && state instanceof ConditionOnset) {
+            ConditionOnset condition = (ConditionOnset) state;
+            procedure.reasons.addAll(condition.codes);
+          }
         }
       }
       if ((duration != null || distribution != null) && this.stop == null) {
@@ -2286,6 +2310,12 @@ public boolean process(Person person, long time) {
               reason = entry.codes.get(0);
             }
           }
+        } else if (module.getStateNames().contains(conditionOnset)) {
+          State state = module.getState(conditionOnset);
+          if (state != null && state instanceof ConditionOnset) {
+            ConditionOnset condition = (ConditionOnset) state;
+            reason = condition.codes.get(0);
+          }
         }
       } else if (referencedByAttribute != null) {
         Entry entry = (Entry) person.attributes.get(referencedByAttribute);

src/main/java/org/mitre/synthea/export/CSVExporter.java

@@ -308,14 +308,14 @@ private void writeCSVHeaders() throws IOException {
         "START,STOP,PATIENT,PAYER,ENCOUNTER,CODE,DESCRIPTION,BASE_COST,PAYER_COVERAGE,DISPENSES,"
         + "TOTALCOST,REASONCODE,REASONDESCRIPTION");
     medications.write(NEWLINE);
-    conditions.write("START,STOP,PATIENT,ENCOUNTER,CODE,DESCRIPTION");
+    conditions.write("START,STOP,PATIENT,ENCOUNTER,SYSTEM,CODE,DESCRIPTION");
     conditions.write(NEWLINE);
     careplans.write(
         "Id,START,STOP,PATIENT,ENCOUNTER,CODE,DESCRIPTION,REASONCODE,REASONDESCRIPTION");
     careplans.write(NEWLINE);
     observations.write("DATE,PATIENT,ENCOUNTER,CATEGORY,CODE,DESCRIPTION,VALUE,UNITS,TYPE");
     observations.write(NEWLINE);
-    procedures.write("START,STOP,PATIENT,ENCOUNTER,CODE,DESCRIPTION,BASE_COST,"
+    procedures.write("START,STOP,PATIENT,ENCOUNTER,SYSTEM,CODE,DESCRIPTION,BASE_COST,"
         + "REASONCODE,REASONDESCRIPTION");
     procedures.write(NEWLINE);
     immunizations.write("DATE,PATIENT,ENCOUNTER,CODE,DESCRIPTION,BASE_COST");
@@ -752,7 +752,7 @@ private String encounter(String personID,
    * @throws IOException if any IO error occurs
    */
   private void condition(String personID, String encounterID, Entry condition) throws IOException {
-    // START,STOP,PATIENT,ENCOUNTER,CODE,DESCRIPTION
+    // START,STOP,PATIENT,ENCOUNTER,SYSTEM,CODE,DESCRIPTION
     StringBuilder s = new StringBuilder();
 
     s.append(dateFromTimestamp(condition.start)).append(',');
@@ -765,6 +765,7 @@ private void condition(String personID, String encounterID, Entry condition) thr
 
     Code coding = condition.codes.get(0);
 
+    s.append(coding.system).append(',');
     s.append(coding.code).append(',');
     s.append(clean(coding.display));
 
@@ -903,7 +904,7 @@ private void observation(String personID,
    */
   private void procedure(String personID, String encounterID,
       Procedure procedure) throws IOException {
-    // START,STOP,PATIENT,ENCOUNTER,CODE,DESCRIPTION,COST,REASONCODE,REASONDESCRIPTION
+    // START,STOP,PATIENT,ENCOUNTER,SYSTEM,CODE,DESCRIPTION,COST,REASONCODE,REASONDESCRIPTION
     StringBuilder s = new StringBuilder();
 
     s.append(iso8601Timestamp(procedure.start)).append(',');
@@ -915,6 +916,7 @@ private void procedure(String personID, String encounterID,
     s.append(encounterID).append(',');
     // CODE
     Code coding = procedure.codes.get(0);
+    s.append(coding.system).append(',');
     s.append(coding.code).append(',');
     // DESCRIPTION
     s.append(clean(coding.display)).append(',');

src/main/java/org/mitre/synthea/export/ExportHelper.java

@@ -213,6 +213,10 @@ public static long nextFriday(long time) {
   private static final String RXNORM_URI = "http://www.nlm.nih.gov/research/umls/rxnorm";
   private static final String CVX_URI = "http://hl7.org/fhir/sid/cvx";
   private static final String DICOM_DCM_URI = "http://dicom.nema.org/medical/dicom/current/output/chtml/part16/sect_CID_29.html";
+  private static final String CDT_URI = "http://www.ada.org/cdt";
+  private static final String ICD9_URI = "http://hl7.org/fhir/sid/icd-9-cm";
+  private static final String ICD10_URI = "http://hl7.org/fhir/sid/icd-10";
+  private static final String ICD10_CM_URI = "http://hl7.org/fhir/sid/icd-10-cm";
 
   /**
    * Translate the system name (e.g. SNOMED-CT) into the official
@@ -231,6 +235,14 @@ public static String getSystemURI(String system) {
       system = CVX_URI;
     } else if (system.equals("DICOM-DCM")) {
       system = DICOM_DCM_URI;
+    } else if (system.equals("CDT")) {
+      system = CDT_URI;
+    } else if (system.equals("ICD9")) {
+      system = ICD9_URI;
+    } else if (system.equals("ICD10")) {
+      system = ICD10_URI;
+    } else if (system.equals("ICD10-CM")) {
+      system = ICD10_CM_URI;
     }
     return system;
   }
@@ -253,6 +265,14 @@ public static String getSystemFromURI(String uri) {
         return "CVX";
       case DICOM_DCM_URI:
         return "DICOM-DCM";
+      case CDT_URI:
+        return "CDT";
+      case ICD9_URI:
+        return "ICD9";
+      case ICD10_URI:
+        return "ICD10";
+      case ICD10_CM_URI:
+        return "ICD10-CM";
       default:
         return "Unknown";
     }

src/main/java/org/mitre/synthea/export/Exporter.java

@@ -390,7 +390,8 @@ private static boolean exportRecord(Person person, String fileTag, long stopTime
       writeNewFile(outFilePath, consolidatedNotes);
     }
 
-    if (patientExporters != null && !patientExporters.isEmpty()) {
+    if (Config.getAsBoolean("exporter.custom.export", true)
+            && patientExporters != null && !patientExporters.isEmpty()) {
       for (PatientExporter patientExporter : patientExporters) {
         patientExporter.export(person, stopTime, options);
       }
@@ -618,7 +619,8 @@ public static void runPostCompletionExports(Generator generator, ExporterRuntime
               parser.encodeResourceToString(parameters));
     }
 
-    if (postCompletionExporters != null && !postCompletionExporters.isEmpty()) {
+    if (Config.getAsBoolean("exporter.custom.export", true)
+            && postCompletionExporters != null && !postCompletionExporters.isEmpty()) {
       for (PostCompletionExporter postCompletionExporter : postCompletionExporters) {
         postCompletionExporter.export(generator, options);
       }