WIP: try to make a builder for ctus

gsrs-module-clinical-trials-spring-boot-autoconfigure/src/main/java/gov/hhs/gsrs/clinicaltrial/us/modelBuilders/AbstractClinicalTrialUSBuilder.java

@@ -0,0 +1,63 @@
+package gov.hhs.gsrs.clinicaltrial.us.modelBuilders;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import gov.hhs.gsrs.clinicaltrial.us.models.ClinicalTrialUS;
+import ix.core.controllers.EntityFactory;
+import ix.core.models.Group;
+import ix.core.models.Keyword;
+import ix.core.models.Principal;
+
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public abstract class AbstractClinicalTrialUSBuilder <S extends ClinicalTrialUS, T extends AbstractClinicalTrialUSBuilder<S,T>>{
+   Function<S, S> xandThen = (f->f);
+
+    public abstract Supplier<S> getSupplier();
+
+    protected abstract T getThis();
+
+    public T setTitle(String title) {
+        return andThen(s -> {
+            s.trialNumber = title;
+        });
+    }
+    public T setTrialNumber(String trialNumber) {
+        return andThen(s -> {s.trialNumber=trialNumber;});
+    }
+
+    public T andThen(Consumer<S> fun){
+        xandThen = xandThen.andThen(s ->{ fun.accept(s); return s;});
+        return getThis();
+    }
+
+    public S build(){
+        S s = getSupplier().get();
+        return additionalSetup().apply(afterCreate().apply(s));
+    }
+
+    public final Function<S, S> afterCreate(){
+        return xandThen;
+    }
+
+    public  Function<S, S> additionalSetup(){
+        return Function.identity();
+    }
+    public AbstractClinicalTrialUSBuilder(){
+    }
+
+    public AbstractClinicalTrialUSBuilder(ClinicalTrialUS copy) {
+
+        String trialNumber = copy.getTrialNumber();
+        if (trialNumber != null) {
+            setTrialNumber(trialNumber);
+        }
+
+    }
+
+
+}

gsrs-module-clinical-trials-spring-boot-autoconfigure/src/main/java/gov/hhs/gsrs/clinicaltrial/us/modelBuilders/ClinicalTrialUSBuilder.java

@@ -0,0 +1,59 @@
+package gov.hhs.gsrs.clinicaltrial.us.modelBuilders;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import gov.hhs.gsrs.clinicaltrial.us.models.ClinicalTrialUS;
+import ix.core.controllers.EntityFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.function.Supplier;
+
+public class ClinicalTrialUSBuilder extends AbstractClinicalTrialUSBuilder<ClinicalTrialUS, ClinicalTrialUSBuilder> {
+
+    private static final ObjectMapper mapper = EntityFactory.EntityMapper.FULL_ENTITY_MAPPER();
+
+    public ClinicalTrialUSBuilder() {
+    }
+
+    public ClinicalTrialUSBuilder(ClinicalTrialUS copy) {
+        super(copy);
+    }
+
+    @Override
+    public Supplier<ClinicalTrialUS> getSupplier() {
+        return ClinicalTrialUS::new;
+    }
+
+    @Override
+    protected ClinicalTrialUSBuilder getThis() {
+        return this;
+    }
+
+    protected <S extends ClinicalTrialUS> ClinicalTrialUSBuilder(AbstractClinicalTrialUSBuilder<S,?> builder){
+        this.xandThen = (s)-> (ClinicalTrialUS) builder.xandThen.apply((S) s);
+    }
+
+    public static <S extends ClinicalTrialUS, B extends AbstractClinicalTrialUSBuilder<S,B>> B  from(String json) throws IOException {
+        return from(mapper.readTree(json));
+    }
+    public static <S extends ClinicalTrialUS, B extends AbstractClinicalTrialUSBuilder<S,B>> B  from(File json) throws IOException{
+        return from(mapper.readTree(json));
+    }
+    public static <S extends ClinicalTrialUS, B extends AbstractClinicalTrialUSBuilder<S,B>> B  from(InputStream json) throws IOException{
+        return from(mapper.readTree(json));
+    }
+    public static <S extends ClinicalTrialUS, B extends AbstractClinicalTrialUSBuilder<S,B>> B  from(JsonNode json){
+        ClinicalTrialUS ctus = null;
+        try {
+            ctus = mapper.treeToValue(json, ClinicalTrialUS.class);
+            return (B) new ClinicalTrialUSBuilder(ctus);
+
+        } catch (JsonProcessingException e) {
+            throw new IllegalStateException("JSON parse error:" + e.getMessage(), e);
+        }
+    }
+
+}

gsrs-module-clinical-trials-spring-boot-autoconfigure/src/main/java/gov/hhs/gsrs/clinicaltrial/us/models/ClinicalTrialUS.java

@@ -1,10 +1,12 @@
 package gov.hhs.gsrs.clinicaltrial.us.models;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import gsrs.security.GsrsSecurityUtils;
 import ix.core.models.*;
 import gov.hhs.gsrs.clinicaltrial.base.models.ClinicalTrialBase;
+import ix.core.util.EntityUtils;
 import ix.ginas.models.serialization.GsrsDateDeserializer;
 import ix.ginas.models.serialization.GsrsDateSerializer;
 import lombok.*;
@@ -262,5 +264,13 @@ public String toString() {
         rtsb.setExcludeNullValues(true);
         return rtsb.toString();
     }
+    public JsonNode toInternalJsonNode(){
+        return EntityUtils.EntityWrapper.of(this).toInternalJsonNode();
+    }
+
+    @Override
+    protected Object clone() throws CloneNotSupportedException {
+        return super.clone();
+    }
 
 }

gsrs-module-clinical-trials-spring-boot-autoconfigure/src/main/java/gov/hhs/gsrs/clinicaltrial/us/scrubbers/BasicCtusScrubber.java

@@ -0,0 +1,101 @@
+package gov.hhs.gsrs.clinicaltrial.us.scrubbers;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
+import com.jayway.jsonpath.Predicate;
+
+import gov.hhs.gsrs.clinicaltrial.ClinicalTrialDataSourceConfig;
+import gov.hhs.gsrs.clinicaltrial.us.models.ClinicalTrialUS;
+import gov.nih.ncats.common.stream.StreamUtil;
+import ix.core.EntityFetcher;
+import ix.core.models.Group;
+import ix.core.util.EntityUtils;
+import ix.ginas.exporters.RecordScrubber;
+// import ix.ginas.modelBuilders.SubstanceBuilder;
+import ix.ginas.models.GinasAccessControlled;
+// import ix.ginas.models.GinasAccessReferenceControlled;
+import ix.ginas.models.GinasCommonData;
+;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import net.minidev.json.JSONArray;
+
+/*
+Note: as of 22 September, most of this class is commented out and a quick and dirty implementation is in place.
+This will change in the ensuing weeks
+ */
+@Slf4j
+public class BasicCtusScrubber implements RecordScrubber<ClinicalTrialUS>{
+
+
+private static final String TO_DELETE = "TO_DELETE";
+private static final String TO_FORCE_KEEP = "TO_FORCE_KEEP";
+private BasicCtusScrubberParameters scrubberSettings;
+private static Set<Group> toDelete = new HashSet<>();
+private static Set<Group> toKeep = new HashSet<>();
+
+
+
+public BasicCtusScrubber(BasicCtusScrubberParameters scrubberSettings){
+this.scrubberSettings = scrubberSettings;
+}
+
+//public void setResolver( ClinicalTrialUSReferenceResolver newResolver) {
+//this.resolver= newResolver;
+//}
+
+
+    @SneakyThrows
+    @Override
+    public Optional<ClinicalTrialUS> scrub(ClinicalTrialUS ctus) {
+        log.trace("starting in scrub");
+        String ctusJson;
+        try {
+            ctusJson = ctus.toInternalJsonNode().toString();
+        } catch (Exception ex){
+            log.error("Error retrieving substance; using alternative method");
+            EntityUtils.Key trialNumber = EntityUtils.Key.of(ClinicalTrialUS.class, ctus.trialNumber);
+            Optional<ClinicalTrialUS> ctusRefetch = EntityFetcher.of(trialNumber).getIfPossible().map(o->(ClinicalTrialUS) o);
+            ctusJson = ctusRefetch.get().toInternalJsonNode().toString();
+        }
+
+//        try {
+//            //TODO: confirm if this forces as a concept. It should not,
+//            // but we need to check.
+////            ClinicalTrialUS ctusNew = ClinicalTrialUSBuilder.from(ctusJson).build();
+//  //          return Optional.of(ctusNew);
+//        }
+//        catch (Exception ex) {
+//            log.warn("error processing record; Will return empty", ex);
+//        }
+        return Optional.empty();
+    }
+
+
+
+
+
+
+
+
+
+
+}

gsrs-module-clinical-trials-spring-boot-autoconfigure/src/main/java/gov/hhs/gsrs/clinicaltrial/us/scrubbers/BasicCtusScrubberFactory.java

@@ -0,0 +1,74 @@
+package gov.hhs.gsrs.clinicaltrial.us.scrubbers;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import gov.nih.ncats.common.util.CachedSupplier;
+import gsrs.springUtils.AutowireHelper;
+import ix.ginas.exporters.RecordScrubber;
+import ix.ginas.exporters.RecordScrubberFactory;
+import gov.hhs.gsrs.clinicaltrial.us.models.ClinicalTrialUS;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.util.FileCopyUtils;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Optional;
+
+@Slf4j
+public class BasicCtusScrubberFactory implements RecordScrubberFactory<ClinicalTrialUS> {
+private final static String JSONSchema = getSchemaString();
+
+private static CachedSupplier<JsonNode> schemaSupplier = CachedSupplier.of(()->{
+ObjectMapper mapper =new ObjectMapper();
+try {
+JsonNode schemaNode=mapper.readTree(JSONSchema);
+return schemaNode;
+} catch (JsonProcessingException e) {
+e.printStackTrace();
+}
+return null;//todo: alternate return?
+});
+
+@Override
+public RecordScrubber<ClinicalTrialUS> createScrubber(JsonNode settings) {
+log.trace("in BasicCtusScrubberFactory.createScrubber");
+BasicCtusScrubberParameters settingsObject = (new ObjectMapper()).convertValue(settings, BasicCtusScrubberParameters.class);
+log.trace(" settingsObject: {}", (settingsObject==null || settings.size()==0 ? "null" : "not null"));
+
+if(settingsObject==null){
+log.warn("no settings supplied to createScrubber");
+RecordScrubber<ClinicalTrialUS> identityScrubber = (t)-> Optional.of(t);
+return identityScrubber;
+}
+
+BasicCtusScrubber scrubber = new BasicCtusScrubber(settingsObject);
+scrubber= AutowireHelper.getInstance().autowireAndProxy(scrubber);
+
+return scrubber;
+}
+
+@Override
+public JsonNode getSettingsSchema() {
+return schemaSupplier.get();
+}
+
+private  static String readFileAsString(File textFile)throws Exception
+{
+String data = "";
+data = new String(Files.readAllBytes(textFile.toPath()));
+return data;
+}
+
+@SneakyThrows
+private static String getSchemaString() {
+log.trace("starting getSchemaString");
+ClassPathResource fileResource = new ClassPathResource("schemas/scrubberSchema.json");
+byte[] binaryData = FileCopyUtils.copyToByteArray(fileResource.getInputStream());
+String schemaString =new String(binaryData, StandardCharsets.UTF_8);
+//log.trace("read schema:{}", schemaString);
+return schemaString;
+}
+}