Skip to content

Commit

Permalink
Harmonize do-load-file across ftr / zen-lang/fhir
Browse files Browse the repository at this point in the history 
Co-authored-by: KGOH <[email protected]>
  • Loading branch information
@ApricotLace @KGOH
ApricotLace and KGOH committed Dec 22, 2022
1 parent cda328b commit fc0f154
Showing 1 changed file with 26 additions and 222 deletions.
248 changes: 26 additions & 222 deletions src/zen/fhir/loader.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,19 +539,13 @@
:zen.fhir/schema-ns (symbol (str (name package-ns) \. (:id res)))}))))))


(defmulti process-on-load
(fn [res] (keyword (:resourceType res))))
(def process-on-load ftr.extraction.ig.core/process-on-load)


(defmethod process-on-load :SearchParameter [res]
(defmethod ftr.extraction.ig.core/process-on-load :SearchParameter [res]
(search-parameter.loader/process-on-load res))


(defmethod process-on-load :default
[res]
#_(println :WARN :no-process-on-load :for (:resourceType res)))


(defn build-designation [ds]
(reduce (fn [acc d]
(assoc-in acc [(or (get-in d [:use :code]) "display")
Expand All @@ -573,164 +567,22 @@
{} ps))


(defn reduce-concept [acc id-fn sys parents c]
(let [con (-> c
(select-keys [:code :display :definition])
(assoc :id (id-fn c)
:system sys
:_source "zen.fhir"
:resourceType "Concept")
(cond-> (:designation c) (assoc :designation (build-designation (:designation c)))
(seq parents) (assoc :hierarchy parents)
(:property c) (assoc :property (build-property (:property c)))))
acc (conj acc con)]
(if-let [cs (:concept c)]
(reduce (fn [acc c']
(reduce-concept acc id-fn sys (conj parents (:code con)) c'))
acc cs)
acc)))


(defn extract-concepts [inter-part id-fn sys concept-parts]
(->> concept-parts
(reduce (fn [acc c] (reduce-concept acc id-fn sys [] c))
[])
(map (fn [concept]
(-> concept
(merge inter-part)
(assoc :zen.fhir/resource concept))))))


(def loader-keys ftr.extraction.ig.core/loader-keys)
(def reduce-concept ftr.extraction.ig.core/reduce-concept)


(defmethod process-on-load :ValueSet [res] (ftr.extraction.ig.core/process-on-load res))
(def extract-concepts ftr.extraction.ig.core/extract-concepts)


(defmethod process-on-load :CodeSystem [res] (ftr.extraction.ig.core/process-on-load res))
(def loader-keys ftr.extraction.ig.core/loader-keys)


(defmethod process-on-load :StructureDefinition
(defmethod ftr.extraction.ig.core/process-on-load :StructureDefinition
[res]
(load-intermidiate res))


#_"NOTE: We know that hl7.fhir.r4.core ValueSet resources are clashing by url with hl7.terminology.r4 ValueSets resources.
Currently we always choose hl7.terminology.r4 ValueSets over those from hl7.fhir.r4.core.
We assume that there are no other ValueSets clashing.
If we find such a clash, we throw an exception to make it noticeable."
(def package-priority
{:hl7.terminology.r4 #{:hl7.fhir.r4.core :hl7.fhir.us.carin-bb :hl7.fhir.us.davinci-pdex}})


(defn clash-ex-data [code old new]
{:code code
:old {:rt (:resourceType old)
:url (:url old)
:package (get-in old [:zen/loader :package :name])
:file (get-in old [:zen/loader :file])}
:new {:rt (:resourceType new)
:url (:url new)
:package (get-in new [:zen/loader :package :name])
:file (get-in new [:zen/loader :file])}})


(defn resolve-clash-dispatch [rt old new]
(keyword rt))


(defmulti resolve-clash #'resolve-clash-dispatch)


(defmethod resolve-clash :default [_rt old new]
(throw (Exception. (str (clash-ex-data ::no-resolve-clash-rules-defined old new)))))


(defn get-package-priority [rt priority-map old new]
(let [old-package (keyword (get-in old [:zen/loader :package :name]))
new-package (keyword (get-in new [:zen/loader :package :name]))]
{:old (get-in priority-map [old-package new-package])
:new (get-in priority-map [new-package old-package])}))


(defn decide-on-clash [compare-key old new]
(let [compare-res (compare (compare-key old)
(compare-key new))]
(cond
(neg? compare-res) ::override-with-higher-priority
(zero? compare-res) ::unresolved-clash
(pos? compare-res) ::skip-lower-priority)))


(defmethod resolve-clash :CodeSystem [rt old new]
(let [status-weight {:active 0
:draft -10
:unknown -20
:retired -30}
content-weight {:complete 0
:fragment -10
:supplement -20
:example -30
:not-present -40}

{old-priority :old, new-priority :new}
(get-package-priority rt package-priority old new)

result (decide-on-clash (juxt #(get status-weight (keyword (:status %)))
#(get content-weight (keyword (:content %)))
:priority)
(assoc old :priority old-priority)
(assoc new :priority new-priority))]
(if (and (= ::unresolved-clash result)
(= "not-present" (:content new) (:content old)))
(decide-on-clash (juxt :date :version #_#(get-in % [:zen/loader :package :name]))
old
new)
result)))


(defmethod resolve-clash :ValueSet [rt old new]
(let [{old-priority :old, new-priority :new} (get-package-priority rt package-priority old new)]
(decide-on-clash (juxt :priority)
(assoc old :priority old-priority)
(assoc new :priority new-priority))))


(defn check-priority [inter-old inter-new]
(let [rt-kw (keyword (:resourceType inter-old))]
(if (nil? inter-old)
::no-clash
(resolve-clash rt-kw inter-old inter-new))))


(defn ensure-no-clash [old new]
(case (check-priority old new)
(::no-clash ::override-with-higher-priority)
new

::skip-lower-priority
old

::unresolved-clash
(throw (Exception. (str (clash-ex-data ::unresolved-clash old new))))))


;; TODO filter by resource type
(defn load-definiton [ztx {:as opts, :keys [skip-concept-processing]} res]
(let [rt (:resourceType res)
url (or (:url res) (:url opts))]
(if (or (nil? url) (nil? rt))
(println :skip-resource "no url or rt" (get-in res [:zen/loader :file]))
(when-let [processed-res (process-on-load res)]
(let [processed-res (cond-> processed-res
(and (comp #{"CodeSystem" "ValueSet"} :resourceType)
skip-concept-processing)
(assoc :fhir/concepts '()))
processed-res (merge processed-res
{:_source "zen.fhir"
:zen.fhir/version (:zen.fhir/version @ztx)}
(select-keys res (conj loader-keys :_source)))]
(swap! ztx update-in [:fhir/inter rt url] ensure-no-clash processed-res))))))
(def load-definiton ftr.extraction.ig.core/load-definition)
(def load-definition ftr.extraction.ig.core/load-definition)


(def read-json ftr.extraction.ig.core/read-json)
Expand All @@ -741,13 +593,15 @@
(def process-concepts ftr.extraction.ig.core/process-concepts)


(defn process-resources
"this is processing of resources with context"
[ztx & [{:as _params, :keys [skip-concept-processing]}]]
(structure-definition.loader/process-structure-definitions ztx)
(search-parameter.loader/process-search-parameters ztx)
(when-not skip-concept-processing
(ftr.extraction.ig.core/process-concepts ztx)))
(def process-resources ftr.extraction.ig.core/process-resources)


(defmethod ftr.extraction.ig.core/process-resource-type :StructureDefinition [_ ztx & [_params]]
(structure-definition.loader/process-structure-definitions ztx))


(defmethod ftr.extraction.ig.core/process-resource-type :SearchParameter [_ ztx & [_params]]
(search-parameter.loader/process-search-parameters ztx))


(defn dir? [^java.io.File file]
Expand All @@ -765,32 +619,8 @@
(def blacklisted-package? ftr.extraction.ig.core/blacklisted-package?)


(defn do-load-file [ztx {:as opts :keys [whitelist blacklist params]} package f]
(let [file-name (.getName f)
content (cond
(str/ends-with? file-name ".json")
(try (cheshire.core/parse-string (str/replace (slurp f) \ufeff \space) keyword)
(catch Exception e
(println :WARN :invalid-json (.getName f) (.getMessage e))))

(str/ends-with? file-name ".edn")
(edamame/parse-string (slurp f)))
rt-whitelist (get whitelist (:resourceType content))
rt-blacklist (get blacklist (:resourceType content))]
(when (and (not (blacklisted-package? package))
content
(or (nil? rt-blacklist)
(not (contains? rt-blacklist (:url content))))
(or (nil? rt-whitelist)
(contains? rt-whitelist (:url content))))
(load-definiton ztx opts (assoc content
:_source "zen.fhir"
:zen.fhir/version (:zen.fhir/version @ztx)
:zen/loader {:package package :file (.getPath f)}
:zen.fhir/package package
:zen.fhir/file (.getPath f)
:zen.fhir/package-ns (or (:zen.fhir/package-ns params)
(some-> package :name (str/replace #"\." "-") symbol)))))))
(def do-load-file ftr.extraction.ig.core/do-load-file)


(comment
(def b (init-ztx))
Expand All @@ -804,36 +634,10 @@

)

(defn init-ztx
([]
(init-ztx (zen.core/new-context)))

([ztx]
(swap! ztx assoc :zen.fhir/version (slurp (io/resource "zen-fhir-version")))
ztx))

(defn preload-all [ztx & [{:keys [params node-modules-folder whitelist blacklist skip-concept-processing]
:or {node-modules-folder "node_modules"}}]]
(init-ztx ztx)
(doseq [pkg-dir (ftr.extraction.ig.core/find-packages node-modules-folder)]
(let [package (ftr.extraction.ig.core/read-json (str (.getPath pkg-dir) "/package.json"))
package-params (get params (:name package))]
(assert package (str "No package for " pkg-dir))
(doseq [f (.listFiles pkg-dir)]
(do-load-file ztx
{:params package-params
:skip-concept-processing skip-concept-processing
:whitelist whitelist
:blacklist (merge-with merge
{"StructureDefinition" #{"http://hl7.org/fhir/StructureDefinition/familymemberhistory-genetic"
"http://hl7.org/fhir/uv/sdc/StructureDefinition/parameters-questionnaireresponse-extract-in"}
"SearchParameter" #{"http://hl7.org/fhir/SearchParameter/example"}}
blacklist)}
package
f)))))


(defn load-all [ztx _ & [params]]
(preload-all ztx params)
(process-resources ztx params)
:done)
(def init-ztx ftr.extraction.ig.core/init-ztx)


(def preload-all ftr.extraction.ig.core/load-all)


(def load-all ftr.extraction.ig.core/load-all)

0 comments on commit fc0f154

Please sign in to comment.