Merge pull request #159 from formal-methods-mpi/filters_rework

Filter for RecordDataBase

Author: nickhaf

src/Taxonomy.jl

@@ -17,21 +17,26 @@ module Taxonomy
 
     export DOI, UsualDOI, UnusualDOI
     include("metadata/doi.jl")
+
+    export Study
+    include("study.jl")
+
+    export Record, id, judgements,  location, spec, data, ExtractStudy
+    include("record.jl")
+
+    export RecordDatabase, check_uuid, check_url
+    import Base: UUID
+    include("database.jl")
 
     import Base.convert, Base.==
     export Judgements, J, Judgement, NoJudgement, convert, rating, comment, certainty
+    export Empirical, Lang, N, Standardized, Quest
     export JudgementNumber, JudgementBool, JudgementInt, JudgementString
     export JudgementVecNumber, JudgementVecBool, JudgementVecInt, JudgementVecString
     include("judgements/Judgements.jl")
 
     using Taxonomy.Judgements
 
-    export Study
-    include("study.jl")
-
-    export Record, id, judgements,  location, spec, data, ExtractStudy
-    include("record.jl")
-
     import HTTP
     import JSON
     export MetaData, MinimalMeta, IncompleteMeta, ExtensiveMeta, url, year, author, journal, apa, json
@@ -57,17 +62,15 @@ module Taxonomy
     export generate_id
     include("uuid.jl")
 
-    export RecordDatabase, check_uuid, check_url
-    import Base: UUID
-    include("database.jl")
+    export Model
+    include("model.jl")
 
-    export factor_variance, structural_model, doi
+    export factor_variance, structural_model, doi, measurement_model
     include("extractors.jl")
 
-    include("pretty_printing.jl")
+    include("filter.jl")
 
-    export Model
-    include("model.jl")
+    include("pretty_printing.jl")
 
     include("deprecated.jl")
 end

src/extractors.jl

@@ -17,10 +17,7 @@ rating(factor_variance(f))
 1.0
 ```
 """
-function factor_variance(x::Measurement)
-    x.factor_variance
-end
-
+factor_variance(x::Measurement) = x.factor_variance
 
 """
 Function to extract the `StenoGraphs` structural model from [`Structural`].
@@ -47,11 +44,7 @@ struct_model = Structural(structural_model = graph)
 structural_model(struct_model)
 ```
 """
-function structural_model(x::Structural)
-    x.structural_model
-end
-
-
+structural_model(x::Structural) = x.structural_model
 
 """
 Function to extract a Judgement from a `JudgementLevel` (e.g. [Model](@ref), [Record](@ref), [Study](@ref)).
@@ -81,4 +74,32 @@ Dict{Symbol, Vector{Union{Study, AbstractJudgement}}} with 1 entry:
 """
 judgements(x::JudgementLevel) = x.judgements
 url(x::Record) = url(location(x))
-url(x::RecordDatabase) = map(x -> url(x.second), collect(x)) 
+url(x::RecordDatabase) = map(x -> url(x.second), collect(x))
+
+Base.get(r::JudgementLevel, field::Symbol, default=Vector{Union{JudgementLevel,AbstractJudgement}}[]) = get(judgements(r), field, default)
+
+"""
+Extract all `Studies` from a `Record`. 
+"""
+Study(r::Record)::Vector{Union{Study,AbstractJudgement}} = get(judgements(r), :Study, [])
+Study(r::Dict{Symbol,Vector{Union{Study,AbstractJudgement}}})::Vector{Union{Study,AbstractJudgement}} = get(r, :Study, [])
+
+"""
+Extract all `Models` from a `Study`. 
+"""
+Model(s::Study)::Vector{Union{Model,AbstractJudgement}} = get(judgements(s), :Model, [])
+
+"""
+Extract all Taxons from a Model.
+"""
+Taxon(m::Model)::Vector{Union{Taxon,AbstractJudgement}} = get(judgements(m), :Taxon, [])
+
+"""
+Extract Measurement part from a Taxon. 
+"""
+measurement_model(t::Taxon) = t.measurement_model
+
+"""
+Extract Structural part from a Taxon. 
+"""
+structural_model(t::Taxon) = t.structural_model

src/filter.jl

@@ -0,0 +1,145 @@
+function Base.filter(f, s::Vector{Union{T,AbstractJudgement}} where {T<:JudgementLevel})
+    res_vec = eltype(s)[]
+    for i in eachindex(s)
+        if f(s[i])
+            res_vec = push!(res_vec, s[i])
+        end
+    end
+
+    return res_vec
+end
+Base.filter(f, s::JudgementLevel)::Vector{Union{JudgementLevel,AbstractJudgement}} = f(s) ? [s] : []
+
+"""
+    filter(f, db::RecordDatabase, level::AbstractString)::RecordDatabase
+
+Filter the database at a given nesting level. 
+
+# Arguments
+- `f`: A function returning a boolean value for filtering a dict at the specified level. 
+- `db`: A RecordDatabase object.
+- `level`: The character string of the level at which to filter the database. Possible values are `"Record"`, `"Study"`, `"Model"`, `"Taxon"`.
+
+# Examples
+```jldoctest filter-examples
+
+## Example just for demonstration, coding is not actually derived from the paper!
+
+using Taxonomy
+using Taxonomy.Judgements
+
+test_db = RecordDatabase()
+
+test_db += Record(
+    rater="NH",
+    id="2a129694-550c-4396-be6f-00507b1dc7ba",
+    location=DOI("10.1007/s10869-019-09648-5"),
+    Lang("en"),
+    Study(
+        N(100, 0.8),
+        Model(Standardized(false),
+            LatentPathmodel(
+                Structural(structural_model=missing),
+                Dict(
+                    :IP => Measurement(n_variables=3, factor_variance=missing, loadings=missing, quest_scale=7),
+                    :IN => Measurement(n_variables=4, factor_variance=missing, loadings=missing, quest_scale=6),
+                    :DN => Measurement(n_variables=3, factor_variance=missing, loadings=missing, quest_scale=7),
+                    :BC => Measurement(n_variables=3, factor_variance=missing, loadings=missing, quest_scale=5),
+                    :IB => Measurement(n_variables=5, factor_variance=missing, loadings=missing, quest_scale=5)
+                )
+            )
+        ),
+        Model(Standardized(true),
+            NoTaxon()
+        )
+    ),
+    Study(
+        N(200, 0.98),
+        Model(Standardized(false),
+        ),
+        Model(Standardized(false))
+    )
+)
+
+## Filtering on Taxon level
+filter_Pathmodel = filter(x -> typeof(x) == NoTaxonEver, test_db, "Taxon")
+Model(Study(filter_Pathmodel[Base.UUID("2a129694-550c-4396-be6f-00507b1dc7ba")])[1])[2]
+
+# output
+Model(Dict{Symbol, Vector{Union{Taxon, AbstractJudgement}}}(:Taxon => [NoTaxonEver
+], :Standardized => [Standardized{Bool}(true, 1.0, missing)]))
+```
+
+```jldoctest filter-examples
+
+## Filtering on Model level
+filter_Standardized = filter(x -> rating(x, :Standardized) == true, test_db, "Model")
+Model(Study(filter_Standardized[Base.UUID("2a129694-550c-4396-be6f-00507b1dc7ba")])[1])
+
+# output
+1-element Vector{Union{Model, AbstractJudgement}}:
+ Model(Dict{Symbol, Vector{Union{Taxon, AbstractJudgement}}}(:Taxon => [NoTaxonEver
+], :Standardized => [Standardized{Bool}(true, 1.0, missing)]))
+
+```
+
+```jldoctest filter-examples
+
+## Filtering on Study level
+filter_N = filter(x -> certainty(x, :N) > 0.9, test_db, "Study")
+Study(filter_N[Base.UUID("2a129694-550c-4396-be6f-00507b1dc7ba")])
+
+# output
+1-element Vector{Union{Study, AbstractJudgement}}:
+ Study(Dict{Symbol, Vector{Union{JudgementLevel, AbstractJudgement}}}(:N => [N{Int64}(200, 0.98, missing)]))
+```
+"""
+function Base.filter(f, db::RecordDatabase, level::AbstractString)::RecordDatabase
+
+    @assert level in ["Record", "Study", "Model", "Taxon"] "Invalid level. Choose from 'Record', 'Study', 'Model', 'Taxon'"
+
+    if level == "Record"
+        db = filter(f, db)
+    else
+
+        for current_record in keys(db)
+            studies = Study(db[current_record])
+
+            if level == "Study"
+                filtered_study = filter(f, studies)
+                if length(filtered_study) > 0
+                    judgements(db[current_record])[:Study] = filtered_study
+                else
+                    delete!(judgements(db[current_record]), :Study)
+                end
+            else
+
+                for current_study in eachindex(studies)
+                    models = Model(studies[current_study])
+                    if level == "Model"
+                        filtered_model = filter(f, models)
+                        if length(filtered_model) > 0
+                            judgements(Study(db[current_record])[current_study])[:Model] = filtered_model
+                        else
+                            delete!(judgements(judgements(db[current_record])[:Study][current_study]), :Model)
+                        end
+                    else
+                        for current_model in eachindex(models)
+                            taxons = Taxon(models[current_model])
+                            if level == "Taxon"
+                                filtered_taxons = filter(f, taxons)
+
+                                if length(filtered_taxons) > 0
+                                    judgements(Model(Study(db[current_record])[current_study])[current_model])[:Taxon] = filtered_taxons
+                                else
+                                    delete!(judgements(Model(Study(db[current_record])[current_study])[current_model]), :Taxon)
+                                end
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+    return db
+end

src/judgements/Judgements.jl

@@ -1,12 +1,12 @@
 module Judgements
-    import Taxonomy: AbstractJudgement, Taxon, JudgementLevel
+    import Taxonomy: AbstractJudgement, Taxon, JudgementLevel, Record, Base.==
     check_certainty(c) = ((c < 0.0) || (c > 1.0)) ? throw(ArgumentError("Certainty must be between 0 and 1.")) : nothing
     export AnyLevelJudgement, RecordJudgement, StudyJudgement, ModelJudgement
     export check_judgement_level, correct_judgement_level, judgements
     include("level.jl")
 
     export @newjudgement
-    export J, Judgement, NoJudgement, convert, rating, comment, certainty, judgement_key
+    export J, Judgement, NoJudgement, convert, rating, comment, certainty, judgement_key, judgement_level
     include("judgement.jl")
 
     # exports via code gen within the file
@@ -15,6 +15,6 @@ module Judgements
     export judgement_dict
     include("dict.jl")
 
-    export CFI, Empirical, Lang, N
+    export CFI, Empirical, Lang, N, Standardized, Quest
     include("predefined_judgements.jl")
 end

src/judgements/judgement.jl

@@ -63,19 +63,19 @@ macro newjudgement(name, level, doc, type=Any, check=x -> nothing, unique=true)
     unique = :(Judgements.judgement_unique(::$name) = $unique)
     key = :(Judgements.judgement_key(::$name) = Symbol($name))
     level = :(Judgements.judgement_level(::$name) = $level())
-   
+
     # Create a new function with the same name as the judgement
     # This function accepts keyword arguments and creates a NamedTuple
     # The NamedTuple is then passed to the judgement constructor
     keyword_func = quote
-        function $name(;kwargs...)
+        function $name(; kwargs...)
             kwargs_tuple = NamedTuple{tuple(keys(kwargs)...)}(tuple(values(kwargs)...))
             $name(kwargs_tuple)
         end
     end
-        
 
-    
+
+
     return quote
         $(esc(inner))
         $(esc(outer))
@@ -86,7 +86,7 @@ macro newjudgement(name, level, doc, type=Any, check=x -> nothing, unique=true)
         $(esc(doc))
         $(esc(keyword_func))
     end
-    end
+end
 
 @newjudgement(
     Judgement,
@@ -109,16 +109,43 @@ macro newjudgement(name, level, doc, type=Any, check=x -> nothing, unique=true)
 """
 Extract rating from Judgement.
 
-If `rating` is called on a `Judgement` it returns the rating, on everything it returns identity.
+If `rating` is called on a `Judgement` it returns the rating, on everything it returns identity. 
+If `rating` is called on a `JudgementLevel` together with a field name, it returns the rating of that field. 
 
 """
 rating(x) = x
 rating(x::AbstractJudgement) = x.rating
+rating(x::JudgementLevel, field::Symbol) = rating(get(x, field))
+rating(x::Pair{Base.UUID, Record}, field::Symbol) = rating(x.second, field)
+function rating(x::Vector{Union{T,AbstractJudgement}}) where T <: JudgementLevel
+    if length(x) == 1
+        return rating(x[1])
+    elseif length(x) == 0
+        return x
+    else
+        error("It seems like you have provided a vector that contains multiple judgements. Only the rating of single judgement can be extracted.")
+    end
+end
 
 """
 Extract certainty from Judgement.
+
+If `certainty` is called on a `Judgement` it returns the certainty, on everything it returns identity. 
+If `certainty` is called on a `JudgementLevel` together with a field name, it returns the certainty of that field. 
+
 """
 certainty(x::AbstractJudgement) = x.certainty
+certainty(x::Pair{Base.UUID, Record}, field::Symbol) = certainty(x.second, field)
+certainty(x::JudgementLevel, field::Symbol) = certainty(get(x, field))
+function certainty(x::Vector{Union{T,AbstractJudgement}}) where T <: JudgementLevel
+    if length(x) == 1
+        return certainty(x[1])
+    elseif length(x) == 0
+        return x
+    else
+        error("It seems like you have provided a vector that contains multiple judgements. Only the certainty of single judgement can be extracted.")
+    end
+end
 
 """
 Extract comment from Judgement.

src/judgements/predefined_judgements.jl

@@ -60,4 +60,34 @@ Judgement for N.
     Int # Input type. In this case Integer. 
 )
 
+@newjudgement(
+    Standardized,
+    ModelJudgement,  
+    """
+    Any part of the data or model is standardized
+
+    Procedure:
+    * Search for standard*
+    * If present, give True
+    """,
+    Bool # Input type. In this case boolean.
+)
 
+@newjudgement(
+    Quest,
+    ModelJudgement,
+    """
+    What questionnaire was used to measure the latent variable.
+    
+    Procedure:
+    * Look for first instance of questionnaire, 
+    * acronym or name is fine, copy and paste citation if availible
+    * Do not bother whether the scale is translated, modified or shortened.
+    * spend little time (<30s) on searching for it
+    * should be given multiple times for all quests present in a model.
+
+    """,
+    AbstractString,
+    x -> nothing,
+    unique = false
+)