doc & cleanups

.gitignore

@@ -5,6 +5,7 @@ rsrc/remnants
 *.ods#
 *__pycache__*
 site
+logs
 logs/*
 build
 bycon.egg-info

bycon/config/entity_defaults.yaml

@@ -123,9 +123,7 @@ biosample:
 individual:
   description: >-
     The type of response used for the endpoint depends on the requested and granted
-    `responseGranularity`, _i.e._ while a "record" level `responseGranularity`
-    has to be served through a `beaconResultsetsResponse` other granularities
-    might employ summary response types.
+    `responseGranularity`.
   request_entity_path_id: individuals
   response_entity_id: individual
   collection: individuals
@@ -145,8 +143,12 @@ individual:
 
 phenopacket:
   description: >-
-    The `phenopacket` entity type is represents a version of the GA4GH Phenopacket
-    schema. It is not (yet) part of the Beacon default model.
+    The type of response used for the endpoint depends on the requested and granted
+    `responseGranularity`.
+
+    In the `bycon` framework Phenopackets are generated at export time by aggregating
+    the relevant information from the matched `individual`, `biosample`s, `analysis`(/es)
+    and `genomicVariation`s.
   request_entity_path_id: phenopackets
   response_entity_id: phenopacket
   collection: individuals

bycon/schemas/models/src/bycon-model/phenopackets/defaultSchema.yaml

@@ -3,7 +3,11 @@
 title: Phenopacket
 description: >-
   The Phenopacket class is a bare-bones JSON-schema rewrite of the Phenopackets v2
-  standard.
+  standard ("PXF"), for the representation of attributes supported in the `bycon` framework.
+  At this time the Phenopackets schema is not part of the Beacon v2 default data
+  model. However, many sub-schemas in Beacon v2 have been informed by the PXF
+  model and development process, allowing a straightforward cross-mapping of the
+  data structures.
 properties:
   id:
     type: string
@@ -35,13 +39,13 @@ properties:
       $ref: ../common/file.yaml
   metaData:
     ref: ../common/metaData.yaml
-  # interpretations:
-  #   type: array
-  #   items:
-  #     $ref: Interpretation.yaml
-  # medicalActions:
-  #   type: array
-  #   items:
-  #     $ref: MedicalAction.yaml
+  interpretations:
+    type: array
+    items:
+      $ref: Interpretation.yaml
+  medicalActions:
+    type: array
+    items:
+      $ref: MedicalAction.yaml
 
 

docs/generated/beacon-responses.md

@@ -210,7 +210,7 @@ The type of response used for the endpoint depends on the requested and granted
 
 None
 
-The type of response used for the endpoint depends on the requested and granted `responseGranularity`, _i.e._ while a "record" level `responseGranularity` has to be served through a `beaconResultsetsResponse` other granularities might employ summary response types.
+The type of response used for the endpoint depends on the requested and granted `responseGranularity`.
 
 
 
@@ -226,7 +226,8 @@ The type of response used for the endpoint depends on the requested and granted
 
 The Phenopacket class is a bare-bones JSON-schema rewrite of the Phenopackets v2 standard.
 
-The `phenopacket` entity type is represents a version of the GA4GH Phenopacket schema. It is not (yet) part of the Beacon default model.
+The type of response used for the endpoint depends on the requested and granted `responseGranularity`.
+In the `bycon` framework Phenopackets are generated at export time by aggregating the relevant information from the matched `individual`, `biosample`s, `analysis`(/es) and `genomicVariation`s.
 
 
 

housekeepers/collationsFrequencymapsCreator.py

@@ -4,6 +4,7 @@
 import time
 from pymongo import MongoClient
 from progress.bar import Bar
+from random import shuffle as random_shuffle
 
 from bycon import *
 from byconServiceLibs import assertSingleDatasetOrExit, ask_limit_reset, ByconBundler, GenomeBins, set_collation_types
@@ -18,8 +19,10 @@ def main():
     binning = GB.get_genome_binning()
     print(f'=> Using data values from {ds_id} for {interval_count} intervals...')
 
-    if "skip existing" in (BYC_PARS.get("mode", "")).lower():
+    if "skip" in (BYC_PARS.get("mode", "")).lower():
         skip_existing = True
+    else:
+        skip_existing = False
 
     BYC.update({"PAGINATED_STATUS": False})
 
@@ -34,12 +37,12 @@ def main():
 
     coll_ids = _filter_coll_ids(coll_coll)    
     coll_no = len(coll_ids)
-   
+
     if not BYC["TEST_MODE"]:
         bar = Bar(f'{coll_no} {ds_id} fMaps', max = coll_no, suffix='%(percent)d%%'+f' of {coll_no}' )
 
     coll_i = 0
-    for c_id in coll_ids:
+    for c_id in random_shuffle(coll_ids):
         coll = coll_coll.find_one({"id": c_id})
         c_o_id = coll.get("_id")
         if not coll:
@@ -49,24 +52,25 @@ def main():
             continue
         coll_i += 1
 
+        prdbug(f'??? skip {c_id} with existing frequencymap')
+
         if not BYC["TEST_MODE"]:
             bar.next()
-        if skip_existing and coll_coll.get("frequencymap"):
-            continue
+        if skip_existing is True:
+            if "frequencymap" in coll:
+                prdbug(f'!!! skip {c_id} with existing frequencymap')
+                continue
 
         start_time = time.time()
 
-        # prdbug(coll)
-
         BYC.update({"BYC_FILTERS":[{"id":c_id}, {"id": "EDAM:operation_3961"}]})
         BYC.update({"FMAPS_SCOPE": coll.get("scope", "biosamples")})
 
-        prdbug(f'=> processing {c_id} with limit {BYC_PARS.get("limit")}')
+        prdbug(f'\n=> processing {c_id} with limit {BYC_PARS.get("limit")}')
         RSS = ByconResultSets().datasetsResults()
         pdb = ByconBundler().resultsets_frequencies_bundles(RSS)
         if_bundles = pdb.get("interval_frequencies_bundles")
 
-
         if len(if_bundles) < 1:
             prdbug(f'No interval_frequencies for {c_id}')
             continue