Externalizing update PET json's

pypet2bids/pypet2bids/ecat.py

@@ -12,24 +12,28 @@
 import os
 import json
 import pathlib
+import pandas as pd
 
 try:
     import helper_functions
     import sidecar
     import read_ecat
     import ecat2nii
     import dcm2niix4pet
+    from update_json_pet_file import get_metadata_from_spreadsheet, check_meta_radio_inputs
 except ModuleNotFoundError:
     import pypet2bids.helper_functions as helper_functions
     import pypet2bids.sidecar as sidecar
     import pypet2bids.read_ecat as read_ecat
     import pypet2bids.ecat2nii as ecat2nii
     import pypet2bids.dcm2niix4pet as dcm2niix4pet
+    from pypet2bids.update_json_pet_file import get_metadata_from_spreadsheet, check_meta_radio_inputs
 
 from dateutil import parser
 
 logger = helper_functions.logger('pypet2bids')
 
+
 def parse_this_date(date_like_object) -> str:
     """
     Uses the `dateutil.parser` module to extract a date from a variety of differently formatted date strings
@@ -51,7 +55,8 @@ class Ecat:
     viewing in stdout. Additionally, this class can be used to convert an ECAT7.X image into a nifti image.
     """
 
-    def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_data=True):
+    def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_data=True, metadata_path=None,
+                 kwargs={}):
         """
         Initialization of this class requires only a path to an ecat file.
 
@@ -69,9 +74,24 @@ def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_da
         self.decay_factors = []  # stored here
         self.sidecar_template = sidecar.sidecar_template_full  # bids approved sidecar file with ALL bids fields
         self.sidecar_template_short = sidecar.sidecar_template_short  # bids approved sidecar with only required bids fields
+        self.sidecar_path = None
         self.directory_table = None
+        self.spreadsheet_metadata = {'nifti_json': {}, 'blood_tsv': {}, 'blood_json': {}}
+        self.kwargs = kwargs
+        self.output_path = None
+        self.metadata_path = metadata_path
+
+        # load config file
+        default_json_path = helper_functions.check_pet2bids_config('DEFAULT_METADATA_JSON')
+        if default_json_path and pathlib.Path(default_json_path).exists():
+            with open(default_json_path, 'r') as json_file:
+                try:
+                    self.spreadsheet_metadata.update(json.load(json_file))
+                except json.decoder.JSONDecodeError:
+                    logger.warning(f"Unable to load default metadata json file at {default_json_path}, skipping.")
+
         if os.path.isfile(ecat_file):
-            self.ecat_file = ecat_file
+            self.ecat_file = str(ecat_file)
         else:
             raise FileNotFoundError(ecat_file)
 
@@ -114,6 +134,25 @@ def __init__(self, ecat_file, nifti_file=None, decompress=True, collect_pixel_da
         else:
             self.nifti_file = nifti_file
 
+        # que up metadata path for spreadsheet loading later
+        if self.metadata_path:
+            if pathlib.Path(metadata_path).is_file() and pathlib.Path(metadata_path).exists():
+                self.metadata_path = metadata_path
+        elif metadata_path == '':
+            self.metadata_path = pathlib.Path(self.ecat_file).parent
+        else:
+            self.metadata_path = None
+
+        if self.metadata_path:
+            load_spreadsheet_data = get_metadata_from_spreadsheet(metadata_path=self.metadata_path,
+                                                                  image_folder=pathlib.Path(self.ecat_file).parent,
+                                                                  image_header_dict={})
+
+            self.spreadsheet_metadata['nifti_json'].update(load_spreadsheet_data['nifti_json'])
+            self.spreadsheet_metadata['blood_tsv'].update(load_spreadsheet_data['blood_tsv'])
+            self.spreadsheet_metadata['blood_json'].update(load_spreadsheet_data['blood_json'])
+
+
     def make_nifti(self, output_path=None):
         """
         Outputs a nifti from the read in ECAT file.
@@ -261,25 +300,28 @@ def populate_sidecar(self, **kwargs):
         self.sidecar_template['ConversionSoftware'] = 'pypet2bids'
         self.sidecar_template['ConversionSoftwareVersion'] = helper_functions.get_version()
 
-
+        # update sidecar values from spreadsheet
+        if self.spreadsheet_metadata.get('nifti_json', None):
+            self.sidecar_template.update(self.spreadsheet_metadata['nifti_json'])
 
         # include any additional values
         if kwargs:
             self.sidecar_template.update(**kwargs)
 
-        if not self.sidecar_template.get('TimeZero', None):
-            if not self.sidecar_template.get('AcquisitionTime', None):
+        if not self.sidecar_template.get('TimeZero', None) and not kwargs.get('TimeZero', None):
+            if not self.sidecar_template.get('AcquisitionTime', None) and not kwargs.get('TimeZero', None):
                 logger.warn(f"Unable to determine TimeZero for {self.ecat_file}, you need will need to provide this"
-                      f" for a valid BIDS sidecar.")
+                            f" for a valid BIDS sidecar.")
             else:
                 self.sidecar_template['TimeZero'] = self.sidecar_template['AcquisitionTime']
 
+        # clear any nulls from json sidecar and replace with none's
+        self.sidecar_template = helper_functions.replace_nones(self.sidecar_template)
+
         # lastly infer radio data if we have it
-        meta_radio_inputs = dcm2niix4pet.check_meta_radio_inputs(self.sidecar_template)
+        meta_radio_inputs = check_meta_radio_inputs(self.sidecar_template)
         self.sidecar_template.update(**meta_radio_inputs)
 
-        # clear any nulls from json sidecar and replace with none's
-        self.sidecar_template = helper_functions.replace_nones(self.sidecar_template)
 
     def prune_sidecar(self):
         """
@@ -336,6 +378,89 @@ def show_sidecar(self, output_path=None):
         else:
             print(json.dumps(helper_functions.replace_nones(self.sidecar_template), indent=4))
 
+    def write_out_blood_files(self, new_file_name_with_entities=None, destination_folder=None):
+        recording_entity = "_recording-manual"
+
+        if not new_file_name_with_entities:
+            new_file_name_with_entities = pathlib.Path(self.nifti_file)
+        if not destination_folder:
+            destination_folder = pathlib.Path(self.nifti_file).parent
+
+        if '_pet' in new_file_name_with_entities.name:
+            if new_file_name_with_entities.suffix == '.gz' and len(new_file_name_with_entities.suffixes) > 1:
+                new_file_name_with_entities = new_file_name_with_entities.with_suffix('').with_suffix('')
+
+            blood_file_name = new_file_name_with_entities.stem.replace('_pet', recording_entity + '_blood')
+        else:
+            blood_file_name = new_file_name_with_entities.stem + recording_entity + '_blood'
+
+        if self.spreadsheet_metadata.get('blood_tsv', {}) != {}:
+            blood_tsv_data = self.spreadsheet_metadata.get('blood_tsv')
+            if type(blood_tsv_data) is pd.DataFrame or type(blood_tsv_data) is dict:
+                if type(blood_tsv_data) is dict:
+                    blood_tsv_data = pd.DataFrame(blood_tsv_data)
+                # write out blood_tsv using pandas csv write
+                blood_tsv_data.to_csv(os.path.join(destination_folder, blood_file_name + ".tsv")
+                                      , sep='\t',
+                                      index=False)
+
+            elif type(blood_tsv_data) is str:
+                # write out with write
+                with open(os.path.join(destination_folder, blood_file_name + ".tsv"), 'w') as outfile:
+                    outfile.writelines(blood_tsv_data)
+            else:
+                raise (f"blood_tsv dictionary is incorrect type {type(blood_tsv_data)}, must be type: "
+                       f"pandas.DataFrame")
+
+        # if there's blood data in the tsv then write out the sidecar file too
+        if self.spreadsheet_metadata.get('blood_json', {}) != {} \
+                and self.spreadsheet_metadata.get('blood_tsv', {}) != {}:
+            blood_json_data = self.spreadsheet_metadata.get('blood_json')
+            if type(blood_json_data) is dict:
+                # write out to file with json dump
+                pass
+            elif type(blood_json_data) is str:
+                # write out to file with json dumps
+                blood_json_data = json.loads(blood_json_data)
+            else:
+                raise (f"blood_json dictionary is incorrect type {type(blood_json_data)}, must be type: dict or str"
+                       f"pandas.DataFrame")
+
+            with open(os.path.join(destination_folder, blood_file_name + '.json'), 'w') as outfile:
+                json.dump(blood_json_data, outfile, indent=4)
+
+    def update_pet_json(self, pet_json_path):
+        """given a json file (or a path ending in .json) update or create a PET json file with information collected
+        from an ecat file.
+        :param pet_json: a path to a json file
+        :type pet_json: str or pathlib.Path
+        :return: None
+        """
+
+        # open the json file if it exists
+        if isinstance(pet_json_path, str):
+            pet_json = pathlib.Path(pet_json_path)
+        if pet_json.exists():
+            with open(pet_json_path, 'r') as json_file:
+                try:
+                    pet_json = json.load(json_file)
+                except json.decoder.JSONDecodeError:
+                    logger.warning(f"Unable to load json file at {pet_json_path}, skipping.")
+
+            # update the template with values from the json file
+            self.sidecar_template.update(pet_json)
+
+        if self.spreadsheet_metadata.get('nifti_json', None):
+            self.sidecar_template.update(self.spreadsheet_metadata['nifti_json'])
+
+        self.populate_sidecar(**self.kwargs)
+        self.prune_sidecar()
+
+        # check metadata radio inputs
+        self.sidecar_template.update(check_meta_radio_inputs(self.sidecar_template))
+
+        self.show_sidecar(output_path=pet_json_path)
+
     def json_out(self):
         """
         Dumps entire ecat header and header info into stdout formatted as json.
@@ -344,3 +469,16 @@ def json_out(self):
         """
         temp_json = json.dumps(self.ecat_info, indent=4)
         print(temp_json)
+
+    def convert(self):
+        """
+        Convert ecat to nifti
+        :return: None
+        """
+        self.output_path = pathlib.Path(self.make_nifti())
+        self.sidecar_path = self.output_path.parent / self.output_path.stem
+        self.sidecar_path = self.sidecar_path.with_suffix('.json')
+        self.populate_sidecar(**self.kwargs)
+        self.prune_sidecar()
+        self.show_sidecar(output_path=self.sidecar_path)
+        self.write_out_blood_files()

pypet2bids/pypet2bids/ecat_cli.py

@@ -10,15 +10,15 @@
 import sys
 import textwrap
 from os.path import join
-from pypet2bids.ecat import Ecat
 
 try:
     import helper_functions
+    import Ecat
+    from update_json_pet_file import check_json, check_meta_radio_inputs
 except ModuleNotFoundError:
     import pypet2bids.helper_functions as helper_functions
-
-#from pypet2bids.helper_functions import load_vars_from_config, ParseKwargs
-
+    from pypet2bids.ecat import Ecat
+    from pypet2bids.update_json_pet_file import check_json, check_meta_radio_inputs
 
 epilog = textwrap.dedent('''
 
@@ -62,14 +62,17 @@ def cli():
     :type --director_table: flag
     :param --show-examples: shows verbose example usage of this cli
     :type --show-examples: flag
+    :param --metadata-path: path to a spreadsheet containing PET metadata
+    :type --metadata-path: path
 
     :return: argparse.ArgumentParser.args for later use in executing conversions or ECAT methods
     """
-    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,epilog=epilog)
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
+    update_or_convert = parser.add_mutually_exclusive_group()
     parser.add_argument("ecat", nargs='?', metavar="ecat_file", help="Ecat image to collect info from.")
     parser.add_argument("--affine", "-a", help="Show affine matrix", action="store_true", default=False)
-    parser.add_argument("--convert", "-c", required=False, action='store_true',
-                        help="If supplied will attempt conversion.")
+    update_or_convert.add_argument("--convert", "-c", required=False, action='store_true',
+                                   help="If supplied will attempt conversion.")
     parser.add_argument("--dump", "-d", help="Dump information in Header", action="store_true", default=False)
     parser.add_argument("--json", "-j", action="store_true", default=False, help="""
         Output header and subheader info as JSON to stdout, overrides all other options""")
@@ -95,6 +98,20 @@ def cli():
                         action="store_true", default=False)
     parser.add_argument('--show-examples', '-E', '--HELP', '-H', help='Shows example usage of this cli.',
                         action='store_true')
+    parser.add_argument('--metadata-path', '-m', help='Path to a spreadsheet containing PET metadata.')
+    update_or_convert.add_argument('--update', '-u', type=str, default="",
+                                   help='Update/create a json sidecar file from an ECAT given a path to that each '
+                                        'file,. e.g.'
+                                        'ecatpet2bids ecatfile.v --update path/to/sidecar.json '
+                                        'additionally one can pass metadat to the sidecar via inclusion of the '
+                                        '--kwargs flag or'
+                                        'the --metadata-path flag. If both are included the --kwargs flag will '
+                                        'override any'
+                                        'overlapping values in the --metadata-path flag or found in the ECAT file \n'
+                                        'ecatpet2bids ecatfile.v --update path/to/sidecar.json --kwargs '
+                                        'TimeZero="12:12:12"'
+                                        'ecatpet2bids ecatfile.v --update path/to/sidecar.json --metadata-path '
+                                        'path/to/metadata.xlsx')
 
     return parser
 
@@ -166,7 +183,7 @@ def main():
         sys.exit(0)
 
     collect_pixel_data = False
-    if cli_args.convert:
+    if cli_args.convert or cli_args.update:
         collect_pixel_data = True
     if cli_args.scannerparams is not None:
         # if no args are supplied to --scannerparams/-s
@@ -180,7 +197,7 @@ def main():
             if scanner_txt is None:
                 called_dir = os.getcwd()
                 error_string = f'No scanner file found in {called_dir}. Either create a parameters.txt file, omit ' \
-                               f'the --scannerparams argument, or specify a full path to a scanner.txt file after the '\
+                               f'the --scannerparams argument, or specify a full path to a scanner.txt file after the ' \
                                f'--scannerparams argument.'
                 raise Exception(error_string)
         else:
@@ -195,7 +212,9 @@ def main():
 
     ecat = Ecat(ecat_file=cli_args.ecat,
                 nifti_file=cli_args.nifti,
-                collect_pixel_data=collect_pixel_data)
+                collect_pixel_data=collect_pixel_data,
+                metadata_path=cli_args.metadata_path,
+                kwargs=cli_args.kwargs)
     if cli_args.json:
         ecat.json_out()
         sys.exit(0)
@@ -214,11 +233,39 @@ def main():
         ecat.populate_sidecar(**cli_args.kwargs)
         ecat.show_sidecar()
     if cli_args.convert:
-        output_path = pathlib.Path(ecat.make_nifti())
-        ecat.populate_sidecar(**cli_args.kwargs)
-        ecat.prune_sidecar()
-        sidecar_path = pathlib.Path(join(str(output_path.parent), output_path.stem + '.json'))
-        ecat.show_sidecar(output_path=sidecar_path)
+        ecat.convert()
+    if cli_args.update:
+        ecat.update_pet_json(cli_args.update)
+
+
+def update_json_with_ecat_value_cli():
+    """
+    Updates a json sidecar with values extracted from an ecat file, optionally additional values can be included
+    via the -k --additional-arguments flag and/or a metadata spreadsheet can be supplied via the --metadata-path flag.
+    Command can be accessed after installation via `upadatepetjsonfromecat`
+    """
+    json_update_cli = argparse.ArgumentParser()
+    json_update_cli.add_argument("-j", "--json", help="Path to a json to update file.", required=True)
+    json_update_cli.add_argument("-e", "--ecat", help="Path to an ecat file.", required=True)
+    json_update_cli.add_argument("-m", "--metadata-path", help="Path to a spreadsheet containing PET metadata.")
+    json_update_cli.add_argument("-k", "--additional-arguments", nargs='*', action=helper_functions.ParseKwargs, default={},
+                                 help="Include additional values in the sidecar json or override values extracted "
+                                      "from the supplied ECAT or metadata spreadsheet. "
+                                      "e.g. including `--kwargs TimeZero=\"12:12:12\"` would override the calculated "
+                                      "TimeZero."
+                                      "Any number of additional arguments can be supplied after --kwargs e.g. `--kwargs"
+                                      "BidsVariable1=1 BidsVariable2=2` etc etc."
+                                      "Note: the value portion of the argument (right side of the equal's sign) should "
+                                      "always be surrounded by double quotes BidsVarQuoted=\"[0, 1 , 3]\"")
+
+    args = json_update_cli.parse_args()
+
+    update_ecat = Ecat(ecat_file=args.ecat, nifti_file=None, collect_pixel_data=True,
+                       metadata_path=args.metadata_path, kwargs=args.additional_arguments)
+    update_ecat.update_pet_json(args.json)
+
+    # lastly check the json
+    check_json(args.json, logger='check_json', silent=False)
 
 
 if __name__ == "__main__":

pypet2bids/pypet2bids/helper_functions.py

@@ -54,6 +54,11 @@
 bids_schema_path = os.path.join(metadata_dir, 'schema.json')
 schema = json.load(open(bids_schema_path, 'r'))
 
+# putting these paths here as they are reused in dcm2niix4pet.py, update_json_pet_file.py, and ecat.py
+module_folder = Path(__file__).parent.resolve()
+python_folder = module_folder.parent
+pet2bids_folder = python_folder.parent
+metadata_folder = os.path.join(pet2bids_folder, 'metadata')
 
 loggers = {}