From a22d22b5d6be464d31c60674518cfac542b74d36 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 16:17:27 -0600 Subject: [PATCH 01/23] Use artifact for BIDSFile functionality --- bids/layout/models.py | 71 ++++++++++++++++++++++++++++--------------- bids/variables/io.py | 2 +- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index af3a011ef..b8cd71060 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -1,18 +1,13 @@ """ Model classes used in BIDSLayouts. """ -import re import os from pathlib import Path -import warnings import json -from copy import deepcopy -from itertools import chain -from functools import lru_cache +from ancpbids.model_v1_8_0 import Artifact from ..utils import listify from .writing import build_path, write_to_file -from ..config import get_option -from .utils import BIDSMetadata, PaddedInt +from .utils import BIDSMetadata class BIDSFile: @@ -40,17 +35,43 @@ def from_path(cls, path): break return cls(path) - def __init__(self, filename, root=None): - self.path = str(filename) - self.filename = self._path.name - self.dirname = str(self._path.parent) - self.is_dir = not self.filename + @classmethod + def from_artifact(cls, ancp): + """ Load from ANCPBids Artifact """ + return cls(artifact=ancp.path) + + def __init__(self, artifact=None, filename=None, root=None): + if artifact is not None: + self.artifact = artifact + else: + # Problem here is we need to extract the entities, suffix, etc from filename + # and we need the root, which is not available in Artifact object + raise NotImplementedError + # self.artifact = Artifact( + # suffix=suffix, entities=entities, name=str(filename), extension=extension, uri=uri) + self._root = root + @property + def path(self): + return self.artifact.path + @property def _path(self): return Path(self.path) + @property + def filename(self): + return self._path.name + + @property + def is_dir(self): + return not self.filename + + @property + def dirname(self): + return str(self._path.parent) + def __repr__(self): return f"<{self.__class__.__name__} filename='{self.path}'>" @@ -60,7 +81,9 @@ def __fspath__(self): @property def relpath(self): """Return path relative to layout root""" - return str(Path(self.path).relative_to(self._root)) + if self._root is None: + raise NotImplementedError + return str(self._path.relative_to(self._root)) def get_associations(self, kind=None, include_parents=False): """Get associated files, optionally limiting by association kind. @@ -91,6 +114,10 @@ def get_metadata(self): md.update(self.get_entities(metadata=True)) return md + @property + def entities(self): + return self.get_entities() + def get_entities(self, metadata=False, values='tags'): """Return entity information for the current file. @@ -115,18 +142,12 @@ def get_entities(self, metadata=False, values='tags'): A dict, where keys are entity names and values are Entity instances. """ - session = object_session(self) - query = (session.query(Tag) - .filter_by(file_path=self.path) - .join(Entity)) - - if metadata not in (None, 'all'): - query = query.filter(Tag.is_metadata == metadata) - - results = query.all() - if values.startswith('obj'): - return {t.entity_name: t.entity for t in results} - return {t.entity_name: t.value for t in results} + entities = self.artifact.get_entities() + if metadata: + entities = {**entities, **self.artifact.get_metadata()} + + if values == 'object': + raise NotImplementedError def copy(self, path_patterns, symbolic_link=False, root=None, conflicts='fail'): diff --git a/bids/variables/io.py b/bids/variables/io.py index 2fedf5e9d..94c286997 100644 --- a/bids/variables/io.py +++ b/bids/variables/io.py @@ -177,7 +177,7 @@ def _load_time_variables(layout, dataset=None, columns=None, scan_length=None, if dataset is None: dataset = NodeIndex() - selectors['datatype'] = 'func' + # selectors['datatype'] = 'func' selectors['suffix'] = 'bold' exts = selectors.pop('extension', ['.nii', '.nii.gz', '.func.gii', '.dtseries.nii']) images = layout.get(return_type='object', scope=scope, extension=exts, **selectors) From d356e0b499bc31b4873e5742234958c87df21bf2 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 16:22:39 -0600 Subject: [PATCH 02/23] Make root mandatory --- bids/layout/models.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index b8cd71060..0165fb42b 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -36,37 +36,42 @@ def from_path(cls, path): return cls(path) @classmethod - def from_artifact(cls, ancp): + def from_artifact(cls, artifact, root): """ Load from ANCPBids Artifact """ - return cls(artifact=ancp.path) + return cls(root, artifact=artifact) - def __init__(self, artifact=None, filename=None, root=None): + def __init__(self, root, artifact=None, filename=None): if artifact is not None: self.artifact = artifact - else: - # Problem here is we need to extract the entities, suffix, etc from filename + elif filename is not None: + # We need to extract the entities, suffix, etc from filename # and we need the root, which is not available in Artifact object raise NotImplementedError # self.artifact = Artifact( # suffix=suffix, entities=entities, name=str(filename), extension=extension, uri=uri) + else: + raise ValueError("Either artifact or filename must be provided") self._root = root @property def path(self): - return self.artifact.path + """ Convenience property for accessing path as a string.""" + return self.artifact.name @property def _path(self): + """ Convenience property for accessing path as a Path object.""" return Path(self.path) @property def filename(self): + """ Convenience property for accessing filename.""" return self._path.name @property def is_dir(self): - return not self.filename + return not self._path.isdir() @property def dirname(self): @@ -81,8 +86,6 @@ def __fspath__(self): @property def relpath(self): """Return path relative to layout root""" - if self._root is None: - raise NotImplementedError return str(self._path.relative_to(self._root)) def get_associations(self, kind=None, include_parents=False): From d531a032f3eb59f08960c3d3f3776570742717e9 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 16:53:44 -0600 Subject: [PATCH 03/23] Create BIDSFiles on the fly from_artifacts --- bids/layout/layout.py | 4 ++-- bids/layout/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 8c0e579ff..61b8230be 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -72,7 +72,7 @@ class BIDSLayout(BIDSLayoutMRIMixin): .. code-block:: dataset_path = 'path/to/your/dataset' - layout = BIDSLayout(dataset_path) + layouBIDSFilet = BIDSLayout(dataset_path) Parameters ---------- @@ -268,7 +268,7 @@ def get(self, return_type: str = 'object', target: str = None, scope: str = None result = natural_sort(result) if return_type == "object": result = natural_sort( - [BIDSFile.from_path(res.get_absolute_path()) for res in result], + [BIDSFile.from_artifact(self.root, res) for res in result], "path" ) return result diff --git a/bids/layout/models.py b/bids/layout/models.py index 0165fb42b..e3b182105 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -36,7 +36,7 @@ def from_path(cls, path): return cls(path) @classmethod - def from_artifact(cls, artifact, root): + def from_artifact(cls, root, artifact): """ Load from ANCPBids Artifact """ return cls(root, artifact=artifact) From 10f128ff84665b0ed1ea079e60b2ea93214b64d3 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 16:56:46 -0600 Subject: [PATCH 04/23] Return entities' --- bids/layout/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bids/layout/models.py b/bids/layout/models.py index e3b182105..77486db6f 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -152,6 +152,8 @@ def get_entities(self, metadata=False, values='tags'): if values == 'object': raise NotImplementedError + return entities + def copy(self, path_patterns, symbolic_link=False, root=None, conflicts='fail'): """Copy the contents of a file to a new location. From 40af0da9c31bce476b86ba3633021e715d589faf Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 17:05:56 -0600 Subject: [PATCH 05/23] Add missing methods from previous API --- bids/layout/layout.py | 187 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 61b8230be..033ea8aeb 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -9,6 +9,7 @@ from .models import BIDSFile from .utils import BIDSMetadata +from .writing import build_path, write_to_file from ..exceptions import ( BIDSEntityError, BIDSValidationError, @@ -25,6 +26,128 @@ from ..utils import natural_sort +class BIDSLayoutWritingMixin: + def build_path(self, source, path_patterns=None, strict=False, + scope='all', validate=True, absolute_paths=None): + """Construct a target filename for a file or dictionary of entities. + Parameters + ---------- + source : str or :obj:`bids.layout.BIDSFile` or dict + The source data to use to construct the new file path. + Must be one of: + - A BIDSFile object + - A string giving the path of a BIDSFile contained within the + current Layout. + - A dict of entities, with entity names in keys and values in + values + path_patterns : list + Optional path patterns to use to construct + the new file path. If None, the Layout-defined patterns will + be used. Entities should be represented by the name + surrounded by curly braces. Optional portions of the patterns + should be denoted by square brackets. Entities that require a + specific value for the pattern to match can pass them inside + angle brackets. Default values can be assigned by specifying a string + after the pipe operator. E.g., (e.g., {type|bold} would + only match the pattern if the entity 'type' was passed and its + value is "image", otherwise the default value "bold" will be + used). + Example: 'sub-{subject}/[var-{name}/]{id}.csv' + Result: 'sub-01/var-SES/1045.csv' + strict : bool, optional + If True, all entities must be matched inside a + pattern in order to be a valid match. If False, extra entities + will be ignored so long as all mandatory entities are found. + scope : str or list, optional + The scope of the search space. Indicates which + BIDSLayouts' path patterns to use. See BIDSLayout docstring + for valid values. By default, uses all available layouts. If + two or more values are provided, the order determines the + precedence of path patterns (i.e., earlier layouts will have + higher precedence). + validate : bool, optional + If True, built path must pass BIDS validator. If + False, no validation is attempted, and an invalid path may be + returned (e.g., if an entity value contains a hyphen). + absolute_paths : bool, optional + Optionally override the instance-wide option + to report either absolute or relative (to the top of the + dataset) paths. If None, will fall back on the value specified + at BIDSLayout initialization. + """ + raise NotImplementedError + + def copy_files(self, files=None, path_patterns=None, symbolic_links=True, + root=None, conflicts='fail', **kwargs): + """Copy BIDSFile(s) to new locations. + The new locations are defined by each BIDSFile's entities and the + specified `path_patterns`. + Parameters + ---------- + files : list + Optional list of BIDSFile objects to write out. If + none provided, use files from running a get() query using + remaining **kwargs. + path_patterns : str or list + Write patterns to pass to each file's write_file method. + symbolic_links : bool + Whether to copy each file as a symbolic link or a deep copy. + root : str + Optional root directory that all patterns are relative + to. Defaults to dataset root. + conflicts : str + Defines the desired action when the output path already exists. + Must be one of: + 'fail': raises an exception + 'skip' does nothing + 'overwrite': overwrites the existing file + 'append': adds a suffix to each file copy, starting with 1 + kwargs : dict + Optional key word arguments to pass into a get() query. + """ + raise NotImplementedError + + + def write_to_file(self, entities, path_patterns=None, + contents=None, link_to=None, copy_from=None, + content_mode='text', conflicts='fail', + strict=False, validate=True): + """Write data to a file defined by the passed entities and patterns. + Parameters + ---------- + entities : dict + A dictionary of entities, with Entity names in + keys and values for the desired file in values. + path_patterns : list + Optional path patterns to use when building + the filename. If None, the Layout-defined patterns will be + used. + contents : object + Contents to write to the generate file path. + Can be any object serializable as text or binary data (as + defined in the content_mode argument). + link_to : str + Optional path with which to create a symbolic link + to. Used as an alternative to and takes priority over the + contents argument. + conflicts : str + Defines the desired action when the output path already exists. + Must be one of: + 'fail': raises an exception + 'skip' does nothing + 'overwrite': overwrites the existing file + 'append': adds a suffix to each file copy, starting with 1 + strict : bool + If True, all entities must be matched inside a + pattern in order to be a valid match. If False, extra entities + will be ignored so long as all mandatory entities are found. + validate : bool + If True, built path must pass BIDS validator. If + False, no validation is attempted, and an invalid path may be + returned (e.g., if an entity value contains a hyphen). + """ + raise NotImplementedError + class BIDSLayoutMRIMixin: def get_tr(self, derivatives=False, **entities): @@ -66,7 +189,7 @@ def get_tr(self, derivatives=False, **entities): return all_trs.pop() -class BIDSLayout(BIDSLayoutMRIMixin): +class BIDSLayout(BIDSLayoutMRIMixin, BIDSLayoutWritingMixin): """A convenience class to provide access to an in-memory representation of a BIDS dataset. .. code-block:: @@ -188,6 +311,68 @@ def get_metadata(self, path, include_entities=False, scope='all'): bmd.update(md) return bmd + def parse_file_entities(self, filename, scope='all', entities=None, + config=None, include_unmatched=False): + """Parse the passed filename for entity/value pairs. + Parameters + ---------- + filename : str + The filename to parse for entity values + scope : str or list, optional + The scope of the search space. Indicates which BIDSLayouts' + entities to extract. See :obj:`bids.layout.BIDSLayout.get` + docstring for valid values. By default, extracts all entities. + entities : list or None, optional + An optional list of Entity instances to use in + extraction. If passed, the scope and config arguments are + ignored, and only the Entities in this list are used. + config : str or :obj:`bids.layout.models.Config` or list or None, optional + One or more :obj:`bids.layout.models.Config` objects, or paths + to JSON config files on disk, containing the Entity definitions + to use in extraction. If passed, scope is ignored. + include_unmatched : bool, optional + If True, unmatched entities are included + in the returned dict, with values set to None. If False + (default), unmatched entities are ignored. + Returns + ------- + dict + Dictionary where keys are Entity names and values are the + values extracted from the filename. + """ + + raise NotImplementedError("parse_file_entities is not implemented") + + def get_nearest(self, path, return_type='filename', strict=True, + all_=False, ignore_strict_entities='extension', + full_search=False, **filters): + """Walk up file tree from specified path and return nearest matching file(s). + Parameters + ---------- + path (str): The file to search from. + return_type (str): What to return; must be one of 'filename' + (default) or 'tuple'. + strict (bool): When True, all entities present in both the input + path and the target file(s) must match perfectly. When False, + files will be ordered by the number of matching entities, and + partial matches will be allowed. + all_ (bool): When True, returns all matching files. When False + (default), only returns the first match. + ignore_strict_entities (str, list): Optional entity/entities to + exclude from strict matching when strict is True. This allows + one to search, e.g., for files of a different type while + matching all other entities perfectly by passing + ignore_strict_entities=['type']. Ignores extension by default. + full_search (bool): If True, searches all indexed files, even if + they don't share a common root with the provided path. If + False, only files that share a common root will be scanned. + filters : dict + Optional keywords to pass on to :obj:`bids.layout.BIDSLayout.get`. + """ + + raise NotImplementedError("get_nearest is not implemented") + + def get(self, return_type: str = 'object', target: str = None, scope: str = None, extension: Union[str, List[str]] = None, suffix: Union[str, List[str]] = None, regex_search=None, From 778cb8df41d9e5608f0f14c3ccf94f159508b455 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 17:29:06 -0600 Subject: [PATCH 06/23] Fix typo in docstring --- bids/layout/layout.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 033ea8aeb..95171127e 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -195,7 +195,7 @@ class BIDSLayout(BIDSLayoutMRIMixin, BIDSLayoutWritingMixin): .. code-block:: dataset_path = 'path/to/your/dataset' - layouBIDSFilet = BIDSLayout(dataset_path) + layout = BIDSLayout(dataset_path) Parameters ---------- @@ -340,6 +340,14 @@ def parse_file_entities(self, filename, scope='all', entities=None, Dictionary where keys are Entity names and values are the values extracted from the filename. """ + # # If either entities or config is specified, just pass through + # if entities is None and config is None: + # layouts = self._get_layouts_in_scope(scope) + # config = chain(*[list(l.config.values()) for l in layouts]) + # config = list(set(config)) + + # return parse_file_entities(filename, entities, config, + # include_unmatched) raise NotImplementedError("parse_file_entities is not implemented") @@ -373,6 +381,7 @@ def get_nearest(self, path, return_type='filename', strict=True, raise NotImplementedError("get_nearest is not implemented") + def get(self, return_type: str = 'object', target: str = None, scope: str = None, extension: Union[str, List[str]] = None, suffix: Union[str, List[str]] = None, regex_search=None, From e77059bcf91d5c5d634b419ea6a35178e2c153e1 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Mon, 12 Dec 2022 18:16:35 -0600 Subject: [PATCH 07/23] Add parse_file_entities and get_nearest --- bids/layout/layout.py | 109 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 94 insertions(+), 15 deletions(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 95171127e..e3cd2bcca 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -1,7 +1,7 @@ import difflib import enum import os.path -from collections import OrderedDict +from collections import defaultdict from functools import partial from pathlib import Path from typing import List, Union, Dict, Optional, Any, Callable @@ -20,11 +20,11 @@ from ancpbids import CustomOpExpr, EntityExpr, AllExpr, ValidationPlugin, load_dataset, validate_dataset, \ write_derivative from ancpbids.query import query, query_entities, FnMatchExpr, AnyExpr -from ancpbids.utils import deepupdate, resolve_segments, convert_to_relative +from ancpbids.utils import deepupdate, resolve_segments, convert_to_relative, parse_bids_name __all__ = ['BIDSLayout, Query'] -from ..utils import natural_sort +from ..utils import natural_sort, listify class BIDSLayoutWritingMixin: def build_path(self, source, path_patterns=None, strict=False, @@ -311,7 +311,7 @@ def get_metadata(self, path, include_entities=False, scope='all'): bmd.update(md) return bmd - def parse_file_entities(self, filename, scope='all', entities=None, + def parse_file_entities(self, filename, scope=None, entities=None, config=None, include_unmatched=False): """Parse the passed filename for entity/value pairs. Parameters @@ -340,16 +340,24 @@ def parse_file_entities(self, filename, scope='all', entities=None, Dictionary where keys are Entity names and values are the values extracted from the filename. """ - # # If either entities or config is specified, just pass through - # if entities is None and config is None: - # layouts = self._get_layouts_in_scope(scope) - # config = chain(*[list(l.config.values()) for l in layouts]) - # config = list(set(config)) + results = parse_bids_name(filename) + entities = results.pop['entities'] + results = {**entities, **results} - # return parse_file_entities(filename, entities, config, - # include_unmatched) + if entities: + # Filter out any entities that aren't in the specified + results = {e: results[e] for e in entities if e in results} - raise NotImplementedError("parse_file_entities is not implemented") + if include_unmatched: + for k in set(self.get_entities()) - set(results)): + results[k] = None + + if scope is not None or config is not None: + # To implement, need to be able to parse given a speciifc scope / config + # Currently, parse_bids_name uses a fixed config in ancpbids + raise NotImplementedError("scope and config are not implemented") + + return results def get_nearest(self, path, return_type='filename', strict=True, all_=False, ignore_strict_entities='extension', @@ -378,9 +386,80 @@ def get_nearest(self, path, return_type='filename', strict=True, Optional keywords to pass on to :obj:`bids.layout.BIDSLayout.get`. """ - raise NotImplementedError("get_nearest is not implemented") - - + path = Path(path).absolute() + + # Make sure we have a valid suffix + if not filters.get('suffix'): + f = self.get_file(path) + if 'suffix' not in f.entities: + raise BIDSValidationError( + "File '%s' does not have a valid suffix, most " + "likely because it is not a valid BIDS file." % path + ) + filters['suffix'] = f.entities['suffix'] + + # Collect matches for all entities for this file + entities = self.parse_file_entities(str(path)) + + # Remove any entities we want to ignore when strict matching is on + if strict and ignore_strict_entities is not None: + for k in listify(ignore_strict_entities): + entities.pop(k, None) + + # Get candidate files + results = self.get(**filters) + + # Make a dictionary of directories --> contained files + folders = defaultdict(list) + for f in results: + folders[f._dirname].append(f) + + # Build list of candidate directories to check + # Walking up from path, add all parent directories with a + # matching file + search_paths = [] + while True: + if path in folders and folders[path]: + search_paths.append(path) + parent = path.parent + if parent == path: + break + path = parent + + if full_search: + unchecked = set(folders.keys()) - set(search_paths) + search_paths.extend(path for path in unchecked if folders[path]) + + def count_matches(f): + # Count the number of entities shared with the passed file + # Returns a tuple of (num_shared, num_perfect) + f_ents = f.entities + keys = set(entities.keys()) & set(f_ents.keys()) + shared = len(keys) + return (shared, sum([entities[k] == f_ents[k] for k in keys])) + + matches = [] + + for path in search_paths: + # Sort by number of matching entities. Also store number of + # common entities, for filtering when strict=True. + num_ents = [(f, ) + count_matches(f) for f in folders[path]] + # Filter out imperfect matches (i.e., where number of common + # entities does not equal number of matching entities). + if strict: + num_ents = [f for f in num_ents if f[1] == f[2]] + num_ents.sort(key=lambda x: x[2], reverse=True) + + if num_ents: + matches += [f_match[0] for f_match in num_ents] + + if not all_: + break + + matches = [match.path if return_type == 'filename' + else match for match in matches] + return matches if all_ else matches[0] if matches else None + def get(self, return_type: str = 'object', target: str = None, scope: str = None, extension: Union[str, List[str]] = None, suffix: Union[str, List[str]] = None, From 6bb4710cb257a6c40d533422be2266a7e23cc035 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:19:52 -0600 Subject: [PATCH 08/23] Load BIDSFile from filename --- bids/layout/models.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index 77486db6f..0f5447993 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -4,6 +4,7 @@ from pathlib import Path import json from ancpbids.model_v1_8_0 import Artifact +from ancpbids import parse_bids_name from ..utils import listify from .writing import build_path, write_to_file @@ -27,13 +28,13 @@ def __init_subclass__(cls, extension, **kwargs): BIDSFile._ext_registry[ext] = cls @classmethod - def from_path(cls, path): - path = Path(path) + def from_filename(cls, root, filename): + path = Path(filename) for ext, subclass in cls._ext_registry.items(): if path.name.endswith(ext): cls = subclass break - return cls(path) + return cls(root==root, filename=path) @classmethod def from_artifact(cls, root, artifact): @@ -44,11 +45,7 @@ def __init__(self, root, artifact=None, filename=None): if artifact is not None: self.artifact = artifact elif filename is not None: - # We need to extract the entities, suffix, etc from filename - # and we need the root, which is not available in Artifact object - raise NotImplementedError - # self.artifact = Artifact( - # suffix=suffix, entities=entities, name=str(filename), extension=extension, uri=uri) + self.artifact = Artifact(**parse_bids_name(filename)) else: raise ValueError("Either artifact or filename must be provided") From dd1343ce4f083d9aa5dd076e84bcf19bdf1e61a1 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:26:31 -0600 Subject: [PATCH 09/23] Remove root from BIDSFile --- bids/layout/models.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index 0f5447993..2270bc604 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -28,20 +28,20 @@ def __init_subclass__(cls, extension, **kwargs): BIDSFile._ext_registry[ext] = cls @classmethod - def from_filename(cls, root, filename): + def from_filename(cls, filename): path = Path(filename) for ext, subclass in cls._ext_registry.items(): if path.name.endswith(ext): cls = subclass break - return cls(root==root, filename=path) + return cls(filename=path) @classmethod - def from_artifact(cls, root, artifact): + def from_artifact(cls, artifact): """ Load from ANCPBids Artifact """ - return cls(root, artifact=artifact) + return cls(artifact=artifact) - def __init__(self, root, artifact=None, filename=None): + def __init__(self, artifact=None, filename=None): if artifact is not None: self.artifact = artifact elif filename is not None: @@ -49,8 +49,6 @@ def __init__(self, root, artifact=None, filename=None): else: raise ValueError("Either artifact or filename must be provided") - self._root = root - @property def path(self): """ Convenience property for accessing path as a string.""" @@ -81,9 +79,9 @@ def __fspath__(self): return self.path @property - def relpath(self): + def relpath(self, root): """Return path relative to layout root""" - return str(self._path.relative_to(self._root)) + return str(self._path.relative_to(root)) def get_associations(self, kind=None, include_parents=False): """Get associated files, optionally limiting by association kind. From 4f895879b04f17d7793c79c1fd908779ace9266c Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:26:38 -0600 Subject: [PATCH 10/23] Remove database references --- bids/layout/db.py | 162 ------------------------------ bids/layout/tests/test_db.py | 13 --- bids/layout/tests/test_models.py | 139 ++++++++++++------------- bids/layout/tests/test_writing.py | 15 +-- 4 files changed, 66 insertions(+), 263 deletions(-) delete mode 100644 bids/layout/db.py delete mode 100644 bids/layout/tests/test_db.py diff --git a/bids/layout/db.py b/bids/layout/db.py deleted file mode 100644 index 7a9062de6..000000000 --- a/bids/layout/db.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Database-related functionality. -""" - -from pathlib import Path -import re -import sqlite3 -from functools import lru_cache - -import sqlalchemy as sa - -from bids.utils import listify -from .models import Base, Config, LayoutInfo - - -def get_database_file(path): - if path is not None: - path = Path(path) - database_file = path / 'layout_index.sqlite' - path.mkdir(exist_ok=True, parents=True) - else: - database_file = None - return database_file - - -class ConnectionManager: - - def __init__(self, database_path=None, reset_database=False, config=None, - init_args=None): - - self.database_file = get_database_file(database_path) - - # Determine if file exists before we create it in _get_engine() - reset_database = ( - reset_database or # manual reset - self.database_file is None or # in-memory DB - not self.database_file.exists() # file hasn't been created yet - ) - - self.engine = self._get_engine(self.database_file) - self.sessionmaker = sa.orm.sessionmaker(bind=self.engine) - self._session = None - - if reset_database: - self.reset_database(init_args, config) - - self._database_reset = reset_database - - def _get_engine(self, database_file): - if database_file is not None: - # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html - # When a file-based database is specified, the dialect will use - # NullPool as the source of connections. This pool closes and - # discards connections which are returned to the pool immediately. - # SQLite file-based connections have extremely low overhead, so - # pooling is not necessary. The scheme also prevents a connection - # from being used again in a different thread and works best - # with SQLite's coarse-grained file locking. - from sqlalchemy.pool import NullPool - engine = sa.create_engine( - 'sqlite:///{dbfilepath}'.format(dbfilepath=database_file), - connect_args={'check_same_thread': False}, - poolclass=NullPool) - else: - # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html - # Using a Memory Database in Multiple Threads - # To use a :memory: database in a multithreaded scenario, the same - # connection object must be shared among threads, since the - # database exists only within the scope of that connection. The - # StaticPool implementation will maintain a single connection - # globally, and the check_same_thread flag can be passed to - # Pysqlite as False. Note that using a :memory: database in - # multiple threads requires a recent version of SQLite. - from sqlalchemy.pool import StaticPool - engine = sa.create_engine( - 'sqlite://', # In memory database - connect_args={'check_same_thread': False}, - poolclass=StaticPool) - - def regexp(expr, item): - """Regex function for SQLite's REGEXP.""" - if not isinstance(item, str): - return False - reg = re.compile(expr, re.I) - return reg.search(item) is not None - - engine.connect() - - # Do not remove this decorator!!! An in-line create_function call will - # work when using an in-memory SQLite DB, but fails when using a file. - # For more details, see https://stackoverflow.com/questions/12461814/ - @sa.event.listens_for(engine, "begin") - def do_begin(conn): - conn.connection.create_function('regexp', 2, regexp) - - return engine - - @classmethod - def exists(cls, database_path): - return get_database_file(database_path).exists() - - def reset_database(self, init_args, config=None): - Base.metadata.drop_all(self.engine) - Base.metadata.create_all(self.engine) - # Add LayoutInfo record - if not isinstance(init_args, LayoutInfo): - layout_info = LayoutInfo(**init_args) - self.session.add(layout_info) - # Add config records - config = listify('bids' if config is None else config) - config = [Config.load(c, session=self.session) for c in listify(config)] - self.session.add_all(config) - self.session.commit() - - def save_database(self, database_path, replace_connection=True): - """Save the current index as a SQLite3 DB at the specified location. - - Note: This is only necessary if a database_path was not specified - at initialization, and the user now wants to save the index. - If a database_path was specified originally, there is no need to - re-save using this method. - - Parameters - ---------- - database_path : str - The path to the desired database folder. By default, - uses .db_cache. If a relative path is passed, it is assumed to - be relative to the BIDSLayout root directory. - replace_connection : bool, optional - If True, returns a new ConnectionManager that points to the newly - created database. If False, returns the current instance. - """ - database_file = get_database_file(database_path) - new_db = sqlite3.connect(str(database_file)) - old_db = self.engine.connect().connection - - with new_db: - for line in old_db.iterdump(): - if line not in ('BEGIN;', 'COMMIT;'): - new_db.execute(line) - new_db.commit() - - if replace_connection: - return ConnectionManager(database_path, init_args=self.layout_info) - else: - return self - - @property - def session(self): - if self._session is None: - self.reset_session() - return self._session - - @property - @lru_cache() - def layout_info(self): - return self.session.query(LayoutInfo).first() - - - def reset_session(self): - """Force a new session.""" - self._session = self.sessionmaker() diff --git a/bids/layout/tests/test_db.py b/bids/layout/tests/test_db.py deleted file mode 100644 index 58764e3af..000000000 --- a/bids/layout/tests/test_db.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Test functionality in the db module--mostly related to connection -management.""" - -from bids.layout.db import (ConnectionManager, get_database_file) - - -def test_get_database_file(tmp_path): - assert get_database_file(None) is None - new_path = tmp_path / "a_new_subdir" - assert not new_path.exists() - db_file = get_database_file(new_path) - assert db_file == new_path / 'layout_index.sqlite' - assert new_path.exists() diff --git a/bids/layout/tests/test_models.py b/bids/layout/tests/test_models.py index 2f7862fa1..34fa2bb22 100644 --- a/bids/layout/tests/test_models.py +++ b/bids/layout/tests/test_models.py @@ -17,14 +17,6 @@ from bids.tests import get_test_data_path - -def create_session(): - engine = create_engine('sqlite://') - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - return Session() - - @pytest.fixture def sample_bidsfile(tmpdir): testfile = 'sub-03_ses-2_task-rest_acq-fullbrain_run-2_bold.nii.gz' @@ -93,26 +85,25 @@ def test_entity_deepcopy(subject_entity): assert e != clone -def test_file_associations(): - session = create_session() - img = BIDSFile('sub-03/func/sub-03_task-rest_run-2_bold.nii.gz') - md1 = BIDSFile('sub-03/func/sub-03_task-rest_run-2_bold.json') - md2 = BIDSFile('task-rest_run-2_bold.json') - assocs = [ - FileAssociation(src=md1.path, dst=img.path, kind="MetadataFor"), - FileAssociation(src=img.path, dst=md1.path, kind="MetadataIn"), - FileAssociation(src=md1.path, dst=md2.path, kind="Child"), - FileAssociation(src=md2.path, dst=md1.path, kind="Parent"), - FileAssociation(src=md2.path, dst=img.path, kind="Informs") - ] - session.add_all([img, md1, md2] + assocs) - session.commit() - assert img._associations == [md1, md2] == img.get_associations() - assert md2._associations == [md1] - assert img.get_associations(kind='MetadataFor') == [] - assert img.get_associations(kind='MetadataIn') == [md1] - results = img.get_associations(kind='MetadataIn', include_parents=True) - assert set(results) == {md1, md2} +# def test_file_associations(): +# img = BIDSFile('sub-03/func/sub-03_task-rest_run-2_bold.nii.gz') +# md1 = BIDSFile('sub-03/func/sub-03_task-rest_run-2_bold.json') +# md2 = BIDSFile('task-rest_run-2_bold.json') +# assocs = [ +# FileAssociation(src=md1.path, dst=img.path, kind="MetadataFor"), +# FileAssociation(src=img.path, dst=md1.path, kind="MetadataIn"), +# FileAssociation(src=md1.path, dst=md2.path, kind="Child"), +# FileAssociation(src=md2.path, dst=md1.path, kind="Parent"), +# FileAssociation(src=md2.path, dst=img.path, kind="Informs") +# ] +# session.add_all([img, md1, md2] + assocs) +# session.commit() +# assert img._associations == [md1, md2] == img.get_associations() +# assert md2._associations == [md1] +# assert img.get_associations(kind='MetadataFor') == [] +# assert img.get_associations(kind='MetadataIn') == [md1] +# results = img.get_associations(kind='MetadataIn', include_parents=True) +# assert set(results) == {md1, md2} def test_tag_init(sample_bidsfile, subject_entity): @@ -137,52 +128,52 @@ def test_tag_dtype(sample_bidsfile, subject_entity): assert all(t.value == 4 for t in tags) -def test_entity_add_file(sample_bidsfile): - session = create_session() - bf = sample_bidsfile - e = Entity('prop', r'-(\d+)') - t = Tag(file=bf, entity=e, value=4) - session.add_all([t, e, bf]) - session.commit() - assert e.files[bf.path] == 4 - - -def test_config_init_with_args(): - session = create_session() - ents = [ - { - "name": "task", - "pattern": "[_/\\\\]task-([a-zA-Z0-9]+)" - }, - { - "name": "acquisition", - "pattern": "[_/\\\\]acq-([a-zA-Z0-9]+)" - } - ] - patterns = ['this_will_never_match_anything', 'and_neither_will_this'] - config = Config('custom', entities=ents, default_path_patterns=patterns) - assert config.name == 'custom' - target = {'task', 'acquisition'} - assert set(ent.name for ent in config.entities.values()) == target - assert config.default_path_patterns == patterns - - -def test_load_existing_config(): - session = create_session() - first = Config('dummy') - session.add(first) - session.commit() - - second = Config.load({"name": "dummy"}, session=session) - assert first == second - session.add(second) - session.commit() - - from sqlalchemy.orm.exc import FlushError - with pytest.raises(FlushError): - second = Config.load({"name": "dummy"}) - session.add(second) - session.commit() +# def test_entity_add_file(sample_bidsfile): +# session = create_session() +# bf = sample_bidsfile +# e = Entity('prop', r'-(\d+)') +# t = Tag(file=bf, entity=e, value=4) +# session.add_all([t, e, bf]) +# session.commit() +# assert e.files[bf.path] == 4 + + +# def test_config_init_with_args(): +# session = create_session() +# ents = [ +# { +# "name": "task", +# "pattern": "[_/\\\\]task-([a-zA-Z0-9]+)" +# }, +# { +# "name": "acquisition", +# "pattern": "[_/\\\\]acq-([a-zA-Z0-9]+)" +# } +# ] +# patterns = ['this_will_never_match_anything', 'and_neither_will_this'] +# config = Config('custom', entities=ents, default_path_patterns=patterns) +# assert config.name == 'custom' +# target = {'task', 'acquisition'} +# assert set(ent.name for ent in config.entities.values()) == target +# assert config.default_path_patterns == patterns + + +# def test_load_existing_config(): +# session = create_session() +# first = Config('dummy') +# session.add(first) +# session.commit() + +# second = Config.load({"name": "dummy"}, session=session) +# assert first == second +# session.add(second) +# session.commit() + +# from sqlalchemy.orm.exc import FlushError +# with pytest.raises(FlushError): +# second = Config.load({"name": "dummy"}) +# session.add(second) +# session.commit() def test_bidsfile_get_df_from_tsv_gz(layout_synthetic): diff --git a/bids/layout/tests/test_writing.py b/bids/layout/tests/test_writing.py index dde6b3a1f..092dd765f 100644 --- a/bids/layout/tests/test_writing.py +++ b/bids/layout/tests/test_writing.py @@ -5,8 +5,6 @@ from os.path import join, exists, islink, dirname import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker from bids.layout.writing import build_path, _PATTERN_FIND from bids.tests import get_test_data_path @@ -16,27 +14,16 @@ @pytest.fixture def writable_file(tmpdir): - engine = create_engine('sqlite://') - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - testfile = 'sub-03_ses-2_task-rest_acq-fullbrain_run-2_bold.nii.gz' fn = tmpdir.mkdir("tmp").join(testfile) fn.write('###') bf = BIDSFile(os.path.join(str(fn))) - tag_dict = { + entity_dict = { 'task': 'rest', 'run': 2, 'subject': '3' } - ents = {name: Entity(name) for name in tag_dict.keys()} - tags = [Tag(bf, ents[k], value=v) - for k, v in tag_dict.items()] - - session.add_all(list(ents.values()) + tags + [bf]) - session.commit() return bf From 0629ddb59522b80f94572fdf6c251b58b81d80e6 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:30:13 -0600 Subject: [PATCH 11/23] Fix typo --- bids/layout/layout.py | 2 +- bids/layout/tests/test_writing.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index e3cd2bcca..eeae0c78e 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -349,7 +349,7 @@ def parse_file_entities(self, filename, scope=None, entities=None, results = {e: results[e] for e in entities if e in results} if include_unmatched: - for k in set(self.get_entities()) - set(results)): + for k in set(self.get_entities()) - set(results): results[k] = None if scope is not None or config is not None: diff --git a/bids/layout/tests/test_writing.py b/bids/layout/tests/test_writing.py index 092dd765f..dacb58c14 100644 --- a/bids/layout/tests/test_writing.py +++ b/bids/layout/tests/test_writing.py @@ -18,12 +18,6 @@ def writable_file(tmpdir): fn = tmpdir.mkdir("tmp").join(testfile) fn.write('###') bf = BIDSFile(os.path.join(str(fn))) - - entity_dict = { - 'task': 'rest', - 'run': 2, - 'subject': '3' - } return bf From 51ff9083f9ffa6c531a83b17ad9a5c1ef97c253c Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:31:05 -0600 Subject: [PATCH 12/23] from ancpbids.utils --- bids/layout/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index 2270bc604..d08445544 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -4,7 +4,7 @@ from pathlib import Path import json from ancpbids.model_v1_8_0 import Artifact -from ancpbids import parse_bids_name +from ancpbids.utils import parse_bids_name from ..utils import listify from .writing import build_path, write_to_file From 4b952e4b5ef67458a7e5d7d1bbb9c6d4fb613042 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:35:52 -0600 Subject: [PATCH 13/23] Remove root from object return --- bids/layout/layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index eeae0c78e..405bf41dd 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -541,7 +541,7 @@ def get(self, return_type: str = 'object', target: str = None, scope: str = None result = natural_sort(result) if return_type == "object": result = natural_sort( - [BIDSFile.from_artifact(self.root, res) for res in result], + [BIDSFile.from_artifact(res) for res in result], "path" ) return result From cbef7255ec98c38777a2310dc617765cd71acbf2 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:36:47 -0600 Subject: [PATCH 14/23] pop syntax --- bids/layout/layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 405bf41dd..4a61dd6bf 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -341,7 +341,7 @@ def parse_file_entities(self, filename, scope=None, entities=None, values extracted from the filename. """ results = parse_bids_name(filename) - entities = results.pop['entities'] + entities = results.pop('entities') results = {**entities, **results} if entities: From b04941475703ecb30ef4777fb7a25b069a101ee2 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 12:38:16 -0600 Subject: [PATCH 15/23] _dirname not dirname --- bids/layout/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index d08445544..865cfe3f3 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -69,7 +69,7 @@ def is_dir(self): return not self._path.isdir() @property - def dirname(self): + def _dirname(self): return str(self._path.parent) def __repr__(self): From 06d28abe6fd7d51f00d1419f6cfb4eb7b22eed1f Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 16:08:01 -0600 Subject: [PATCH 16/23] Make BIDSFile.artifact private --- bids/layout/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index 865cfe3f3..c6181b9e0 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -11,7 +11,7 @@ from .utils import BIDSMetadata -class BIDSFile: +class BIDSFile(Artifact): """Represents a single file or directory in a BIDS dataset. Parameters @@ -43,16 +43,16 @@ def from_artifact(cls, artifact): def __init__(self, artifact=None, filename=None): if artifact is not None: - self.artifact = artifact + self._artifact = artifact elif filename is not None: - self.artifact = Artifact(**parse_bids_name(filename)) + self._artifact = Artifact(**parse_bids_name(filename)) else: raise ValueError("Either artifact or filename must be provided") @property def path(self): """ Convenience property for accessing path as a string.""" - return self.artifact.name + return self._artifact.name @property def _path(self): @@ -140,9 +140,9 @@ def get_entities(self, metadata=False, values='tags'): A dict, where keys are entity names and values are Entity instances. """ - entities = self.artifact.get_entities() + entities = self._artifact.get_entities() if metadata: - entities = {**entities, **self.artifact.get_metadata()} + entities = {**entities, **self._artifact.get_metadata()} if values == 'object': raise NotImplementedError From 3d76830a99ba9dfc10066ae99f295658f0b4f387 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 16:32:59 -0600 Subject: [PATCH 17/23] Remove artifact constructor --- bids/layout/models.py | 59 ++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index c6181b9e0..57515d514 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -16,8 +16,8 @@ class BIDSFile(Artifact): Parameters ---------- - filename : str - The path to the corresponding file. + file_ref : str + The path to the file or directory or an Artifact instance. """ _ext_registry = {} @@ -41,36 +41,37 @@ def from_artifact(cls, artifact): """ Load from ANCPBids Artifact """ return cls(artifact=artifact) - def __init__(self, artifact=None, filename=None): - if artifact is not None: - self._artifact = artifact - elif filename is not None: - self._artifact = Artifact(**parse_bids_name(filename)) - else: - raise ValueError("Either artifact or filename must be provided") + def __init__(self, file_ref: str | os.PathLike | Artifact): + self._path = None + self._artifact = None + if isinstance(file_ref, (str, os.PathLike)): + self._path = Path(file_ref) + elif isinstance(file_ref, Artifact): + self._artifact = file_ref @property def path(self): """ Convenience property for accessing path as a string.""" - return self._artifact.name - - @property - def _path(self): - """ Convenience property for accessing path as a Path object.""" - return Path(self.path) + try: + return self._artifact.get_absolute_path() + except AttributeError: + return str(self._path) @property def filename(self): """ Convenience property for accessing filename.""" - return self._path.name + try: + return self._artifact.name + except AttributeError: + return self._path.name @property def is_dir(self): - return not self._path.isdir() + return Path(self.path).is_dir() @property def _dirname(self): - return str(self._path.parent) + return str(Path(self.path).parent) def __repr__(self): return f"<{self.__class__.__name__} filename='{self.path}'>" @@ -79,9 +80,9 @@ def __fspath__(self): return self.path @property - def relpath(self, root): - """Return path relative to layout root""" - return str(self._path.relative_to(root)) + def relpath(self): + """ No longer have access to the BIDSLayout root directory """ + raise NotImplementedError def get_associations(self, kind=None, include_parents=False): """Get associated files, optionally limiting by association kind. @@ -140,9 +141,12 @@ def get_entities(self, metadata=False, values='tags'): A dict, where keys are entity names and values are Entity instances. """ - entities = self._artifact.get_entities() - if metadata: - entities = {**entities, **self._artifact.get_metadata()} + try: + entities = self._artifact.get_entities() + if metadata: + entities = {**entities, **self._artifact.get_metadata()} + except AttributeError: + raise NotImplementedError if values == 'object': raise NotImplementedError @@ -178,11 +182,8 @@ def copy(self, path_patterns, symbolic_link=False, root=None, if new_filename[-1] == os.sep: new_filename += self.filename - if self._path.is_absolute() or root is None: - path = self._path - else: - path = Path(root) / self._path - + path = Path(self.path) + if not path.exists(): raise ValueError("Target filename to copy/symlink (%s) doesn't " "exist." % path) From 0de0b428b4f421a235d405303c62097d5ad8ad48 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 16:33:11 -0600 Subject: [PATCH 18/23] Remove parse_file_entities --- bids/layout/utils.py | 54 -------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/bids/layout/utils.py b/bids/layout/utils.py index 1655aecab..13c45b9c8 100644 --- a/bids/layout/utils.py +++ b/bids/layout/utils.py @@ -88,60 +88,6 @@ def __hash__(self): return super().__hash__() -def parse_file_entities(filename, entities=None, config=None, - include_unmatched=False): - """Parse the passed filename for entity/value pairs. - - Parameters - ---------- - filename : str - The filename to parse for entity values - entities : list or None, optional - An optional list of Entity instances to use in extraction. - If passed, the config argument is ignored. Default is None. - config : str or :obj:`bids.layout.models.Config` or list or None, optional - One or more :obj:`bids.layout.models.Config` objects or names of - configurations to use in matching. Each element must be a - :obj:`bids.layout.models.Config` object, or a valid - :obj:`bids.layout.models.Config` name (e.g., 'bids' or 'derivatives'). - If None, all available configs are used. Default is None. - include_unmatched : bool, optional - If True, unmatched entities are included in the returned dict, - with values set to None. - If False (default), unmatched entities are ignored. - - Returns - ------- - dict - Keys are Entity names and values are the values from the filename. - """ - # Load Configs if needed - if entities is None: - - if config is None: - config = ['bids', 'derivatives'] - - from .models import Config - config = [Config.load(c) if not isinstance(c, Config) else c - for c in listify(config)] - - # Consolidate entities from all Configs into a single dict - entities = {} - for c in config: - entities.update(c.entities) - entities = entities.values() - - # Extract matches - bf = make_bidsfile(filename) - ent_vals = {} - for ent in entities: - match = ent.match_file(bf) - if match is not None or include_unmatched: - ent_vals[ent.name] = match - - return ent_vals - - def add_config_paths(**kwargs): """Add to the pool of available configuration files for BIDSLayout. From 4d9eca40bd65d2f1ffcd54a7aff983a3924e1e27 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 17:15:40 -0600 Subject: [PATCH 19/23] Fix types and utility removal --- bids/layout/__init__.py | 3 +-- bids/layout/models.py | 3 ++- bids/layout/tests/test_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bids/layout/__init__.py b/bids/layout/__init__.py index 3fdc05895..9eed659d0 100644 --- a/bids/layout/__init__.py +++ b/bids/layout/__init__.py @@ -1,7 +1,7 @@ from .layout import BIDSLayout, Query from .models import (BIDSFile, BIDSImageFile, BIDSDataFile, BIDSJSONFile) from .index import BIDSLayoutIndexer -from .utils import add_config_paths, parse_file_entities +from .utils import add_config_paths # Backwards compatibility from bids_validator import BIDSValidator @@ -10,7 +10,6 @@ "BIDSLayoutIndexer", "BIDSValidator", "add_config_paths", - "parse_file_entities", "BIDSFile", "BIDSImageFile", "BIDSDataFile", diff --git a/bids/layout/models.py b/bids/layout/models.py index 57515d514..e16f10f02 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -1,6 +1,7 @@ """ Model classes used in BIDSLayouts. """ import os +from typing import Union from pathlib import Path import json from ancpbids.model_v1_8_0 import Artifact @@ -41,7 +42,7 @@ def from_artifact(cls, artifact): """ Load from ANCPBids Artifact """ return cls(artifact=artifact) - def __init__(self, file_ref: str | os.PathLike | Artifact): + def __init__(self, file_ref: Union[str, os.PathLike, Artifact]): self._path = None self._artifact = None if isinstance(file_ref, (str, os.PathLike)): diff --git a/bids/layout/tests/test_utils.py b/bids/layout/tests/test_utils.py index 1023fb769..af36c8e3c 100644 --- a/bids/layout/tests/test_utils.py +++ b/bids/layout/tests/test_utils.py @@ -7,7 +7,7 @@ from bids.exceptions import ConfigError from ..models import Entity, Config -from ..utils import BIDSMetadata, parse_file_entities, add_config_paths +from ..utils import BIDSMetadata, add_config_paths def test_bidsmetadata_class(): From d29da20943e05e1ec04c761c5125fbb79e54f548 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Tue, 13 Dec 2022 17:17:58 -0600 Subject: [PATCH 20/23] constructorg --- bids/layout/layout.py | 2 +- bids/layout/models.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/bids/layout/layout.py b/bids/layout/layout.py index 4a61dd6bf..e9a8463c3 100644 --- a/bids/layout/layout.py +++ b/bids/layout/layout.py @@ -541,7 +541,7 @@ def get(self, return_type: str = 'object', target: str = None, scope: str = None result = natural_sort(result) if return_type == "object": result = natural_sort( - [BIDSFile.from_artifact(res) for res in result], + [BIDSFile(res) for res in result], "path" ) return result diff --git a/bids/layout/models.py b/bids/layout/models.py index e16f10f02..c7cf31d24 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -35,12 +35,7 @@ def from_filename(cls, filename): if path.name.endswith(ext): cls = subclass break - return cls(filename=path) - - @classmethod - def from_artifact(cls, artifact): - """ Load from ANCPBids Artifact """ - return cls(artifact=artifact) + return cls(path) def __init__(self, file_ref: Union[str, os.PathLike, Artifact]): self._path = None From d2d12991b5dfc96ccb1c52fb960c9425cbf86ccf Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Wed, 14 Dec 2022 10:22:32 -0600 Subject: [PATCH 21/23] Don't inherent from Artifact --- bids/layout/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index c7cf31d24..7c1ef96ca 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -12,7 +12,7 @@ from .utils import BIDSMetadata -class BIDSFile(Artifact): +class BIDSFile(): """Represents a single file or directory in a BIDS dataset. Parameters From 2a5bb5124ed2d3fe0b5bebf80f8a6c0200b2ff20 Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Wed, 14 Dec 2022 10:36:53 -0600 Subject: [PATCH 22/23] Update bids/layout/models.py Co-authored-by: Chris Markiewicz --- bids/layout/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index 7c1ef96ca..ff12c845d 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -12,7 +12,7 @@ from .utils import BIDSMetadata -class BIDSFile(): +class BIDSFile: """Represents a single file or directory in a BIDS dataset. Parameters From 67ea3d143d558665704b4e8bed9f2c985b99221d Mon Sep 17 00:00:00 2001 From: Alejandro de la Vega Date: Wed, 14 Dec 2022 10:37:35 -0600 Subject: [PATCH 23/23] Update bids/layout/models.py Co-authored-by: Chris Markiewicz --- bids/layout/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids/layout/models.py b/bids/layout/models.py index ff12c845d..a01f1cf0d 100644 --- a/bids/layout/models.py +++ b/bids/layout/models.py @@ -144,7 +144,7 @@ def get_entities(self, metadata=False, values='tags'): except AttributeError: raise NotImplementedError - if values == 'object': + if values != 'tags': raise NotImplementedError return entities