Merge pull request #114 from znamlab/dev

v0.3.8

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Change log
 
+## v0.3.8
+
+### Main changes
+
+- Add `get_data_root` function to get `raw` or `processed` root for a project
+- `get_children` can filter children by attributes before returning results
+- refactor `get_datasets` to be non recursive and add filtering options. Also add 
+  multiple options to filter datasets and format output
+- add `get_datasets_recursively` to get all datasets below a given entity
+
+### Bugfixes
+
+- return empty dataframe if `filter` in `get_children` filters out everything (instead 
+  of crashing)
+- `update_flexilims` correctly uploads tuples parameters
+- `update_flexilims` correctly uploads floats and np.float/np.int parameters
+- `update_flexilims` can overwrite existing datasets (if `conflicts='overwrite'`)
+- Add filelock for token creation to avoid concurrent access and move token to their own
+  file
+
+### Minor
+
+- `harp_dataset.from_folder` will now match csv even if there is nothing before or after
+  `harpmessage` in the file name (i.e. the file is `harpmessage.bin`, and all csvs in 
+  the folder will be matched)
+- private function `config_tools._find_files` has now  a `create_if_missing` argument to 
+  create the file if it does not exist
+
 ## v0.3.7
 
 ### Main changes

flexiznam/camp/sync_data.py

@@ -171,6 +171,7 @@ def upload_yaml(
         list of names of entities created/updated
 
     """
+
     output = []
     # if there are errors, I cannot safely parse the yaml
     errors = find_xxerrorxx(yml_file=source_yaml)
@@ -232,12 +233,17 @@ def upload_yaml(
         root_id = mouse["id"]
 
     # session datasets
+    # use "overwrite" as mode if conflict is "overwrite", otherwise use "safe" mode
+    if conflicts == "overwrite":
+        mode = "overwrite"
+    else:
+        mode = "safe"
     for ds_name, ds in session_data.get("datasets", {}).items():
         ds.genealogy = [mouse["name"], session_data["session"], ds_name]
         ds.project = session_data["project"]
         ds.origin_id = root_id
         ds.flexilims_session = flexilims_session
-        ds.update_flexilims(mode="safe")
+        ds.update_flexilims(mode=mode)
         output.append(ds.full_name)
 
     # now deal with recordings
@@ -276,7 +282,7 @@ def upload_yaml(
             ds.project = session_data["project"]
             ds.origin_id = rec_rep["id"]
             ds.flexilims_session = flexilims_session
-            ds.update_flexilims(mode="safe")
+            ds.update_flexilims(mode=mode)
             output.append(ds.full_name)
 
     # now deal with samples

flexiznam/config/config_tools.py

@@ -10,7 +10,7 @@
 from getpass import getpass
 
 
-def _find_file(file_name, config_folder=None):
+def _find_file(file_name, config_folder=None, create_if_missing=False):
     """Find a file by looking at various places
 
     Only in config_folder (if provided)
@@ -19,25 +19,37 @@ def _find_file(file_name, config_folder=None):
     - then in the ~/.config folder
     - then in the folder contain the file defining this function
     - then in sys.path
+
+    Args:
+        file_name (str): name of the file to find
+        config_folder (str): folder to look for the file
+        create_if_missing (bool): if True, create the file in the config folder if it
+            does not exist. If False, raise an error if the file is not found
     """
     if config_folder is not None:
         in_config_folder = Path(config_folder) / file_name
         if in_config_folder.is_file():
             return in_config_folder
-        raise ConfigurationError("Cannot find %s in %s" % (file_name, config_folder))
-    local = Path.cwd() / file_name
-    if local.is_file():
-        return local
-    config = Path(__file__).parent.absolute() / "config" / file_name
-    home = Path.home() / ".flexiznam"
-    if home.is_dir() and (home / file_name).is_file():
-        return home / file_name
-    if config.is_file():
-        return config
-    for directory in sys.path:
-        fname = Path(directory) / file_name
-        if fname.is_file():
-            return fname
+        missing_file = in_config_folder
+    else:
+        local = Path.cwd() / file_name
+        if local.is_file():
+            return local
+        config = Path(__file__).parent.absolute() / "config" / file_name
+        home = Path.home() / ".flexiznam"
+        if home.is_dir() and (home / file_name).is_file():
+            return home / file_name
+        if config.is_file():
+            return config
+        for directory in sys.path:
+            fname = Path(directory) / file_name
+            if fname.is_file():
+                return fname
+        missing_file = home / file_name
+    if create_if_missing:
+        with open(missing_file, "w") as f:
+            f.write("")
+        return missing_file
     raise ConfigurationError("Cannot find %s" % file_name)
 
 

flexiznam/config/default_config.py

@@ -14,6 +14,12 @@
         "test": "606df1ac08df4d77c72c9aa4",
         "AAVRKO_retina_hva": "60a7757a8901a2357f29080a",
     },
+    project_paths={
+        "example": dict(
+            raw="/camp/project/example_project/raw",
+            processed="/camp/project/example_project/processed",
+        )
+    },
     # a default username can be specified
     flexilims_username="yourusername",
     # Use for data dataset detection