From 7c16cb25e52cb23bf5f88e0350f6c94dd46e0cd8 Mon Sep 17 00:00:00 2001 From: olivier Date: Fri, 21 Jul 2023 08:49:55 +0100 Subject: [PATCH 1/3] copy ephys: tag in remote and force connection to server option --- iblrig/transfer_experiments.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/iblrig/transfer_experiments.py b/iblrig/transfer_experiments.py index db6bf3530..240729634 100644 --- a/iblrig/transfer_experiments.py +++ b/iblrig/transfer_experiments.py @@ -2,7 +2,6 @@ import shutil import traceback -from one.alf.files import filename_parts from iblutil.util import setup_logger from ibllib.io import session_params from ibllib.pipes.misc import rsync_paths @@ -14,12 +13,14 @@ class SessionCopier(): tag = 'behavior' + assert_connect_on_init = False - def __init__(self, session_path, remote_subjects_folder=None): + def __init__(self, session_path, remote_subjects_folder=None, tag=None): + # TODO: allow instantiate with no local path to check status from local server + self.tag = tag or self.tag self.session_path = Path(session_path) self.remote_subjects_folder = Path(remote_subjects_folder) if remote_subjects_folder else None self.experiment_description = session_params.read_params(self.session_path) - self.remote_experiment_description_stub = session_params.read_params(self.session_path) def __repr__(self): return f"{super(SessionCopier, self).__repr__()} \n local: {self.session_path} \n remote: {self.remote_session_path}" @@ -69,7 +70,11 @@ def glob_file_remote_copy_status(self, status='*'): def file_remote_experiment_description(self): if self.remote_subjects_folder: return session_params.get_remote_stub_name( - self.remote_session_path, filename_parts(self.file_experiment_description.name)[3]) + self.remote_session_path, device_id=self.tag) + + @property + def remote_experiment_description_stub(self): + return session_params.read_params(self.file_remote_experiment_description) def _copy_collections(self): """ @@ -136,8 +141,10 @@ def initialize_experiment(self, acquisition_description=None, overwrite=False): session_params.write_yaml(remote_stub_file, merged_description) remote_stub_file.with_suffix('.status_pending').touch() log.info(f'Written data to remote device at: {remote_stub_file}.') - except Exception as ex: - log.warning(f'Failed to write data to remote device at: {remote_stub_file}. \n {ex}') + except Exception as e: + if self.assert_connect_on_init: + raise Exception(f'Failed to write data to remote device at: {remote_stub_file}. \n {e}') from e + log.warning(f'Failed to write data to remote device at: {remote_stub_file}. \n {e}') # then create on the local machine previous_description = session_params.read_params(self.file_experiment_description)\ @@ -166,6 +173,7 @@ def finalize_copy(self, number_of_expected_devices=None): class VideoCopier(SessionCopier): tag = 'video' + assert_connect_on_init = True def initialize_experiment(self, acquisition_description=None, **kwargs): if not acquisition_description: @@ -176,6 +184,7 @@ def initialize_experiment(self, acquisition_description=None, **kwargs): class EphysCopier(SessionCopier): tag = 'spikeglx' + assert_connect_on_init = True def initialize_experiment(self, acquisition_description=None, nprobes=None, **kwargs): if not acquisition_description: From b6be108ab149cc8cb0c920562f10d1a0f8432981 Mon Sep 17 00:00:00 2001 From: olivier Date: Fri, 21 Jul 2023 09:09:53 +0100 Subject: [PATCH 2/3] add ephys sync field to NP1 stubs --- docs/source/description_file.md | 23 +++++++++++++++++------ iblrig/test/test_transfers.py | 1 + iblrig/transfer_experiments.py | 8 ++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/source/description_file.md b/docs/source/description_file.md index 7dc8991fc..2913185e3 100644 --- a/docs/source/description_file.md +++ b/docs/source/description_file.md @@ -11,7 +11,7 @@ is, therefore, very important! Here is an example of a complete experiment description file for a mesoscope experiment running two consecutive tasks, `biasedChoiceWorld` followed by `passiveChoiceWorld`. -``` +```yaml devices: mesoscope: mesoscope: @@ -58,7 +58,7 @@ The devices section in the experiment description file lists the set of devices the experiment. Supported devices are Cameras, Microphone, Mesoscope, Neuropixel, Photometry and Widefield. The convention for this section is to have the device name followed by a list of sub-devices, e.g. -``` +```yaml devices: cameras: belly: @@ -80,7 +80,7 @@ In the above example, `cameras` is the device and the sub-devices are `belly`, ` If there are no sub-devices, the sub-device is given the same name as the device, e.g. -``` +```yaml devices: mesoscope: mesoscope: @@ -101,7 +101,7 @@ The procedures section lists the set of procedures that apply to this experiment procedures can be found [here](https://alyx.internationalbrainlab.org/admin/actions/proceduretype/). As many procedure that apply to the experiment can be added e.g. -``` +```yaml procedures: - Fiber photometry - Optical stimulation @@ -127,7 +127,7 @@ description file and act as the main clock to which other timeseries are synced. An example of an experiment run with bpod as the main syncing device is, -``` +```yaml sync: bpod: collection: raw_behavior_data @@ -135,6 +135,17 @@ sync: ``` +Another example for spikeglx electrophysiology recordings with Neuropixel 1B probes use the +nidq as main synchronisation. + +```yaml +sync: + nidq: + collection: raw_ephys_data + extension: bin + acquisition_software: spikeglx +``` + Each sync device must have at least the following two keys - **collection** - the folder containing the data - **extension** - the file extension of the sync data @@ -145,7 +156,7 @@ Optional keys include, for example `acquisition_software`, the software used to The tasks section contains a list of the behavioral protocols run during the experiment. The name of the protocol must be given in the list e.g. -``` +```yaml tasks: - _biasedChoiceWorld: collection: raw_task_data_00 diff --git a/iblrig/test/test_transfers.py b/iblrig/test/test_transfers.py index a4b1aa178..d6730323e 100644 --- a/iblrig/test/test_transfers.py +++ b/iblrig/test/test_transfers.py @@ -93,6 +93,7 @@ def test_behavior_ephys_video_copy(self): assert ec.state == 0 ec.initialize_experiment() assert ec.state == 1 + assert 'sync' in ec.experiment_description ec.copy_collections() assert ec.state == 2 sc.finalize_copy(number_of_expected_devices=3) diff --git a/iblrig/transfer_experiments.py b/iblrig/transfer_experiments.py index 240729634..c3e87945d 100644 --- a/iblrig/transfer_experiments.py +++ b/iblrig/transfer_experiments.py @@ -16,11 +16,9 @@ class SessionCopier(): assert_connect_on_init = False def __init__(self, session_path, remote_subjects_folder=None, tag=None): - # TODO: allow instantiate with no local path to check status from local server self.tag = tag or self.tag self.session_path = Path(session_path) self.remote_subjects_folder = Path(remote_subjects_folder) if remote_subjects_folder else None - self.experiment_description = session_params.read_params(self.session_path) def __repr__(self): return f"{super(SessionCopier, self).__repr__()} \n local: {self.session_path} \n remote: {self.remote_session_path}" @@ -46,6 +44,10 @@ def get_state(self): elif status_file.name.endswith('final'): return 3, f'Copy finalized {self.file_remote_experiment_description}' + @property + def experiment_description(self): + return session_params.read_params(self.session_path) + @property def remote_session_path(self): if self.remote_subjects_folder: @@ -193,7 +195,9 @@ def initialize_experiment(self, acquisition_description=None, nprobes=None, **kw case 1: stub_name = 'neuropixel_single_probe.yaml' case 2: stub_name = 'neuropixel_dual_probe.yaml' stub_file = Path(deploy.ephyspc.__file__).parent.joinpath('device_stubs', stub_name) + sync_file = Path(deploy.ephyspc.__file__).parent.joinpath('device_stubs', 'sync_nidq_raw_ephys_data.yaml') acquisition_description = session_params.read_params(stub_file) + acquisition_description.update(session_params.read_params(sync_file)) super(EphysCopier, self).initialize_experiment(acquisition_description=acquisition_description, **kwargs) def _copy_collections(self): From 2b55e4c67bcffd27ab0cdff3ac951999f3e6b88b Mon Sep 17 00:00:00 2001 From: olivier Date: Fri, 21 Jul 2023 10:38:55 +0100 Subject: [PATCH 3/3] add some logging --- iblrig/transfer_experiments.py | 1 + 1 file changed, 1 insertion(+) diff --git a/iblrig/transfer_experiments.py b/iblrig/transfer_experiments.py index c3e87945d..f4d1737c9 100644 --- a/iblrig/transfer_experiments.py +++ b/iblrig/transfer_experiments.py @@ -165,6 +165,7 @@ def finalize_copy(self, number_of_expected_devices=None): files_stub = list(self.file_remote_experiment_description.parent.glob('*.yaml')) for file_stub in files_stub: ready_to_finalize += int(file_stub.with_suffix('.status_complete').exists()) + log.info(f"{ready_to_finalize}/{number_of_expected_devices} copy completion status") if ready_to_finalize == number_of_expected_devices: for file_stub in files_stub: session_params.aggregate_device(