Merge branch 'copy' into iblrigv8

docs/source/description_file.md

@@ -11,7 +11,7 @@ is, therefore, very important!
 Here is an example of a complete experiment description file for a mesoscope experiment running
 two consecutive tasks, `biasedChoiceWorld` followed by `passiveChoiceWorld`. 
 
-```
+```yaml
 devices:
   mesoscope:
     mesoscope:
@@ -58,7 +58,7 @@ The devices section in the experiment description file lists the set of devices
 the experiment. Supported devices are Cameras, Microphone, Mesoscope, Neuropixel, Photometry and Widefield.
 
 The convention for this section is to have the device name followed by a list of sub-devices, e.g.
-```
+```yaml
 devices:
   cameras:
     belly:
@@ -80,7 +80,7 @@ In the above example, `cameras` is the device and the sub-devices are `belly`, `
 
 If there are no sub-devices, the sub-device is given the same name as the device, e.g.
 
-```
+```yaml
 devices:
   mesoscope:
     mesoscope:
@@ -101,7 +101,7 @@ The procedures section lists the set of procedures that apply to this experiment
 procedures can be found [here](https://alyx.internationalbrainlab.org/admin/actions/proceduretype/).
 
 As many procedure that apply to the experiment can be added e.g.
-```
+```yaml
 procedures:
 - Fiber photometry
 - Optical stimulation
@@ -127,14 +127,25 @@ description file and act as the main clock to which other timeseries are synced.
 
 An example of an experiment run with bpod as the main syncing device is, 
 
-```
+```yaml
 sync:
   bpod:
     collection: raw_behavior_data
     extension: bin
 
 ```
 
+Another example for spikeglx electrophysiology recordings with Neuropixel 1B probes use the
+nidq as main synchronisation.
+
+```yaml
+sync:
+  nidq:
+    collection: raw_ephys_data
+    extension: bin
+    acquisition_software: spikeglx
+```
+
 Each sync device must have at least the following two keys
 - **collection** - the folder containing the data
 - **extension** - the file extension of the sync data
@@ -145,7 +156,7 @@ Optional keys include, for example `acquisition_software`, the software used to
 
 The tasks section contains a list of the behavioral protocols run during the experiment. The name of the
 protocol must be given in the list e.g.
-```
+```yaml
 tasks:
 - _biasedChoiceWorld:
     collection: raw_task_data_00

iblrig/test/test_transfers.py

@@ -93,6 +93,7 @@ def test_behavior_ephys_video_copy(self):
             assert ec.state == 0
             ec.initialize_experiment()
             assert ec.state == 1
+            assert 'sync' in ec.experiment_description
             ec.copy_collections()
             assert ec.state == 2
             sc.finalize_copy(number_of_expected_devices=3)

iblrig/transfer_experiments.py

@@ -2,7 +2,6 @@
 import shutil
 import traceback
 
-from one.alf.files import filename_parts
 from iblutil.util import setup_logger
 from ibllib.io import session_params
 from ibllib.pipes.misc import rsync_paths
@@ -14,12 +13,12 @@
 
 class SessionCopier():
     tag = 'behavior'
+    assert_connect_on_init = False
 
-    def __init__(self, session_path, remote_subjects_folder=None):
+    def __init__(self, session_path, remote_subjects_folder=None, tag=None):
+        self.tag = tag or self.tag
         self.session_path = Path(session_path)
         self.remote_subjects_folder = Path(remote_subjects_folder) if remote_subjects_folder else None
-        self.experiment_description = session_params.read_params(self.session_path)
-        self.remote_experiment_description_stub = session_params.read_params(self.session_path)
 
     def __repr__(self):
         return f"{super(SessionCopier, self).__repr__()} \n local: {self.session_path} \n remote: {self.remote_session_path}"
@@ -45,6 +44,10 @@ def get_state(self):
         elif status_file.name.endswith('final'):
             return 3, f'Copy finalized {self.file_remote_experiment_description}'
 
+    @property
+    def experiment_description(self):
+        return session_params.read_params(self.session_path)
+
     @property
     def remote_session_path(self):
         if self.remote_subjects_folder:
@@ -69,7 +72,11 @@ def glob_file_remote_copy_status(self, status='*'):
     def file_remote_experiment_description(self):
         if self.remote_subjects_folder:
             return session_params.get_remote_stub_name(
-                self.remote_session_path, filename_parts(self.file_experiment_description.name)[3])
+                self.remote_session_path, device_id=self.tag)
+
+    @property
+    def remote_experiment_description_stub(self):
+        return session_params.read_params(self.file_remote_experiment_description)
 
     def _copy_collections(self):
         """
@@ -136,8 +143,10 @@ def initialize_experiment(self, acquisition_description=None, overwrite=False):
                 session_params.write_yaml(remote_stub_file, merged_description)
                 remote_stub_file.with_suffix('.status_pending').touch()
                 log.info(f'Written data to remote device at: {remote_stub_file}.')
-            except Exception as ex:
-                log.warning(f'Failed to write data to remote device at: {remote_stub_file}. \n {ex}')
+            except Exception as e:
+                if self.assert_connect_on_init:
+                    raise Exception(f'Failed to write data to remote device at: {remote_stub_file}. \n {e}') from e
+                log.warning(f'Failed to write data to remote device at: {remote_stub_file}. \n {e}')
 
         # then create on the local machine
         previous_description = session_params.read_params(self.file_experiment_description)\
@@ -156,6 +165,7 @@ def finalize_copy(self, number_of_expected_devices=None):
         files_stub = list(self.file_remote_experiment_description.parent.glob('*.yaml'))
         for file_stub in files_stub:
             ready_to_finalize += int(file_stub.with_suffix('.status_complete').exists())
+        log.info(f"{ready_to_finalize}/{number_of_expected_devices} copy completion status")
         if ready_to_finalize == number_of_expected_devices:
             for file_stub in files_stub:
                 session_params.aggregate_device(
@@ -166,6 +176,7 @@ def finalize_copy(self, number_of_expected_devices=None):
 
 class VideoCopier(SessionCopier):
     tag = 'video'
+    assert_connect_on_init = True
 
     def initialize_experiment(self, acquisition_description=None, **kwargs):
         if not acquisition_description:
@@ -176,6 +187,7 @@ def initialize_experiment(self, acquisition_description=None, **kwargs):
 
 class EphysCopier(SessionCopier):
     tag = 'spikeglx'
+    assert_connect_on_init = True
 
     def initialize_experiment(self, acquisition_description=None, nprobes=None, **kwargs):
         if not acquisition_description:
@@ -184,7 +196,9 @@ def initialize_experiment(self, acquisition_description=None, nprobes=None, **kw
                 case 1: stub_name = 'neuropixel_single_probe.yaml'
                 case 2: stub_name = 'neuropixel_dual_probe.yaml'
             stub_file = Path(deploy.ephyspc.__file__).parent.joinpath('device_stubs', stub_name)
+            sync_file = Path(deploy.ephyspc.__file__).parent.joinpath('device_stubs', 'sync_nidq_raw_ephys_data.yaml')
             acquisition_description = session_params.read_params(stub_file)
+            acquisition_description.update(session_params.read_params(sync_file))
         super(EphysCopier, self).initialize_experiment(acquisition_description=acquisition_description, **kwargs)
 
     def _copy_collections(self):