Merge pull request #716 from int-brain-lab/iblrigv8dev

8.24.1

.github/workflows/documentation.yaml

@@ -25,6 +25,8 @@ jobs:
           python-version-file: pyproject.toml
       - name: Install requirements
         run: pdm sync -dG doc
+      - name: Install PortAudio and GraphViz
+        run: sudo apt-get install -y libportaudio2 graphviz
       - name: Sphinx build
         run: pdm run sphinx-build docs/source docs/build/html
       - name: Deploy

CHANGELOG.md

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+8.24.1
+------
+* change UI workflow for appending a session
+* Video QC: changed log level from WARNING to INFO if less than 0.1% of frames have been dropped
+* Valve: Use restricted quadratic fit when converting reward volume to valve opening time
+* TrainingCW: add `signed_contrast` to trials_table definition
+
 8.24.0
 ------
 * feature: validate values in `trials_table` using Pydantic

docs/source/usage_behavior.rst

@@ -42,9 +42,9 @@ Starting a Task
 Supplementary Controls
 ~~~~~~~~~~~~~~~~~~~~~~
 
--  If you check the *Append* option before clicking *Start*, the task
-   you initiate will be linked to the preceding task, creating a
-   sequence of connected tasks.
+-  When starting a subsequent task with the same subject, you'll be asked if
+   you want to append to the preceding session. Doing so will result in a
+   sequence of connected tasks sharing the same data folder.
 
 -  The *Flush* button serves to toggle the valve for cleaning purposes.
 

iblrig/__init__.py

@@ -6,7 +6,7 @@
 # 5) git tag the release in accordance to the version number below (after merge!)
 # >>> git tag 8.15.6
 # >>> git push origin --tags
-__version__ = '8.24.0'
+__version__ = '8.24.1'
 
 
 from iblrig.version_management import get_detailed_version_string

iblrig/base_choice_world.py

@@ -99,7 +99,7 @@ class ChoiceWorldTrialData(TrialDataModel):
     # TODO: Yes, this should probably be done differently.
     response_side: Annotated[int, Interval(ge=0, le=0)] = 0
     response_time: IsNan[float] = np.nan
-    trial_correct: Annotated[int, Interval(ge=0, le=0)] = False
+    trial_correct: Annotated[bool, Interval(ge=0, le=0)] = False
 
 
 class ChoiceWorldSession(
@@ -889,6 +889,7 @@ class TrainingChoiceWorldTrialData(ActiveChoiceWorldTrialData):
 
     training_phase: NonNegativeInt
     debias_trial: bool
+    signed_contrast: float | None = None
 
 
 class TrainingChoiceWorldSession(ActiveChoiceWorldSession):

iblrig/base_tasks.py

@@ -24,7 +24,6 @@
 
 import numpy as np
 import pandas as pd
-import scipy.interpolate
 import serial
 import yaml
 from pythonosc import udp_client
@@ -43,6 +42,7 @@
 from iblrig.pydantic_definitions import HardwareSettings, RigSettings, TrialDataModel
 from iblrig.tools import call_bonsai
 from iblrig.transfer_experiments import BehaviorCopier, VideoCopier
+from iblrig.valve import Valve
 from iblutil.io.net.base import ExpMessage
 from iblutil.spacer import Spacer
 from iblutil.util import Bunch, flatten, setup_logger
@@ -1025,43 +1025,32 @@ def start_mixin_rotary_encoder(self):
 
 class ValveMixin(BaseSession, HasBpod):
     def init_mixin_valve(self: object):
-        self.valve = Bunch({})
-        # the template settings files have a date in 2099, so assume that the rig is not calibrated if that is the case
-        # the assertion on calibration is thrown when starting the device
-        self.valve['is_calibrated'] = datetime.date.today() >= self.hardware_settings['device_valve']['WATER_CALIBRATION_DATE']
-        self.valve['fcn_vol2time'] = scipy.interpolate.pchip(
-            self.hardware_settings['device_valve']['WATER_CALIBRATION_WEIGHT_PERDROP'],
-            self.hardware_settings['device_valve']['WATER_CALIBRATION_OPEN_TIMES'],
-        )
+        self.valve = Valve(self.hardware_settings.device_valve)
 
     def start_mixin_valve(self):
-        # if the rig is not on manual settings, then the reward valve has to be calibrated to run the experiment
-        assert self.task_params.AUTOMATIC_CALIBRATION is False or self.valve['is_calibrated'], """
-            ##########################################
-            NO CALIBRATION INFORMATION FOUND IN HARDWARE SETTINGS:
-            Calibrate the rig or use a manual calibration
-            PLEASE GO TO the task settings yaml file and set:
-                'AUTOMATIC_CALIBRATION': false
-                'CALIBRATION_VALUE' = <MANUAL_CALIBRATION>
-            ##########################################"""
+        # assert that valve has been calibrated
+        assert self.valve.is_calibrated, """VALVE IS NOT CALIBRATED - PLEASE CALIBRATE THE VALVE"""
+
         # regardless of the calibration method, the reward valve time has to be lower than 1 second
-        assert self.compute_reward_time(amount_ul=1.5) < 1, """
-            ##########################################
-                REWARD VALVE TIME IS TOO HIGH!
-            Probably because of a BAD calibration file
-            Calibrate the rig or use a manual calibration
-            PLEASE GO TO the task settings yaml file and set:
-                AUTOMATIC_CALIBRATION = False
-                CALIBRATION_VALUE = <MANUAL_CALIBRATION>
-            ##########################################"""
+        assert self.compute_reward_time(amount_ul=1.5) < 1, """VALVE IS NOT PROPERLY CALIBRATED - PLEASE RECALIBRATE"""
         log.info('Water valve module loaded: OK')
 
-    def compute_reward_time(self, amount_ul=None):
+    def compute_reward_time(self, amount_ul: float | None = None) -> float:
+        """
+        Converts the valve opening time from a given volume.
+
+        Parameters
+        ----------
+        amount_ul : float, optional
+            The volume of liquid (μl) to be dispensed from the valve. Defaults to task_params.REWARD_AMOUNT_UL.
+
+        Returns
+        -------
+        float
+            Valve opening time in seconds.
+        """
         amount_ul = self.task_params.REWARD_AMOUNT_UL if amount_ul is None else amount_ul
-        if self.task_params.AUTOMATIC_CALIBRATION:
-            return self.valve['fcn_vol2time'](amount_ul) / 1e3
-        else:  # this is the manual manual calibration value
-            return self.task_params.CALIBRATION_VALUE / 3 * amount_ul
+        return self.valve.values.ul2ms(amount_ul) / 1e3
 
     def valve_open(self, reward_valve_time):
         """

iblrig/gui/ui_wizard.py

@@ -276,15 +276,22 @@ def setupUi(self, wizard):
         self.uiGroupSessionControl.setObjectName("uiGroupSessionControl")
         self.verticalLayout = QtWidgets.QVBoxLayout(self.uiGroupSessionControl)
         self.verticalLayout.setObjectName("verticalLayout")
-        self.uiCheckAppend = QtWidgets.QCheckBox(self.uiGroupSessionControl)
-        self.uiCheckAppend.setObjectName("uiCheckAppend")
-        self.verticalLayout.addWidget(self.uiCheckAppend)
         self.uiPushStart = QtWidgets.QPushButton(self.uiGroupSessionControl)
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.uiPushStart.sizePolicy().hasHeightForWidth())
+        self.uiPushStart.setSizePolicy(sizePolicy)
         self.uiPushStart.setStyleSheet("QPushButton { background-color: red; }")
         self.uiPushStart.setObjectName("uiPushStart")
         self.verticalLayout.addWidget(self.uiPushStart)
         self.uiPushPause = QtWidgets.QPushButton(self.uiGroupSessionControl)
         self.uiPushPause.setEnabled(False)
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.uiPushPause.sizePolicy().hasHeightForWidth())
+        self.uiPushPause.setSizePolicy(sizePolicy)
         self.uiPushPause.setCheckable(True)
         self.uiPushPause.setChecked(False)
         self.uiPushPause.setObjectName("uiPushPause")
@@ -346,8 +353,7 @@ def setupUi(self, wizard):
         wizard.setTabOrder(self.listViewRemoteDevices, self.uiPushFlush)
         wizard.setTabOrder(self.uiPushFlush, self.uiPushReward)
         wizard.setTabOrder(self.uiPushReward, self.uiPushStatusLED)
-        wizard.setTabOrder(self.uiPushStatusLED, self.uiCheckAppend)
-        wizard.setTabOrder(self.uiCheckAppend, self.uiPushStart)
+        wizard.setTabOrder(self.uiPushStatusLED, self.uiPushStart)
         wizard.setTabOrder(self.uiPushStart, self.uiPushPause)
 
     def retranslateUi(self, wizard):
@@ -380,8 +386,6 @@ def retranslateUi(self, wizard):
         self.uiPushStatusLED.setStatusTip(_translate("wizard", "Click to toggle the Bpod\'s status LED"))
         self.uiPushStatusLED.setText(_translate("wizard", " Status &LED   "))
         self.uiGroupSessionControl.setTitle(_translate("wizard", "Session Control"))
-        self.uiCheckAppend.setStatusTip(_translate("wizard", "append to previous session"))
-        self.uiCheckAppend.setText(_translate("wizard", "append to previous Session"))
         self.uiPushStart.setStatusTip(_translate("wizard", "Click to start the session"))
         self.uiPushStart.setText(_translate("wizard", "Start"))
         self.uiPushPause.setStatusTip(_translate("wizard", "Click to pause the session after the current trial"))

iblrig/gui/ui_wizard.ui

@@ -687,18 +687,14 @@
               <string>Session Control</string>
              </property>
              <layout class="QVBoxLayout" name="verticalLayout">
-              <item>
-               <widget class="QCheckBox" name="uiCheckAppend">
-                <property name="statusTip">
-                 <string>append to previous session</string>
-                </property>
-                <property name="text">
-                 <string>append to previous Session</string>
-                </property>
-               </widget>
-              </item>
               <item>
                <widget class="QPushButton" name="uiPushStart">
+                <property name="sizePolicy">
+                 <sizepolicy hsizetype="Minimum" vsizetype="Expanding">
+                  <horstretch>0</horstretch>
+                  <verstretch>0</verstretch>
+                 </sizepolicy>
+                </property>
                 <property name="statusTip">
                  <string>Click to start the session</string>
                 </property>
@@ -715,6 +711,12 @@
                 <property name="enabled">
                  <bool>false</bool>
                 </property>
+                <property name="sizePolicy">
+                 <sizepolicy hsizetype="Minimum" vsizetype="Expanding">
+                  <horstretch>0</horstretch>
+                  <verstretch>0</verstretch>
+                 </sizepolicy>
+                </property>
                 <property name="statusTip">
                  <string>Click to pause the session after the current trial</string>
                 </property>
@@ -814,7 +816,6 @@
   <tabstop>uiPushFlush</tabstop>
   <tabstop>uiPushReward</tabstop>
   <tabstop>uiPushStatusLED</tabstop>
-  <tabstop>uiCheckAppend</tabstop>
   <tabstop>uiPushStart</tabstop>
   <tabstop>uiPushPause</tabstop>
  </tabstops>

iblrig/gui/valve.py

@@ -49,7 +49,7 @@ def update(self):
         if len(self.values.open_times_ms) < 2:
             self._curve.setData(x=[], y=[])
         else:
-            time_range = list(np.linspace(self.values.open_times_ms[0], self.values.open_times_ms[-1], 100))
+            time_range = list(np.linspace(0, self.values.open_times_ms[-1], 100))
             self._curve.setData(x=time_range, y=self.values.ms2ul(time_range))
 
     def clear(self):

iblrig/gui/wizard.py

@@ -286,6 +286,8 @@ class RigWizard(QtWidgets.QMainWindow, Ui_wizard):
     session_info: dict = {}
     task_parameters: dict | None = None
     new_subject_details = QtCore.pyqtSignal()
+    append_session: bool = False
+    previous_subject: str | None = None
 
     def __init__(self, debug: bool = False, remote_devices: bool = False):
         super().__init__()
@@ -967,6 +969,21 @@ def start_stop(self):
                 self.uiPushStart.setIcon(self.style().standardIcon(QStyle.SP_MediaStop))
                 self._enable_ui_elements()
 
+                # Manage appended session
+                self.append_session = False
+                if self.previous_subject == self.model.subject and not self.model.hardware_settings.MAIN_SYNC:
+                    self.append_session = (
+                        QtWidgets.QMessageBox.question(
+                            self,
+                            'Appended Session',
+                            'Would you like to append to the previous session?',
+                            QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
+                            QtWidgets.QMessageBox.No,
+                        )
+                        == QtWidgets.QMessageBox.Yes
+                    )
+
+                # Manage subject weight
                 dlg = QtWidgets.QInputDialog()
                 weight, ok = dlg.getDouble(
                     self,
@@ -986,7 +1003,7 @@ def start_stop(self):
                 self.controller2model()
 
                 logging.disable(logging.INFO)
-                task = EmptySession(subject=self.model.subject, append=self.uiCheckAppend.isChecked(), interactive=False)
+                task = EmptySession(subject=self.model.subject, append=self.append_session, interactive=False)
                 logging.disable(logging.NOTSET)
                 self.model.session_folder = task.paths['SESSION_FOLDER']
                 if self.model.session_folder.joinpath('.stop').exists():
@@ -1027,7 +1044,7 @@ def start_stop(self):
                 cmd.extend(['--weight', f'{weight}'])
                 cmd.extend(['--log-level', 'DEBUG' if self.debug else 'INFO'])
                 cmd.append('--wizard')
-                if self.uiCheckAppend.isChecked():
+                if self.append_session:
                     cmd.append('--append')
                 if self.running_task_process is None:
                     self.tabLog.clear()
@@ -1109,7 +1126,7 @@ def _on_task_finished(self, exit_code, exit_status):
             if (
                 (ntrials := session_data['NTRIALS']) < 42
                 and not any([x in self.model.task_name for x in ('spontaneous', 'passive')])
-                and not self.uiCheckAppend.isChecked()
+                and not self.append_session
             ):
                 answer = QtWidgets.QMessageBox.question(
                     self,
@@ -1120,7 +1137,9 @@ def _on_task_finished(self, exit_code, exit_status):
                 )
                 if answer == QtWidgets.QMessageBox.Yes:
                     shutil.rmtree(self.model.session_folder)
+                    self.previous_subject = None
                     return
+            self.previous_subject = self.model.subject
 
             # manage poop count
             dlg = QtWidgets.QInputDialog()
@@ -1181,7 +1200,6 @@ def _enable_ui_elements(self):
         self.uiPushFlush.setEnabled(not is_running)
         self.uiPushReward.setEnabled(not is_running)
         self.uiPushStatusLED.setEnabled(not is_running)
-        self.uiCheckAppend.setEnabled(not is_running)
         self.uiGroupParameters.setEnabled(not is_running)
         self.uiGroupTaskParameters.setEnabled(not is_running)
         self.uiGroupTools.setEnabled(not is_running)

iblrig/test/tasks/test_passive_choice_world.py

@@ -1,4 +1,4 @@
-import numpy as np
+import pandas as pd
 
 import ibllib.pipes.dynamic_pipeline as dyn
 from ibllib.pipes.behavior_tasks import PassiveTaskNidq
@@ -8,18 +8,54 @@
 
 class TestInstantiatePassiveChoiceWorld(BaseTestCases.CommonTestInstantiateTask):
     def setUp(self) -> None:
-        session_id = 7
+        self.session_id = 7
         self.get_task_kwargs()
 
         # NB: Passive choice world not supported with Bpod as main sync
         assert self.task_kwargs['hardware_settings']['MAIN_SYNC']
         with self.assertLogs('iblrig.task', 40):
-            self.task = PassiveChoiceWorldSession(**self.task_kwargs, session_template_id=session_id)
+            self.task = PassiveChoiceWorldSession(**self.task_kwargs, session_template_id=self.session_id)
         self.task_kwargs['hardware_settings']['MAIN_SYNC'] = False
         with self.assertNoLogs('iblrig.task', 40):
-            self.task = PassiveChoiceWorldSession(**self.task_kwargs, session_template_id=session_id)
+            self.task = PassiveChoiceWorldSession(**self.task_kwargs, session_template_id=self.session_id)
         self.task.mock()
-        assert np.unique(self.task.trials_table['session_id']) == [session_id]
+
+    def test_fixtures(self) -> None:
+        # assert that fixture are loaded correctly
+        trials_table = self.task.trials_table
+        assert trials_table.session_id.unique() == [self.session_id]
+        pqt_file = self.task.get_task_directory().joinpath('passiveChoiceWorld_trials_fixtures.pqt')
+        fixtures = pd.read_parquet(pqt_file)
+        assert fixtures.session_id.unique().tolist() == list(range(12))
+        assert fixtures[fixtures.session_id == self.session_id].stim_type.equals(trials_table.stim_type)
+
+        # loop through fixtures
+        for session_id in fixtures.session_id.unique():
+            f = fixtures[fixtures.session_id == session_id]
+
+            # The task stimuli replays consist of 300 stimulus presentations ordered randomly.
+            assert len(f) == 300
+            assert f.stim_type.iloc[:10].nunique() > 1
+            assert set(f.stim_type.unique()) == {'G', 'N', 'T', 'V'}
+
+            # 180 gabor patches with 300 ms duration
+            # - 20 gabor patches with 0% contrast
+            # - 20 gabor patches at 35 deg left side with 6.25%, 12.5%, 25%, 100% contrast (80 total)
+            # - 20 gabor patches at 35 deg right side with 6.25%, 12.5%, 25%, 100% contrast (80 total)
+            # 40 openings of the water valve
+            # 40 go cues sounds
+            # 40 noise bursts sounds
+            assert len(f[f.stim_type == 'G']) == 180
+            assert sum(f[f.stim_type == 'G'].contrast == 0.0) == 20
+            positions = f[f.stim_type == 'G'].position.unique()
+            assert set(positions) == {-35.0, 35.0}
+            for position in positions:
+                counts = f[(f.stim_type == 'G') & (f.position == position) & (f.contrast != 0.0)].contrast.value_counts()
+                assert set(counts.keys()) == {0.0625, 0.125, 0.25, 1.0}
+                assert all([v == 20 for v in counts.values])
+            assert len(f[f.stim_type == 'V']) == 40
+            assert len(f[f.stim_type == 'T']) == 40
+            assert len(f[f.stim_type == 'N']) == 40
 
     def test_pipeline(self) -> None:
         """Test passive pipeline creation.