ibldsp.utils.sync_timestamps performance no longer impacted by offset…

… (issue #53)

release_notes.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [1.6.3] - 2025-01-15
+
+### changed
+- ibldsp.utils.sync_timestamps performance no longer impacted by timestamp offset
+
+
 ## [1.6.2] - 2025-01-06
 
 ### added
@@ -12,7 +18,7 @@
 - waveforms extractor returns a list of files for registration
 
 ### changed
-- moved the repo contribution guide to automatic ruff formatting 
+- moved the repo contribution guide to automatic ruff formatting
 
 
 ## [1.6.0] - 2024-12-06
@@ -21,7 +27,7 @@
 - single derivative option for CSD for RIGOR metrics
 - radon forward and inverse transforms
 - support headerless binary files (open ephys)
-- 
+-
 ### fixed
 - cadzow iterations
 - numpy 2.0 support
@@ -62,7 +68,7 @@
 - Add features and tests for `extract_wfs_cbin`, including various preprocessing options.
 
 ## 1.1.2 2024-07-03
--  bugfix waveform extraction: reverting refactoring of the function to maintain compatibility with current ibllib 
+-  bugfix waveform extraction: reverting refactoring of the function to maintain compatibility with current ibllib
 
 ## 1.1.1 2024-06-07
 -  Add support for NP2.0 prototype probe with probetype 1030
@@ -79,7 +85,7 @@
 ## 0.10.3 2024-04-18
 -  Patch fixing memory leaks for `waveform_extraction` module.
 ## 0.10.2 2024-04-10
--  Add `waveform_extraction` module to `ibldsp`. This includes the `extract_wfs_array` and `extract_wfs_cbin` methods. 
+-  Add `waveform_extraction` module to `ibldsp`. This includes the `extract_wfs_array` and `extract_wfs_cbin` methods.
 -  Add code for performing subsample shifts of waveforms.
 ## 0.10.1 2024-03-19
 -  ensure compatibility with spikeglx 202309 metadata coordinates
@@ -106,7 +112,7 @@
 
 ## 0.6.1 2023-06-06
 - Fix bug in ADC cycles sampling for Neuropixel 1.0 probes
-- 
+-
 ## 0.6.0 2023-05-15
 - Add waveforms utilities for spike features computations
 
@@ -129,11 +135,11 @@
 
 ## 0.4.0 2022-10-28
 - current source density simple double diff with denoising from raw LFP
-- 
+-
 ## 0.3.2 2022-10-27
 - spikeglx geometry chops the ADC sample shift to the number of channels to accomodate legacy 3A probes with 276 channels
 - agc: gain refers to the inverse of applied gains for agc - done to handle dead channels in destriping
-- 
+-
 ## 0.3.1
 - neurodsp.utils.rises / falls: detects rising and falling edges of oversampled analog signals
 - neuropixel: add functions to reconstruct original files from split NP2.4 files
@@ -154,4 +160,4 @@
 ## minor changes
 -   deprecated SITES_COORDINATES that would default on Neuropixel 1.0 early generations (3A probes with 374 channels)
 -   spikeglx compress: pass dtype to mtscomp
-- 
+-

setup.py

@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name="ibl-neuropixel",
-    version="1.6.2",
+    version="1.6.3",
     author="The International Brain Laboratory",
     description="Collection of tools for Neuropixel 1.0 and 2.0 probes data",
     long_description=long_description,

src/ibldsp/utils.py

@@ -21,13 +21,17 @@ def sync_timestamps(tsa, tsb, tbin=0.1, return_indices=False, linear=False):
      numpy array: of indices ia
      numpy array: of indices ib
     """
+    # determine fixed offset between first timestamp of each series
+    # this reduces the size of array for correlation
+    offset = tsb[0] - tsa[0]
+    tsa += offset
 
     def _interp_fcn(tsa, tsb, ib, linear=linear):
         # now compute the bpod/fpga drift and precise time shift
         ab = np.polyfit(tsa[ib >= 0], tsb[ib[ib >= 0]] - tsa[ib >= 0], 1)
         drift_ppm = ab[0] * 1e6
         if linear:
-            fcn_a2b = lambda x: x * (1 + ab[0]) + ab[1]  # noqa
+            fcn_a2b = lambda x: (x + offset) * (1 + ab[0]) + ab[1]  # noqa
         else:
             fcn_a2b = scipy.interpolate.interp1d(
                 tsa[ib >= 0], tsb[ib[ib >= 0]], fill_value="extrapolate"

src/tests/unit/test_ibldsp.py

@@ -62,6 +62,11 @@ def test_timestamps(self):
         )
         assert np.all(np.isclose(_fcn(tsa[imiss[_ia]]), tsb[imiss2[_ib]]))
 
+        # test timestamps with huge offset (previously caused ArrayMemoryError)
+        tsb -= 1e15
+        _fcn, _drift = utils.sync_timestamps(tsa, tsb)
+        assert np.all(np.isclose(_fcn(tsa), tsb))
+
 
 class TestParabolicMax(unittest.TestCase):
     # expected values
@@ -650,3 +655,7 @@ def test_compute_features(self):
         self.assertEqual(multi_index, list(df.index))
         self.assertEqual(["snippet_id", "channel_id"], list(df.index.names))
         self.assertEqual(num_snippets * (self.nc - 1), len(df))
+
+
+if __name__ == "__main__":
+    unittest.main()