0.7.0 (#41)

Switch to a much simpler multiprocessing implementation, change keyboard backend, and allow tweaking of the remote process WRT garbage collection/priority.

 - Rather than shared arrays, we're sending a dictionary (with arbitrary data) via a `multiprocessing.Pipe`. Notes:
    - Pipe is unidirectional (only data from remote to local).
    - We don't lock access to the connections because a) Reduces performance, esp. >1kHz, and b) Not super necessary (no chance of data corruption, read function never gets a chance to finish at high frequencies, i.e. data is always available).
    - This montage seems to work fine at 2kHz, but above gets a little shaky. Haven't tried a fast "real" device yet.
    - Dicts get us a few things:
        a. Completely arbitrary data shapes, with little effort (no need to keep track of dimensions).
        b. Named data elements (e.g. `data['time']`, `data['rel_wheel']`)
 - Trying [pynput](https://github.com/moses-palmer/pynput) as the backend (key release for the previous version didn't seem quite right).
 - Allow running the remote process as high priority/low niceness (may need root on Unix platforms?), and optional disabling of garbage collection. Neither seems to make a *ton* of difference, but haven't tested beyond a few minutes.

.travis.yml

@@ -12,7 +12,7 @@ install:
   - pip install python-coveralls coverage
   - pip install .[full]
 script:
-  - nosetests -a travis=yes --with-coverage
+  - xvfb-run -s "-screen 0 1024x768x24 -ac +extension GLX +render -noreset" nosetests -a travis=yes --with-coverage
 after_success:
   - coveralls --config_file .coveragerc
 notifications:

requirements.txt

@@ -1,2 +1,3 @@
 future
 numpy
+psutil

setup.py

@@ -11,7 +11,7 @@
 
 setup(
     name='toon',
-    version='0.6.0',
+    version='0.7.0',
     description='Tools for neuroscience experiments',
     url='https://github.com/aforren1/toon',
     author='Alexander Forrence',
@@ -31,10 +31,10 @@
     ],
     install_requires=requirements,
     extras_require={
-        'full': ['hidapi', 'pyserial', 'keyboard', 'nidaqmx;platform_system=="Windows"'],
+        'full': ['hidapi', 'pyserial', 'pynput', 'nidaqmx;platform_system=="Windows"'],
         'hand': ['hidapi'],
         'birds': ['pyserial'],
-        'keyboard': ['keyboard'],
+        'keyboard': ['pynput'],
         'force': ['nidaqmx;platform_system=="Windows"']
     },
     keywords='psychophysics neuroscience input experiment',

toon/examples/try_inputs.py

@@ -7,9 +7,10 @@
 import numpy as np
 if system() is 'Windows':
     from toon.input import ForceTransducers
+import matplotlib.pyplot as plot
 
 # Call via
-# python -m toon.examples.try_inputs --dev keyboard --mp True
+# python -m toon.examples.try_inputs --dev keyboard --mp True --time 10
 import os
 not_travis = 'TRAVIS' not in os.environ
 if not_travis:
@@ -24,15 +25,25 @@
                         dest='dev')
     parser.add_argument('--mp',
                         dest='mp',
-                        type=lambda x: bool(util.strtobool(x)))
+                        type=lambda x: bool(util.strtobool(x)),
+                        default=False)
+    parser.add_argument('--time',
+                        dest='dur', default=5)
+    parser.add_argument('--print', dest='print', default=True)
+    parser.add_argument('--plot', dest='plot', default=False)
     results = parser.parse_args()
 
     mp = results.mp
     device = results.dev
+    duration = float(results.dur)
+    prnt = bool(results.print)
+    plt = bool(results.plot)
+
     if not_travis:
         time = core.monotonicClock.getTime
     else:
-        from time import time
+        from timeit import default_timer
+        time = default_timer
     if device == 'keyboard':
         dev = Keyboard(keys=['a', 's', 'd', 'f'], clock_source=time)
     elif device == 'hand':
@@ -53,15 +64,25 @@
         device = MultiprocessInput(dev)
     else:
         device = dev
-
+    lst = list()
     with device as d:
         t0 = time()
-        t1 = t0 + 10
+        t1 = t0 + duration
         while time() < t1:
-            timestamp, data = d.read()
+            t2 = time()
+            t3 = t2 + 0.016
+            data = d.read()
             if data is not None:
-                print(timestamp - t0)
-                print(data)
-            sleep(0.016)
+                if prnt:
+                    print([d['data'] for d in data])
+                lst.extend([d['time'] for d in data])
+            while time() < t3:
+                pass
+    if plt:
+        d = np.diff(lst)
+        plot.plot(d)
+        plot.show()
+        plot.hist(d)
+        plot.show()
 
     sys.exit()

toon/input/base_input.py

@@ -1,7 +1,5 @@
 import abc
-import numpy as np
-from time import time
-from toon.input.mp_input import check_and_fix_dims
+from timeit import default_timer
 
 class BaseInput(object):
     """
@@ -10,35 +8,11 @@ class BaseInput(object):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def __init__(self, clock_source=time, data_dims=None):
+    def __init__(self, clock_source=default_timer):
         """
         Args:
             clock_source: Clock or timer that returns the current (absolute or relative) time.
-            data_dims: Either a single integer, a list containing a single integer, or a list of
-                    lists, used to pre-allocate data outputted from the device.
-
-                    Examples (good)::
-                        3 # single vector of length 3
-                        [3] # single vector of length 3
-                        [[3], [2]] # two vectors, one of length 3 and the other of 2
-                        [[3, 2]] # one 3x2 matrix
-                        [[2,3], [5,4,3]] # one 2x3 matrix, one 5x4x3 array.
-
-                    Examples (bad)::
-                        [3,2] # ambiguous (two vectors or one matrix?)
-                        [3, [3,2]] # not necessarily bad, but not currently handled
-                        [[[3,2], 2], [5, 5]] # cannot handle deeper levels of nesting
-
         """
-
-        if data_dims is None:
-            raise ValueError('Must specify expected dimensions of data.')
-        data_dims = check_and_fix_dims(data_dims)
-        self.data_dims = data_dims
-
-        # allocate data buffers
-        self._data_buffers = [np.full(dd, np.nan) for dd in data_dims]
-        self._data_elements = len(data_dims)
         self.name = type(self).__name__
         self.time = clock_source
 
@@ -50,12 +24,8 @@ def __enter__(self):
     @abc.abstractmethod
     def read(self):
         """
-        Return the timestamp, and either a single piece of data or
-        multiple pieces of data (as a list).
-
-        Examples:
-            return timestamp, data
-            return timestamp, [data1, data2]
+        Return the data as a dictionary, e.g. {'timestamp': time, 'data': data}.
+        All shapes and sizes allowed.
         """
         pass
 

toon/input/birds.py

@@ -59,8 +59,8 @@ def __init__(self, ports=None, sample_ports=None,
         self._master_index = self.ports.index(self.master)
         self._sample_ports_indices = [self.ports.index(sp) for sp in self.sample_ports]
         self._ndata = 3 * len(self.sample_ports)  # 3 axes per bird of interest
-
-        BaseInput.__init__(self, data_dims=[[self._ndata]], **kwargs)
+        self._data_buffer = np.full(self._ndata, np.nan)
+        BaseInput.__init__(self, **kwargs)
 
         # handle the reordering of axes (bird is [y, z, x] relative to screen)
         # TODO: clean this up (far too complicated)
@@ -72,7 +72,7 @@ def __enter__(self):
                                      bytesize=serial.EIGHTBITS,
                                      xonxoff=0,
                                      rtscts=0,
-                                     timeout=0)
+                                     timeout=(1.0/self.sampling_frequency) * 2.0)
                        for port in self.ports]
 
         for bird in self._birds:
@@ -105,24 +105,24 @@ def __enter__(self):
         return self
 
     def read(self):
-        timestamp = self.time()
         _data_list = list()
         for bird in self._sample_ports_indices:
             _data_list.append(self._birds[bird].read(6))  # assumes position data
         # only convert data if it's there
+        timestamp = self.time()
         if not any(b'' == s for s in _data_list):
             _data_list = [self.decode(msg) for msg in _data_list]
-            self._data_buffers[0][:] = _data_list
-            self._data_buffers[0][:] = self._data_buffers[0][self._reindex[:self._ndata]]
-            temp_x = self._data_buffers[0][::3]
-            temp_y = self._data_buffers[0][1::3]
+            self._data_buffer[:] = np.array(_data_list).reshape(self._ndata)
+            self._data_buffer[:] = self._data_buffer[self._reindex[:self._ndata]]
+            temp_x = self._data_buffer[::3]
+            temp_y = self._data_buffer[1::3]
             # here be magic numbers (very Kinereach-specific)
-            self._data_buffers[0][::3] = temp_x * np.cos(-0.01938) - temp_y * np.sin(-0.01938)
-            self._data_buffers[0][1::3] = temp_y * np.sin(-0.01938) + temp_y * np.cos(-0.01938)
-            self._data_buffers[0][::3] += 61.35 - 60.5
-            self._data_buffers[0][1::3] += 17.69 - 34.0
-            return timestamp, self._data_buffers[0]
-        return None, None
+            self._data_buffer[::3] = temp_x * np.cos(-0.01938) - temp_y * np.sin(-0.01938)
+            self._data_buffer[1::3] = temp_y * np.sin(-0.01938) + temp_y * np.cos(-0.01938)
+            self._data_buffer[::3] += 61.35 - 60.5
+            self._data_buffer[1::3] += 17.69 - 34.0
+            return {'time': timestamp, 'data': np.copy(self._data_buffer)}
+        return None
 
 
     def __exit__(self, type, value, traceback):

toon/input/force_transducers.py

@@ -1,4 +1,5 @@
 import platform
+import numpy as np
 from toon.input.base_input import BaseInput
 
 if platform.system() is 'Windows':
@@ -18,15 +19,15 @@ class ForceTransducers(BaseInput):
 
     def __init__(self, **kwargs):
 
-        BaseInput.__init__(self, data_dims=10, **kwargs)
+        BaseInput.__init__(self, **kwargs)
 
         self._device_name = system.devices[0].name  # Assume the first NIDAQ-mx device is the one we want
         self._channels = [self._device_name + '/ai' + str(n) for n in
                           [2, 9, 1, 8, 0, 10, 3, 11, 4, 12]]
+        self._data_buffer = np.full(10, np.nan)
 
     def __enter__(self):
         self._device = nidaqmx.Task()
-        self._start_time = self.time.getTime()
 
         self._device.ai_channels.add_ai_voltage_chan(
             ','.join(self._channels),
@@ -39,12 +40,12 @@ def __enter__(self):
         return self
 
     def read(self):
-        timestamp = self.time()
         try:
-            self._reader.read_one_sample(self._data_buffers[0], timeout=0)
+            self._reader.read_one_sample(self._data_buffer, timeout=0)
+            timestamp = self.time()
         except DaqError:
-            return None, None
-        return timestamp, self._data_buffers[0]
+            return None
+        return {'time': timestamp, 'data': np.copy(self._data_buffer)}
 
     def __exit__(self, type, value, traceback):
         self._device.stop()

toon/input/hand.py

@@ -18,13 +18,13 @@ class Hand(BaseInput):
     Kata and the BLAM Lab.
 
     """
-    def __init__(self, nonblocking=True, **kwargs):
+    def __init__(self, nonblocking=False, **kwargs):
         """
         Args:
             nonblocking (bool): Whether the HID interface blocks for input.
         Notes:
-        `nonblocking` should typically remain `True`, as I doubt there's any performance
-        benefit and it leads to difficult debugging.
+            If testing the `Hand`, I would suggest setting `nonblocking` to True for
+            the sake of easy debugging.
 
         Data is formatted as [x, y, z] per finger (15 elements, 3 per finger).
 
@@ -34,13 +34,14 @@ def __init__(self, nonblocking=True, **kwargs):
             >>> device = Hand(nonblocking=True)
         """
 
-        super(Hand, self).__init__(data_dims=15, **kwargs)
+        super(Hand, self).__init__(**kwargs)
 
         self._rotval = np.pi / 4.0
         self._sinval = np.sin(self._rotval)
         self._cosval = np.cos(self._rotval)
         self.nonblocking = nonblocking
         self._device = None
+        self._data_buffer = np.full(15, np.nan)
 
     def __enter__(self):
         """HAND-specific initialization.
@@ -55,18 +56,19 @@ def __enter__(self):
 
     def read(self):
         """HAND-specific read function."""
-        timestamp = self.time()
         data = self._device.read(46)
+        timestamp = self.time()
         if data:
             data = struct.unpack('>LhHHHHHHHHHHHHHHHHHHHH', bytearray(data))
             data = np.array(data, dtype='d')
             data[0] /= 1000.0  # device timestamp (since power-up, in milliseconds)
-            data[1:] /= 65535.0
-            self._data_buffers[0][0::3] = data[2::4] * self._cosval - data[3::4] * self._sinval  # x
-            self._data_buffers[0][1::3] = data[2::4] * self._sinval + data[3::4] * self._cosval  # y
-            self._data_buffers[0][2::3] = data[4::4] + data[5::4]  # z
-            return timestamp, self._data_buffers[0]
-        return None, None
+            data[2:] /= 65535.0
+            self._data_buffer[0::3] = data[2::4] * self._cosval - data[3::4] * self._sinval  # x
+            self._data_buffer[1::3] = data[2::4] * self._sinval + data[3::4] * self._cosval  # y
+            self._data_buffer[2::3] = data[4::4] + data[5::4]  # z
+            return {'time': timestamp, 'data': np.copy(self._data_buffer),
+                    'device_time': data[0], 'us_deviation': data[1]}
+        return None
 
     def __exit__(self, type, value, traceback):
         """Close the HID interface."""