- Added framework for filtering data

- Implemented a Bessel low pass filter

abfTrajIO.py

@@ -31,10 +31,10 @@ def __init__(self, **kwargs):
 		# base class processing first
 		super(abfTrajIO, self).__init__(**kwargs)
 
-	def appenddata(self, fname):
+	def readdata(self, fname):
 		"""
 			Read one or more files and append their data to the data pipeline.
-			Set a class attribute FsHz with the sampling frequency in Hz.
+			Set a class attribute Fs with the sampling frequency in Hz.
 
 			Args:
 				fname  list of data files to read
@@ -43,6 +43,8 @@ def appenddata(self, fname):
 			Errors:
 				SamplingRateChangedError if the sampling rate for any data file differs from previous
 		"""
+		tempdata=np.array([])
+
 		# Read a single file or a list of files.		
 		for f in fname:
 			[freq, hdr, dat] = abf.abfload_gp(f)
@@ -52,15 +54,17 @@ def appenddata(self, fname):
 
 			# set the sampling frequency in Hz. The times are in ms.
 			# If the Fs attribute doesn't exist set it
-			if not hasattr(self, 'FsHz'):	
-				self.FsHz=freq
+			if not hasattr(self, 'Fs'):	
+				self.Fs=freq
 			# else check if it s the same as before
 			else:
-				if self.FsHz!=freq:
+				if self.Fs!=freq:
 					raise metaTrajIO.SamplingRateChangedError("The sampling rate in the data file '{0}' has changed.".format(f))
 
 			# Add new data to the existing array
-			self.currDataPipe=np.hstack((self.currDataPipe, dat))
+			tempdata=np.hstack((tempdata, dat))
+
+		return tempdata
 
 	def formatsettings(self):
 		"""

besselLowpassFilter.py

@@ -0,0 +1,73 @@
+"""
+	Implementation of an 'N' order Bessel filter
+
+	Author: Arvind Balijepalli
+	Created: 7/1/2013
+
+	ChangeLog:
+		7/1/13		AB	Initial version
+"""
+import numpy as np 
+import scipy.signal as sig
+
+import metaIOFilter
+
+class besselLowpassFilter(metaIOFilter.metaIOFilter):
+	"""
+	"""
+	def __init__(self, **kwargs):
+		"""
+			Keyword Args:
+			In addition to metaIOFilter.__init__ args,
+				filterOrder		the filter order
+				filterCutoff	filter cutoff frequency in Hz
+		"""
+		# base class processing last
+		super(besselLowpassFilter, self).__init__(**kwargs)
+
+		try:
+			self.filterOrder=float(kwargs['filterOrder'])
+			self.filterCutoff=float(kwargs['filterCutoff'])
+		except KeyError:
+			print "Missing mandatory arguments 'filterOrder' or 'filterCutoff'"
+
+
+	def filterData(self, icurr, Fs):
+		"""
+			Filter self.eventData with the filter model setup in __init__
+		"""
+		self.eventData=icurr
+		self.Fs=Fs
+
+		self.filterModel=sig.filter_design.bessel(
+							N=self.filterOrder, 
+							Wn=(self.filterCutoff/(self.Fs/2)), 
+							btype='lowpass', 
+							analog=False, 
+							output='ba'
+						)
+
+		# calculate the initial state of the filter and scale it with the first data point
+		# so there is no sharp transient at the start of the data
+		zi=sig.lfilter_zi(b=self.filterModel[0], a=self.filterModel[1])*self.eventData[0]
+
+		[self.eventData, zf]=sig.lfilter(
+								b=self.filterModel[0],
+								a=self.filterModel[1],
+								x=self.eventData,
+								zi=zi
+							)
+
+	def formatsettings(self):
+		"""
+			Return a formatted string of filter settings
+		"""
+		fmtstr=""
+
+		fmtstr+='\tFilter settings: \n'
+		fmtstr+='\t\tFilter type = {0}\n'.format(self.__class__.__name__)
+		fmtstr+='\t\tFilter order = {0}\n'.format(self.filterOrder)
+		fmtstr+='\t\tFilter cutoff = {0} kHz\n'.format(self.filterCutoff*1e-3)
+		fmtstr+='\t\tDecimation = {0}\n'.format(self.decimate)
+
+		return fmtstr

binTrajIO.py

@@ -51,13 +51,13 @@ def __init__(self, **kwargs):
 		self.fileFormat='bin'
 
 		# set the sampling frequency in Hz.
-		if not hasattr(self, 'FsHz'):	
-			self.FsHz=self.SamplingFrequency
+		if not hasattr(self, 'Fs'):	
+			self.Fs=self.SamplingFrequency
 
-	def appenddata(self, fname):
+	def readdata(self, fname):
 		"""
 			Read one or more files and append their data to the data pipeline.
-			Set a class attribute FsHz with the sampling frequency in Hz.
+			Set a class attribute Fs with the sampling frequency in Hz.
 
 			Args:
 				fname  list of data files to read
@@ -66,10 +66,13 @@ def appenddata(self, fname):
 			Errors:
 				None
 		"""
+		tempdata=np.array([])
 		# Read binary data and add it to the data pipe
 		for f in fname:
-			self.currDataPipe=np.hstack(( self.currDataPipe, self.readBinaryFile(f) ))
+			tempdata=np.hstack(( tempdata, self.readBinaryFile(f) ))
 
+		return tempdata
+
 	def formatsettings(self):
 		"""
 			Return a formatted string of settings for display

metaIOFilter.py

@@ -0,0 +1,69 @@
+"""
+	A meta class that defines the interface for filtering data that is read in by any 
+	implementation of metaTrajIO
+
+	Author: 	Arvind Balijepalli
+	Created:	7/1/2013
+
+	ChangeLog:
+		7/1/13		AB	Initial version
+"""
+from abc import ABCMeta, abstractmethod
+import util
+
+class metaIOFilter(object):
+	"""
+		Defines the interface for specific filter implementations. Each filtering
+		algorithm must sub-class metaIOFilter and implement the following abstract
+		function:
+			filterData: 	apply a filter to self.eventData
+	"""
+	__metaclass__=ABCMeta
+
+	def __init__(self, **kwargs):
+		"""
+			Keyword Args:
+				decimate			sets the downsampling ratio of the filtered data (default:1, no decimation). 
+
+			Properties:
+				filteredData		list of filtered and decimated data
+				filterFs			sampling frequency after filtering and decimation
+		"""
+		self.decimate=int(kwargs.pop('decimate', 1))
+
+	@abstractmethod
+	def filterData(self, icurr, Fs):
+		"""
+			This is the equivalent of a pure virtual function in C++. Specific filtering
+			algorithms must implement this method and then call this base function using super for
+			additional processing.
+
+			Implementations of this method MUST store (1) a ref to the raw event data in self.eventData AND 
+			(2) the sampling frequence in self.Fs.
+
+			Args:
+				icurr				ionic current in pA
+				Fs 					original sampling frequency in Hz
+		"""
+		pass
+
+	@abstractmethod
+	def formatsettings(self):
+		"""
+			Return a formatted string of filter settings
+		"""
+		pass
+
+	@property
+	def filteredData(self):
+		"""
+			Decimate the data when it is accessed
+		"""
+		# return util.decimate(self.eventData, self.decimate)
+		return self.eventData[::self.decimate]
+
+	@property
+	def filterFs(self):
+		return self.Fs/self.decimate
+
+

metaTrajIO.py

@@ -11,6 +11,8 @@
 import glob
 import numpy as np
 
+import settings
+
 # define custom exceptions
 class IncompatibleArgumentsError(Exception):
 	pass
@@ -45,11 +47,16 @@ def __init__(self, **kwargs):
 							cannot be used in conjuction with dirname/nfiles. The filter 
 							argument is ignored when used in combination with fnames. 
 
-				filter='<wildcard filter>' (optional, filter is '*'' if not specified)
+				filter		'<wildcard filter>' (optional, filter is '*'' if not specified)
 				start 		Data start point. This allows the first 'n' specified to be skipped
 							and excluded from any data analysis
+				datafilter	Handle to the algorithm to use to filter the data. If no algorithm is specified, datafilter
+							is None and no filtering is performed.
 			Returns:
 				None
+			Properties:
+				FsHz		sampling frequency in Hz. If the data was decimated, this property will hold the
+							sampling frequency after decimation.
 			Errors:
 				IncompatibleArgumentsError when arguments defined above are not used properly
 		"""
@@ -90,6 +97,15 @@ def __init__(self, **kwargs):
 		self.fileFormat='N/A'
 		self.datPath="/".join((self.dataFiles[0].split('/'))[:-1])
 
+
+		# setup data filtering
+		if hasattr(self, 'datafilter'):
+			self.dataFilter=True
+			self.dataFilterObj=self.__setupDataFilter()
+		else:
+			self.dataFilter=False
+
+
 		# initialize an empty data pipeline
 		self.currDataPipe=np.array([])
 		# Track the start point of the queue. This var is used to manage
@@ -111,6 +127,13 @@ def __init__(self, **kwargs):
 	#################################################################
 	# Public API: functions
 	#################################################################
+	@property
+	def FsHz(self):
+		if not self.dataFilter:
+			return self.Fs
+		else:
+			return self.dataFilterObj.filterFs
+
 	def popdata(self, n):
 		"""
 			Pop data points from self.currDataPipe. This function uses recursion 
@@ -197,7 +220,11 @@ def formatsettings(self):
 		"""
 		fmtstr=""
 
-		fmtstr+='\tTrajectory I/O settings: \n'
+		# add the filter settings
+		if self.dataFilter:
+			fmtstr+=self.dataFilterObj.formatsettings()
+
+		fmtstr+='\n\tTrajectory I/O settings: \n'
 		fmtstr+='\t\tFiles processed = {0}\n'.format(self.nFiles-len(self.dataFiles))
 		fmtstr+='\t\tData path = {0}\n'.format(self.datPath)
 		fmtstr+='\t\tFile format = {0}\n'.format(self.fileFormat)
@@ -209,18 +236,33 @@ def formatsettings(self):
 	# Private API: Interface functions, implemented by sub-classes.
 	# Should not be called from external classes
 	#################################################################
-	@abstractmethod
 	def appenddata(self, fname):
 		"""
 			Read the specified data file(s) and append its data to the data pipeline. Set 
-			a class attribute FsHz with the sampling frequency in Hz.
+			a class property FsHz with the sampling frequency in Hz.
 
 			Args:
 				fname	list of filenames
 
 			
 			See implementations of metaTrajIO for specfic documentation.
 		"""
+		data=self.readdata(fname)
+		if self.dataFilter:
+			self.dataFilterObj.filterData(data, self.Fs)
+			self.currDataPipe=np.hstack((self.currDataPipe, self.dataFilterObj.filteredData ))
+		else:
+			self.currDataPipe=np.hstack((self.currDataPipe, data ))
+
+	@abstractmethod
+	def readdata(self, fname):
+		"""
+			Read the specified data file(s) and  return the data as an array. Set 
+			a class property Fs with the sampling frequency in Hz.
+
+			Args:
+				fname	list of filenames
+		"""
 		pass
 
 	def popfnames(self, n):
@@ -241,4 +283,14 @@ def popfnames(self, n):
 			pass
 		return poplist
 
+	#################################################################
+	# Private Functions
+	#################################################################
+	def __setupDataFilter(self):
+		filtsettings=settings.settings( self.datPath ).getSettings(self.datafilter.__name__)
+		if filtsettings=={}:
+			print "No settings found for '{0}'. Data filtering is disabled".format(str(self.datafilter.__name__))
+			self.dataFilter=False
+			return
 
+		return self.datafilter(**filtsettings)

qdfTrajIO.py

@@ -42,10 +42,10 @@ def __init__(self, **kwargs):
 		# additional meta data
 		self.fileFormat='qdf'
 
-	def appenddata(self, fname):
+	def readdata(self, fname):
 		"""
 			Read one or more files and append their data to the data pipeline.
-			Set a class attribute FsHz with the sampling frequency in Hz.
+			Set a class attribute Fs with the sampling frequency in Hz.
 
 			Args:
 				fname  list of data files to read
@@ -60,18 +60,17 @@ def appenddata(self, fname):
 
 		# set the sampling frequency in Hz. The times are in ms.
 		# If the Fs attribute doesn't exist set it
-		if not hasattr(self, 'FsHz'):	
-			self.FsHz=1000./(q[1][0]-q[0][0])
+		if not hasattr(self, 'Fs'):	
+			self.Fs=1000./(q[1][0]-q[0][0])
 		# else check if it s the same as before
 		else:
-			if self.FsHz!=1000./(q[1][0]-q[0][0]):
+			if self.Fs!=1000./(q[1][0]-q[0][0]):
 				raise metaTrajIO.SamplingRateChangedError("The sampling rate in the data file '{0}' has changed.".format(fname))
 
 		# Slice the data to remove the time-stamps to conserve memory		
 		# and add new data to the existing array
 		#print "last raw current val in file ", fname, " = ", q[-1]
-
-		self.currDataPipe=np.hstack((self.currDataPipe, q[ : , 1]))
+		return q[ : , 1]
 
 	def formatsettings(self):
 		"""