merged devel-1.0 changes and reorganization into ticket69 branch. Mer…

…ge conflicts defaulted to devel-1.0 blocks

.gitignore

@@ -21,7 +21,7 @@ data/a_eventMD-tempMSA.sqlite
 data/eventMD-20150723-204414.csv
 data/eventMD-PEG28-cusumLevelAnalysis.csv
 data/eventMD-PEG28-stepResponseAnalysis.csv
-data/eventProcessing.log
+data/eventProcessing*.log
 dist/
 gists/
 icons/.DS_Store
@@ -30,7 +30,7 @@ mosaic/.DS_Store
 mosaic/utest/.DS_Store
 mosaic/utest/gentests/
 mosaic/utest/testdata/.DS_Store
-mosaic/utest/testdata/eventProcessing.log
+mosaic/utest/testdata/eventProcessing*.log
 mosaic_devel_image.sh
 mosaic_nist.egg-info/
 mosaicgui/csvexportview/.DS_Store
@@ -52,3 +52,9 @@ data/eventMD-P28-bin.sqlite
 commit-hash
 mosaic/utest/testdata/*.sqlite
 data/eventMD-PEG28-ADEPT2State.sqlite
+doctrees
+latex
+_nistpages/
+data/SingleChan-*.*sv
+data/SingleChan-*.bin
+pyinstaller/icon.*

.scripts/pyinstaller-sh

@@ -6,7 +6,7 @@ plat=`uname`
 
 v1=$(<mosaic/_version.py)
 ver=`echo $v1 | cut -d= -f2 | cut -d\" -f2`
-build=`cat commit-hash`
+build=`git rev-parse --short HEAD`
 read -d '' info1 <<-"EOF"
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">

.travis.yml

@@ -15,7 +15,7 @@ addons:
     - libblas-dev
     - liblapack-dev
     - gfortran
-    - libzmq3-dev
+    #- libzmq3-dev
     - libqt4-dev
     - qt4-dev-tools 
     - python-qt4-dev 
@@ -25,7 +25,7 @@ before_install:
   - pip install -U pip
 install:
   - travis_wait travis_retry pip install -r requirements.txt --upgrade
-  - pip install pyzmq --upgrade
+  #- pip install pyzmq --upgrade
   - ln -s /usr/lib/python2.7/dist-packages/PyQt4/ $VIRTUAL_ENV/lib/python2.7/site-packages/
   - ln -s /usr/lib/python2.7/dist-packages/sip.so $VIRTUAL_ENV/lib/python2.7/site-packages/
 script: 

README.rst

@@ -45,22 +45,26 @@ Change Log
 
 **v1.3**
 
-- Added a brand new MOSAIC examples repository that contains plotting scripts and Jupyter notebooks (https://github.com/usnistgov/mosaic-examples)
+- Added a new MOSAIC examples repository that contains plotting scripts and Jupyter notebooks (https://github.com/usnistgov/mosaic-examples)
 
-- Added CUSUM+ algorithm (see pull requests #34, #43, #45, and #46)
+- Added the CUSUM+ algorithm (see pull requests #34, #43, #45, and #46)
+- Added the ability to read arbitrary binary files.
 - Renamed StepResponseAnalysis to ADEPT 2-state.
 - Renamed multiStateAnalysis to ADEPT.
 - CUSUM+ is now used to generate initial guesses for ADEPT
 - Streamlined unit test framework. Added new tests for CUSUM+.
 - Updated MOSAIC dependencies to include newer package versions and pandas. Run 'python setup.py mosaic_deps' to update.
 - Added a new metadata column (mdStateResTime) that saves the residence time of each state to the database. This affects ADEPT and CUSUM+.
+- Processing time per event is now saved to the database.
+- Added support for warning status codes that begin with 'w'. These codes preserve metadata in the SQLite output and are used to flag events that need closer inspection.
 - Removed mosaicgui from PyPi. 'pip install mosaic-nist' only installs command line modules. 
-- Top level ConvertToCSV supports arbitrary file extensions.
+- Top level ConvertTrajIO supports binary and ASCII files.
 - Added Travis continous integration support.
 - [GUI] Added a table that displays fit parameters in the event viewer window.
 - [GUI] Added more descriptive fit errors.
 - [GUI] Added CUSUM+ support to MOSAIC GUI.
 - [GUI] Export SQLITE database to CSV (#56).
+- [GUI] Support for warning codes in fit event viewer.
 - [GUI] Performance optimizations.
 - [GUI] Fit window in MOSAIC GUI displays idealized pulses overlays.
 - [GUI] Added additional analysis statistics.
@@ -69,7 +73,6 @@ Change Log
 - [Addons] Mathematica queries are *optionally* handled through an external Python script to improve performance.
 - [Addons] Added an option to limit PlotEvents in Mathematica addon to N events.
 
-- Fixes issues #36, #37, #38, #39, #47, #48, #54, #56, #58, #62, #65 and #66.
 - Known Issues: See #8, #10, #40, #63 and #69.
 
 **v1.2**

addons/mathematica/pyquery.py

@@ -8,11 +8,12 @@
 	:ChangeLog:
 	.. line-block::
 		3/23/15		AB	Initial version
+		7/5/16 		AB 	Updated code to use new module layout
 """
 
 import argparse
 import os
-import mosaic.sqlite3MDIO as sqlite3MDIO
+import mosaic.mdio.sqlite3MDIO as sqlite3MDIO
 import sqlite3
 
 def parseCLArgs():

bin/analysis.py

@@ -6,40 +6,40 @@
 import resource
 import csv
 
-import mosaic.SingleChannelAnalysis
+import mosaic.apps.SingleChannelAnalysis
 
-import mosaic.eventSegment as es
+import mosaic.partition.eventSegment as es
 
-import mosaic.singleStepEvent as sse
-import mosaic.adept2State as adept2State
-import mosaic.adept as adept 
-import mosaic.cusumPlus as cusumPlus
+import mosaic.process.singleStepEvent as sse
+import mosaic.process.adept2State as adept2State
+import mosaic.process.adept as adept 
+import mosaic.process.cusumPlus as cusumPlus
 
-from mosaic.qdfTrajIO import *
-from mosaic.abfTrajIO import *
-from mosaic.tsvTrajIO import *
-from mosaic.binTrajIO import *
+from mosaic.trajio.qdfTrajIO import *
+from mosaic.trajio.abfTrajIO import *
+from mosaic.trajio.tsvTrajIO import *
+from mosaic.trajio.binTrajIO import *
 
-from mosaic.besselLowpassFilter import *
-from mosaic.waveletDenoiseFilter import *
+from mosaic.filters.besselLowpassFilter import *
+from mosaic.filters.waveletDenoiseFilter import *
 
-mosaic.SingleChannelAnalysis.SingleChannelAnalysis(
+mosaic.apps.SingleChannelAnalysis.SingleChannelAnalysis(
 			'data/',
 			qdfTrajIO, 
 			None,
 			es.eventSegment,
 			adept.adept
 		).Run()
 
-mosaic.SingleChannelAnalysis.SingleChannelAnalysis(
+mosaic.apps.SingleChannelAnalysis.SingleChannelAnalysis(
 			'data/',
 			qdfTrajIO, 
 			None,
 			es.eventSegment,
 			adept2State.adept2State
 		).Run()
 
-mosaic.SingleChannelAnalysis.SingleChannelAnalysis(
+mosaic.apps.SingleChannelAnalysis.SingleChannelAnalysis(
 			'data/',
 			qdfTrajIO, 
 			None,

data/.settings

@@ -9,7 +9,7 @@
     "adept" : {
        "FitTol"	: "1.e-7",
        "FitIters" : "50000",
-       "InitThreshold" : "2.5",
+       "StepSize" : "2.5",
        "MinStateLength" : "10",
        "MaxEventLength" : "10000",
        "UnlinkRCConst" 	: "1"

mosaic/__init__.py

@@ -1,3 +1,79 @@
 from ._version import __version__
 from ._version import __build__
-from ._global import *
+from ._global import *
+
+import mosaic.utilities
+
+import mosaic.commonExceptions
+import mosaic.errors
+# import mosaic.zmqIO
+import mosaic.settings
+
+import mosaic.trajio.metaTrajIO 
+import mosaic.partition.metaEventPartition
+import mosaic.process.metaEventProcessor
+import mosaic.filters.metaIOFilter
+import mosaic.mdio.metaMDIO
+
+import mosaic.trajio.qdfTrajIO
+import mosaic.trajio.binTrajIO 
+import mosaic.trajio.abfTrajIO 
+import mosaic.trajio.tsvTrajIO 
+
+import mosaic.filters.besselLowpassFilter
+import mosaic.filters.convolutionFilter
+import mosaic.filters.waveletDenoiseFilter
+
+import mosaic.partition.eventSegment
+
+import mosaic.process.adept
+import mosaic.process.adept2State
+import mosaic.process.cusumPlus
+import mosaic.process.singleStepEvent
+
+import mosaic.mdio.sqlite3MDIO
+
+import mosaic.apps.ConvertTrajIO
+import mosaic.apps.SingleChannelAnalysis
+
+from mosaic.utilities.mosaicLogging import mosaicExceptionHandler
+
+import sys
+
+sys.excepthook=mosaicExceptionHandler
+
+__all__=[]
+
+__all__.extend(mosaic.utilities.__all__)
+
+__all__.extend(mosaic.commonExceptions.__all__)
+__all__.extend(mosaic.errors.__all__)
+# __all__.extend(mosaic.zmqIO.__all__)
+__all__.extend(mosaic.settings.__all__)
+
+__all__.extend(mosaic.trajio.metaTrajIO.__all__)
+__all__.extend(mosaic.partition.metaEventPartition.__all__)
+__all__.extend(mosaic.process.metaEventProcessor.__all__)
+__all__.extend(mosaic.filters.metaIOFilter.__all__)
+__all__.extend(mosaic.mdio.metaMDIO.__all__)
+
+__all__.extend(mosaic.trajio.qdfTrajIO.__all__)
+__all__.extend(mosaic.trajio.binTrajIO.__all__)
+__all__.extend(mosaic.trajio.abfTrajIO.__all__)
+__all__.extend(mosaic.trajio.tsvTrajIO.__all__)
+
+__all__.extend(mosaic.filters.besselLowpassFilter.__all__)
+__all__.extend(mosaic.filters.convolutionFilter.__all__)
+__all__.extend(mosaic.filters.waveletDenoiseFilter.__all__)
+
+__all__.extend(mosaic.partition.eventSegment.__all__)
+
+__all__.extend(mosaic.process.adept.__all__)
+__all__.extend(mosaic.process.adept2State.__all__)
+__all__.extend(mosaic.process.cusumPlus.__all__)
+__all__.extend(mosaic.process.singleStepEvent.__all__)
+
+__all__.extend(mosaic.mdio.sqlite3MDIO.__all__)
+
+__all__.extend(mosaic.apps.ConvertTrajIO.__all__)
+__all__.extend(mosaic.apps.SingleChannelAnalysis.__all__)

mosaic/_global.py

@@ -1,3 +1,8 @@
 # Control global settings
-DeveloperMode=False
-Logging=False
+DeveloperMode=False		# Turn on developer options.
+
+CodeProfiling='summary'	# Either 'summary' to print a summary at the end of a run, 
+						# 'none' for not timing, or 
+						# 'all' to print timing of every function call profiled.
+LogProperties=False		# Log all class properties defined with mosaic_property.
+LogSizeBytes=int(2<<20) # 2 MB

mosaic/_version.py

@@ -1,7 +1,7 @@
 import subprocess
 from mosaic.utilities.resource_path import resource_path
 
-__version__="1.3b2"
+__version__="1.3b3"
 
 try:
 	__build__=subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], stderr=subprocess.STDOUT).strip()

mosaic/ConvertToCSV.py → mosaic/apps/ConvertTrajIO.py

@@ -5,21 +5,24 @@
  	:Author: 	Arvind Balijepalli <[email protected]>
 	:License:	See LICENSE.TXT	
 """
-__docformat__ = 'restructuredtext'
-
 import itertools
 import string
 
-from mosaic.metaTrajIO import EmptyDataPipeError
+from mosaic.trajio.metaTrajIO import EmptyDataPipeError
+from mosaic.utilities.resource_path import format_path
 import numpy
+import pandas as pd
+
+__all__ = ["ConvertTrajIO"]
 
-class ConvertToCSV(object):
+class ConvertTrajIO(object):
 	"""
-		Convert data read from a sub-class of metaTrajIO to a comma separated text file
+		Convert data from a sub-class of metaTrajIO to either a delimited text file or binary file format.
 
 		:Parameters:
 			- `trajDataObj` : a trajIO data object
 			- `outdir` : the output directory. Default is *None*, which causes the output to be saved in the same directory as the input data.
+			- `extension` : 'csv' for comma separated values (default), 'tsv' for tab separated values, or 'bin' for 64-bit double precision binary. 
 	"""
 	def __init__(self, trajDataObj, outdir=None, extension="csv"):
 		self.trajDataObj=trajDataObj
@@ -35,6 +38,12 @@ def __init__(self, trajDataObj, outdir=None, extension="csv"):
 
 		self.filePrefix=None
 		self._creategenerator()
+
+		self._outputFormat={
+			"csv" : ('to_csv', {'sep': ',', 'header' : False}),
+			"tsv" :	('to_csv', {'sep': '\t', 'header' : False})
+		}
+
 
 	def Convert(self, blockSize):
 		"""
@@ -45,7 +54,15 @@ def Convert(self, blockSize):
 		"""
 		try:
 			while(True):
-				(self.trajDataObj.popdata(blockSize)).tofile(self._filename(), sep=',')
+
+
+				if self.extension=="bin":
+					numpy.array(self.trajDataObj.popdata(blockSize), dtype=numpy.float64).tofile(self._filename(), sep="")
+				else:
+					dat=pd.DataFrame(self.trajDataObj.popdata(blockSize))
+					f=self._outputFormat[self.extension]
+
+					getattr(dat, f[0])(self._filename(), index=False, **f[1])
 		except EmptyDataPipeError:
 			pass
 
@@ -73,6 +90,20 @@ def _filename(self):
 		"""
 		self._creategenerator()
 
-		return self.outDir+'/'+next(self.fileGenerator)+'.'+self.extension
+		return format_path(self.outDir+'/'+next(self.fileGenerator)+'.'+self.extension)
+
+
+if __name__ == '__main__':
+	import mosaic.trajio.qdfTrajIO as qdfTrajIO
+
+	for ext in ['bin','csv','tsv']:
+		q=qdfTrajIO.qdfTrajIO(dirname='data', filter='*qdf', Rfb=9.1e9, Cfb=1.07e-12)
+		ConvertTrajIO(
+			q,
+			outdir='data',
+			extension=ext
+		).Convert(100000)
 
-
+	print numpy.fromfile('data/SingleChan-0001_1.bin')[:5]
+	print numpy.hstack(numpy.fromfile('data/SingleChan-0001_1.csv', sep='\n'))[:5]
+	print numpy.hstack(numpy.fromfile('data/SingleChan-0001_1.tsv', sep='\t'))[:5]