merge dev into release

merge dev into release

docs/conf.py

@@ -22,29 +22,13 @@
 
 
 # -- General configuration ---------------------------------------------------
-
-on_rtd = input('Generating a pdf? (y/[n])')
-if on_rtd == 'y':
-    on_rtd = False
-else:
-    on_rtd = True
-
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-if on_rtd:
-    extensions = [
-        'sphinx.ext.napoleon',
-        'sphinx.ext.autodoc',
-        'sphinx.ext.autosummary',
-        'sphinx_click.ext'
-    ]
-else:
-    extensions = [
+
+extensions = [
     'sphinx.ext.napoleon',
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',
-    'rst2pdf.pdfbuilder',
     'sphinx_click.ext'
 ]
 
@@ -72,11 +56,33 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Source: https://gist.github.com/alfredodeza/7fb5c667addb1c6963b9
-# index - master document
-# rst2pdf - Documentation
-# MoSeq2-PCA Documentation - title of the pdf
-# Datta Lab - author name in the pdf
-pdf_documents = [('index', u'Documentation', u'MoSeq2-PCA Python Documentation', u'Datta Lab'),]
+html_static_path = []
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # 'fontpkg': '\\usepackage{euler}'
+
+    # The font size ('10pt', '11pt' or '12pt').
+
+    # 'pointsize': '12pt',
+
+    # fontpkg': r'''
+    # \setmainfont{Arial Regular}
+    # \setsansfont{Arial Regular}
+    # \setmonofont{Menlo Regular}
+    # ''',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+latex_documents = [
+    ('index', u'moseq2-pca.tex', u'moseq2-pca Documentation', u'Datta Lab', 'manual'),
+]

docs/index.rst

@@ -7,7 +7,7 @@ Welcome to moseq2-pca's documentation!
 ======================================
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
    :caption: Contents:
 
    modules

moseq2_pca/cli.py

@@ -1,12 +1,6 @@
-'''
-
-CLI front-end operations. This module contains all the functionality and configurable parameters
-users can alter to most accurately process their data.
-
-Note: These functions simply read all the parameters into a dictionary,
- and then call the corresponding wrapper function with the given input parameters.
-
-'''
+"""
+CLI for PCA and model-free changepoint analysis.
+"""
 
 import os
 import click
@@ -33,20 +27,20 @@ def cli():
     pass
 
 def common_pca_options(function):
-    '''
-    This is a decorator function that is used to group common Click parameters/dependencies for PCA-related operations.
-    Parameters
-    ----------
-    function (Click command function): Decorated function to add enclosed parameter options to.
-    Returns
-    -------
-    function (Click command function): Decorated function now including 7 additional input parameters.
-    '''
+    """
+    Decorator function for common Click parameters/dependencies for PCA-related operations.
+    
+    Args:
+    function: Function to add enclosed parameters to as click options.
+
+    Returns:
+    function: Updated function including shared parameters.
+    """
 
     function = click.option('--cluster-type', type=click.Choice(['local', 'slurm', 'nodask']),
-                  default='local', help='Cluster type')(function)
-    function = click.option('--input-dir', '-i', type=click.Path(), default=os.getcwd(), help='Directory to find h5 files')(function)
-    function = click.option('--output-dir', '-o', default=join(os.getcwd(), '_pca'), type=click.Path(), help='Directory to store results')(function)
+                  default='local', help='Compute enviornment the command runs in')(function)
+    function = click.option('--input-dir', '-i', type=click.Path(), default=os.getcwd(), help='Directory to find extracted h5 files')(function)
+    function = click.option('--output-dir', '-o', default=join(os.getcwd(), '_pca'), type=click.Path(), help='Directory to store PCA results')(function)
     function = click.option('--config-file', type=click.Path(), help="Path to configuration file")(function)
 
     function = click.option('--h5-path', default='/frames', type=str, help='Path to data in h5 files')(function)
@@ -57,18 +51,18 @@ def common_pca_options(function):
 
 
 def common_dask_parameters(function):
-    '''
-    This is a decorator function that is used to group common Click parameters for Dask-related dependencies.
-    Parameters
-    ----------
-    function (Click command function): Decorated function to add enclosed parameter options to.
-    Returns
-    -------
-    function (Click command function): Decorated function now including 7 additional input parameters.
-    '''
+    """
+    Decorator function for common Click parameters for dask.
+
+    Args:
+    function: Function to add enclosed parameters to as click options.
+    
+    Returns:
+    function: Updated function including shared parameters.
+    """
 
     function = click.option('--dask-cache-path', '-d', default=os.path.join(os.getcwd(), '_pca'), type=click.Path(),
-                            help='Path to spill data to disk for dask local scheduler')(function)
+                            help='Path to spill data to disk for dask')(function)
     function = click.option('--dask-port', default='8787', type=str, help="Port to access dask dashboard")(function)
     function = click.option('-q', '--queue', type=str, default='debug',
                             help="Cluster queue/partition for submitting jobs")(function)
@@ -77,31 +71,32 @@ def common_dask_parameters(function):
     function = click.option('-p', '--processes', type=int, default=1, help="Number of processes to run on each worker")(
         function)
     function = click.option('-m', '--memory', type=str, default="15GB", help="Total RAM usage per worker")(function)
-    function = click.option('-w', '--wall-time', type=str, default="06:00:00", help="Wall time for workers")(function)
+    function = click.option('-w', '--wall-time', type=str, default="06:00:00", help="Wall time (compute time) for workers")(function)
     function = click.option('--timeout', type=float, default=5,
                             help="Time to wait for workers to initialize before proceeding (minutes)")(function)
 
     return function
 
 
-@cli.command(name='train-pca', cls=command_with_config('config_file'), help='Trains PCA on all extracted results (h5 files) in input directory')
+@cli.command(name='train-pca', cls=command_with_config('config_file'), help='Train PCA on all extracted results (h5 files) in input directory')
 @common_pca_options
 @common_dask_parameters
-@click.option('--gaussfilter-space', default=(1.5, 1), type=(float, float), help="Spatial filter for data (Gaussian)")
-@click.option('--gaussfilter-time', default=0, type=float, help="Temporal filter for data (Gaussian)")
-@click.option('--medfilter-space', default=[0], type=int, help="Median spatial filter", multiple=True)
-@click.option('--medfilter-time', default=[0], type=int, help="Median temporal filter", multiple=True)
+@click.option('--gaussfilter-space', default=(1.5, 1), type=(float, float), help="x, y sigma for kernel in Spatial filter for data (Gaussian)")
+@click.option('--gaussfilter-time', default=0, type=float, help="sigma for temporal filter for data (Gaussian)")
+@click.option('--medfilter-space', default=[0], type=int, help="kernel size for median spatial filter", multiple=True)
+@click.option('--medfilter-time', default=[0], type=int, help="kernel size for median temporal filter", multiple=True)
 @click.option('--missing-data', is_flag=True, type=bool, help="Use missing data PCA; will be automatically set to True if cable-filter-iters > 1 from the extract step.")
-@click.option('--missing-data-iters', default=10, type=int, help="Missing data PCA iterations")
-@click.option('--mask-threshold', default=-16, type=float, help="Threshold for mask (missing data only)")
+@click.option('--missing-data-iters', default=10, type=int, help="number of missing data PCA iterations")
+@click.option('--mask-threshold', default=-16, type=float, help="Threshold for mask (missing data PCA only)")
 @click.option('--mask-height-threshold', default=5, type=float, help="Threshold for mask based on floor height")
 @click.option('--min-height', default=10, type=int, help='Min mouse height from floor (mm)')
 @click.option('--max-height', default=120, type=int, help='Max mouse height from floor (mm)')
 @click.option('--tailfilter-size', default=(9, 9), type=(int, int), help='Tail filter size')
 @click.option('--tailfilter-shape', default='ellipse', type=str, help='Tail filter shape')
 @click.option('--use-fft', type=bool, is_flag=True, help='Use 2D fft')
+@click.option('--train-on-subset', default=1, type=float, help="The fraction of the total frames the PCA is trained on; default PCA is trained on all frames")
 @click.option('--recon-pcs', type=int, default=10, help='Number of PCs to use for missing data reconstruction')
-@click.option('--rank', default=25, type=int, help="Rank for compressed SVD (generally>>nPCS)")
+@click.option('--rank', default=25, type=int, help="Rank for compressed SVD")
 @click.option('--output-file', default='pca', type=str, help='Name of h5 file for storing pca results')
 @click.option('--local-processes', default=False, type=bool, help='Used with a local cluster. If True: use processes, If False: use threads')
 @click.option('--overwrite-pca-train', default=False, type=bool, help='Used to bypass the pca overwrite question. If True: skip question, run automatically')
@@ -114,14 +109,14 @@ def train_pca(input_dir, output_dir, output_file, **cli_args):
         combine_new_config(cli_args.get('config_file'), config_data)
 
 
-@cli.command(name='apply-pca', cls=command_with_config('config_file'), help='Computes PCA Scores of extraction data given a pre-trained PCA')
+@cli.command(name='apply-pca', cls=command_with_config('config_file'), help='Compute PCA Scores of extraction data given a pre-trained PCA')
 @common_pca_options
 @common_dask_parameters
 @click.option('--output-file', default='pca_scores', type=str, help='Name of h5 file for storing pca results')
-@click.option('--pca-path', default='/components', type=str, help='Path to pca components')
+@click.option('--pca-path', default='/components', type=str, help='Path to pca components in h5 file')
 @click.option('--pca-file', default=None, type=click.Path(), help='Path to PCA results')
 @click.option('--fill-gaps', default=True, type=bool, help='Fill dropped frames with nans')
-@click.option('--fps', default=30, type=int, help='Fps (only used if no timestamps found)')
+@click.option('--fps', default=30, type=int, help='Frames per second (frame rate)')
 @click.option('--detrend-window', default=0, type=float, help="Length of detrend window (in seconds, 0 for no detrending)")
 @click.option('--verbose', '-v', is_flag=True, help='Print sessions as they are being loaded.')
 @click.option('--overwrite-pca-apply', default=False, type=bool, help='Used to bypass the pca overwrite question. If True: skip question, run automatically')
@@ -134,7 +129,7 @@ def apply_pca(input_dir, output_dir, output_file, **cli_args):
         combine_new_config(cli_args.get('config_file'), config_data)
 
 
-@cli.command('compute-changepoints', cls=command_with_config('config_file'), help='Computes the Model-Free Syllable Changepoints based on the PCA/PCA_Scores')
+@cli.command('compute-changepoints', cls=command_with_config('config_file'), help='Compute the Model-Free Syllable Changepoints based on the PCA/PCA_Scores')
 @common_pca_options
 @common_dask_parameters
 @click.option('--output-file', default='changepoints', type=str, help='Name of h5 file for storing pca results')
@@ -146,8 +141,8 @@ def apply_pca(input_dir, output_dir, output_file, **cli_args):
 @click.option('-k', '--klags', type=int, default=6, help="Lag to use for derivative calculation")
 @click.option('-s', '--sigma', type=float, default=3.5, help="Standard deviation of gaussian smoothing filter")
 @click.option('-d', '--dims', type=int, default=300, help="Number of random projections to use")
-@click.option('--fps', default=30, type=int, help='Fps (only used if no timestamps found)')
-@click.option('--verbose', '-v', is_flag=True, help='Print sessions as they are being loaded.')
+@click.option('--fps', default=30, type=int, help="Frames per second (frame rate)")
+@click.option('--verbose', '-v', is_flag=True, help="Print sessions as they are being loaded.")
 def compute_changepoints(input_dir, output_dir, output_file, **cli_args):
     # function writes output changepoint path to config_data
     config_data = compute_changepoints_wrapper(input_dir, cli_args, output_dir, output_file)
@@ -157,10 +152,10 @@ def compute_changepoints(input_dir, output_dir, output_file, **cli_args):
         combine_new_config(cli_args.get('config_file'), config_data)
 
 
-@cli.command('clip-scores',  help='Clips specified number of frames from PCA scores at the beginning or end')
+@cli.command('clip-scores',  help='Clip specified number of frames from PCA scores at the beginning or end')
 @click.argument('pca_file', type=click.Path(exists=True, resolve_path=True))
 @click.argument('clip_samples', type=int)
-@click.option('--from-end', type=bool, is_flag=True)
+@click.option('--from-end', type=bool, is_flag=True, help="if true clip from end rather than beginning")
 def clip_scores(pca_file, clip_samples, from_end):
     clip_scores_wrapper(pca_file, clip_samples, from_end)
 

moseq2_pca/gui.py

@@ -1,12 +1,7 @@
-'''
+"""
+GUI front-end operations for PCA.
 
-GUI front-end operations. This module contains all the functionality and configurable parameters
-users can alter to most accurately process their data.
-
-Note: These functions perform jupyter notebook specific preprocessing, loads in corresponding parameters from the
-CLI functions, then call the corresponding wrapper function with the given input parameters.
-
-'''
+"""
 
 import warnings
 import ruamel.yaml as yaml
@@ -17,18 +12,16 @@
 
 
 def train_pca_command(progress_paths, output_dir, output_file):
-    '''
+    """
     Train PCA through Jupyter notebook, and updates config file.
 
-    Parameters
-    ----------
+    Args:
     progress_paths (dict): dictionary containing notebook progress paths
     output_dir (str): path to output pca directory
     output_file (str): name of output pca file.
 
-    Returns
-    -------
-    '''
+    Returns:
+    """
     # Get default CLI params
     default_params = {tmp.name: tmp.default for tmp in train_pca.params if not tmp.required}
 
@@ -54,18 +47,16 @@ def train_pca_command(progress_paths, output_dir, output_file):
 
 
 def apply_pca_command(progress_paths, output_file):
-    '''
+    """
     Compute PCA Scores given trained PCA using Jupyter Notebook.
 
-    Parameters
-    ----------
+    Args:
     progress_paths (dict): dictionary containing notebook progress paths
     output_file (str): name of output pca file.
 
-    Returns
-    -------
+    Returns:
     (str): success string.
-    '''
+    """
     # Get default CLI params
     default_params = {tmp.name: tmp.default for tmp in apply_pca.params if not tmp.required}
 
@@ -109,19 +100,17 @@ def apply_pca_command(progress_paths, output_file):
 
 
 def compute_changepoints_command(input_dir, progress_paths, output_file):
-    '''
+    """
     Compute Changepoint distribution using Jupyter Notebook.
 
-    Parameters
-    ----------
+    Args:
     input_dir (str): path to directory containing training data
     progress_paths (dict): dictionary containing notebook progress paths
     output_file (str): name of output pca file.
 
-    Returns
-    -------
+    Returns:
     (str): success string.
-    '''
+    """
     # Get default CLI params
     default_params = {tmp.name: tmp.default for tmp in compute_changepoints.params
                       if not tmp.required}