Eac/docs (#177)

* WIP on docs

* WIP, docs

* updated docs to autogenerate lots of stuff

* updated .gitignore to ignore locally rendered jupyter notebooks

* tweak docs/conf.py

* adding some content to docs

* doc updates

.gitignore

@@ -71,7 +71,10 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/api
+docs/api.rst
 _readthedocs/
+docs/*_examples/*.html
 
 # PyBuilder
 target/

docs/conf.py

@@ -16,14 +16,14 @@
 import subprocess
 import sys
 sys.path.insert(0, os.path.abspath('..'))
+
 import rail
+
 from rail.core import RailEnv
 
 for rail_path in rail.__path__:
     sys.path.insert(0, rail_path)
 
-print(sys.path)
-
 
 # Use unittest mock module to shield some modules away from docs building.
 # This way one does not need to install them when dealing with the doc.
@@ -46,11 +46,14 @@
     'fsps',
     'dsps',
     'dsps.cosmology',
+    'lephare',
     'pzflow',
     'pzflow.bijectors',
     'sklearn',
     'sklearn.cluster',
     'sklearn.decomposition',
+    'sklearn.tree',
+    'sklearn.ensemble',
     'gal_pop_model_components',
     'qp_flexzboost',
 ]
@@ -97,6 +100,7 @@
     'sphinx.ext.napoleon',
     'sphinx.ext.autosectionlabel',
     'sphinx_tabs.tabs',
+    'sphinx_click',    
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -130,6 +134,25 @@
 # Allow NB to fail
 nbsphinx_allow_errors = True
 
+# use type hints in autodoc
+autodoc_typehints = "description"
+
+autodoc_type_aliases = {
+    'DataLike': 'rail.core.data.DataLike',
+    'FileLike': 'rail.core.data.FileLike',
+    'GroupLike': 'rail.core.data.GroupLike',
+    'ModelLike': 'rail.core.data.ModelLike',
+    'TableLike': 'rail.core.data.TableLike',
+}
+
+autodoc_class_signature = "separated"
+
+autodoc_default_options = {
+    'special-members': '__init__',
+    'undoc-members': True,
+    'exclude-members': 'config_options'
+}
+
 # By default, tabs can be closed by selecting the open tab. This
 # functionality can be disabled using the sphinx_tabs_disable_tab_closing
 # configuration option:
@@ -192,19 +215,12 @@
 # from sphinxcontrib.apidoc import main as apidoc_main
 
 def run_apidoc(_):
-    #os.system('ln -s ../examples')
-
-    from sphinx.ext.apidoc import main as apidoc_main
-    cur_dir = os.path.normpath(os.path.dirname(__file__))
-    output_path = os.path.join(cur_dir, 'api')
-
-    for full_path in rail.__path__:
-        paramlist = ['--separate', '--implicit-namespaces', '--no-toc', '-M', '-o', output_path, '-f', full_path]
-        print(f"running {paramlist}")
-        apidoc_main(paramlist)
 
+    if os.environ.get('RAIL_NO_REBUILD_API', False):
+        return
+    RailEnv.do_stage_type_api_rst()    
     RailEnv.do_api_rst()
-
+    
 
 def setup(app):
     app.connect('builder-inited', run_apidoc)

docs/index.rst

@@ -12,15 +12,37 @@ To get involved, chime in on the issues in any of the RAIL repositories describe
 
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 4
    :caption: Getting Started
 
    source/overview
    source/installation
    source/citing
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 4
+   :caption: Concepts
+
+   source/stages
+   source/data_handles
+   source/pipelines
+   source/catalog_tags
+
+.. toctree::
+   :maxdepth: 4
+   :caption: Details
+
+   source/documentation
+   api/stage_types
+
+.. toctree::
+   :maxdepth: 4
+   :caption: Usage
+
+   source/rail_cli
+
+.. toctree::
+   :maxdepth: 4
    :caption: Contributing
 
    source/contributing
@@ -30,13 +52,13 @@ To get involved, chime in on the issues in any of the RAIL repositories describe
    source/sharing_pipeline
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 5
    :caption: API
 
    api
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 4
    :caption: Usage Demos
 
    Overview of Demonstrations <source/demonstrations>

docs/source/catalog_tags.rst

@@ -0,0 +1,46 @@
+*******************************
+Configuring RAIL for input data
+*******************************
+
+====================================
+Variation in input catalog structure
+====================================
+
+`RAIL` is designed to be used with a variety of different data.
+Depending on the data in question, things like the names of the
+columns associted to the particular quantities like the true redshift
+of a simulated object, or the names of the columns with the various
+observed magnitudes in different filters, will vary.  By enforcing
+consistency in naming conventions between different `RailStage`
+sub-classes we have made it simple configure `RAIL` to read data from
+a particular source, rather than having to edit the configurations
+for many different `RailStages`.
+
+=================
+Shared Parameters
+=================
+
+.. autoclass:: rail.core.common_params.SharedParams
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+============
+Catalog Tags
+============
+
+:py:class:`rail.util.catalog_utils.CatalogConfigBase` provides an
+interface to switch between different input catalogs.
+
+.. autoclass:: rail.utils.catalog_utils.CatalogConfigBase
+    :noindex:
+
+.. automethod:: rail.utils.catalog_utils.CatalogConfigBase.apply
+    :noindex:
+
+.. automethod:: rail.utils.catalog_utils.CatalogConfigBase.active_class
+    :noindex:
+
+.. automethod:: rail.utils.catalog_utils.CatalogConfigBase.active_tag
+    :noindex:

docs/source/contributing.rst

@@ -6,9 +6,8 @@ RAIL is a constellation of multiple packages developed publicly on GitHub and
 welcomes all interested developers, regardless of DESC membership or LSST data rights.
 
 If you're interested in contributing, but don't know where to start, take a look 
-at the list of good first issues from all RAIL repositories 
-`here <https://github.com/orgs/LSSTDESC/projects/6/views/20>`_.
-Or, create a new issue `here <https://github.com/LSSTDESC/rail/issues/new>`_ to 
+at the list of `good first issues from all RAIL repositories <https://github.com/orgs/LSSTDESC/projects/6/views/20>`_.
+Or, `create a new issue <https://github.com/LSSTDESC/rail/issues/new>`_ to 
 suggest a change, and the team will route it to the appropriate repository.
 
 In addition to GitHub, the RAIL team uses the LSSTC Slack workspace for organization.
@@ -19,7 +18,7 @@ the `\#desc-pz-rail <https://lsstc.slack.com/archives/CQGKM0WKD>`_ channel on
 the LSSTC Slack workspace.
 
 Those without data rights who wish to gain access to the Slack channel should 
-`create an Issue <https://github.com/LSSTDESC/RAIL/issues/new>`_ to request that 
+`create an issue <https://github.com/LSSTDESC/RAIL/issues/new>`_ to request that 
 the team leads initiate the process for adding a DESC External Collaborator.
 
 
@@ -73,8 +72,7 @@ While developing in a branch, don't forget to pull from ``main`` regularly (at
 least daily) to make sure your work is compatible with other recent changes.
 
 When you're ready to merge your branch into the ``main`` branch, create a pull request
-("PR") in the rail repository you cloned from. GitHub has instructions 
-`here <https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request>`_.
+("PR") in the rail repository you cloned from. `GitHub has instructions <https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request>`_.
 
 Several continuous integration checks will be performed for new pull requests. 
 If any of these automatic processes find issues with the code, you should address 
@@ -133,8 +131,8 @@ We follow the `pep8 <https://peps.python.org/pep-0008/#descriptive-naming-styles
 recommendations for naming new modules and ``RailStage`` classes within them.
 
 
-Modules
--------
+Modules Names
+-------------
 
 Modules should use all lowercase, with underscores where it aids the readability
 of the module name. 
@@ -148,8 +146,8 @@ It's good for the module name to specify the source of the implementation of a p
 Note that these names should not be identical to the name of the package the algorithm came from, to avoid introducing namespace collisions for users who have imported the original package as well, i.e. ``pzflow_nf`` is a safer name than ``pzflow``.
 
 
-Stages
-------
+Stage Names
+-----------
 
 RailStages are python classes and so should use the CapWords convention. All 
 rail stages using the same algorithm should use the same short, descriptive 
@@ -170,6 +168,32 @@ Possible suffixes include:
 * Degrader
 * Evaluator
 
+
+
+RAIL Documentation
+==================
+
+All of the documentation on this site is built as part of the 
+`RAIL Package <https://github.com/lsstdesc/rail>`_, and the
+configuration and skeleton for the documenation live in the
+`rail/docs` directory.
+
+The documation skeleton is setup to work with whatever rail
+packages are installed, e.g., if you are just working on one
+of the algorithms, you can just install that algorithm in
+addition to `rail_base` and `rail` and when you generate the
+docs you will just get the docs for `rail_base` and that package.
+
+The documentation uses `sphinx
+<https://www.sphinx-doc.org/en/master/>`_
+to automatically generate some content from the source code,
+this requires being very careful with docstring formatting.   The rail_base package is pretty good about using the numpy docstring style, so I think we should migrate things towards that: https://numpydoc.readthedocs.io/en/latest/format.html
+the documentation uses some python cleverness to find all the rail code in your current python environment, this lives in: rail_base/src/rail/core/introspection.py 
+the rail package is linked to readthedocs so that anytime we push to main it will get pulled over to readthedocs. 
+
+
+
+
 
 Contribution Types
 ==================