Field guide and workflows (#4)

* set up docs pipeline

* rm __init__ in scope.Scope

* make exec

* debug docs GA

* deploy docs only when github.ref == 'refs/heads/main'

* test deployment

* set up field guide-related code

* fix requirements

* use default config on CI

* use K token on CI

* use default config on CI

* os.getenv -> os.environ

* os.getenv -> os.environ

* os.getenv -> os.environ

* secrets are not available to prs from forks, sigh

* secrets are not available to prs from forks, sigh

* secrets are not available to prs from forks, sigh

* secrets are not available to prs from forks, sigh

* generate images

* generate images

* generate images

* generate images

* generate hr diagrams

* field guide

* generate docs only if Kowalski connection is established

* generate docs only if Kowalski connection is established

* added 'Contributing Field Guide sections' section to docs

* minor doc fixes

* minor doc fixes

* parametrize cone search

* filter flagged data; fix msip start date and typos

* generate taxonomy vizualization with tdtax

* generate taxonomy vizualization with tdtax

* add status text

* make Gaia collection name configurable

.github/workflows/docs.yml

@@ -0,0 +1,48 @@
+name: Build, test, and upload docs
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.8'
+
+    - name: Install dependencies and use default config
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+        pip install -r requirements.txt
+        cp config.defaults.yaml config.yaml
+        ./scope.py develop
+    - name: Lint sources
+      run: |
+        ./scope.py lint
+    - name: Build docs
+      env:
+        KOWALSKI_TOKEN: ${{ secrets.KOWALSKI_TOKEN }}
+      run: |
+        ./scope.py doc
+    - name: Install SSH Client 🔑
+      if: github.ref == 'refs/heads/main'
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: ${{ secrets.CI_DEPLOY_KEY }}
+
+    - name: Deploy docs
+      if: github.ref == 'refs/heads/main'
+      uses: JamesIves/github-pages-deploy-action@releases/v3
+      with:
+        FOLDER: doc/_build/html
+        REPOSITORY_NAME: ZwickyTransientFacility/scope-docs
+        BRANCH: master
+        SINGLE_COMMIT: true
+        SSH: true

.github/workflows/lint.yml

@@ -14,10 +14,13 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: '3.8'
-      - name: Set up pre-commit
+      - name: Install dependencies
         run: |
-          pip install pre-commit
-          pre-commit install
+          python -m pip install --upgrade pip
+          pip install wheel
+          pip install -r requirements.txt
+          cp config.defaults.yaml config.yaml
+          ./scope.py develop
       - name: Lint sources
         run: |
-          pre-commit run --all-files
+          ./scope.py lint

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+config.yaml

.requirements/dev.txt

@@ -0,0 +1,4 @@
+deepdiff>=5.0.2
+matplotlib>=3.3
+numpy>=1.20
+questionary>=1.8.1

.requirements/doc.txt

@@ -0,0 +1,9 @@
+fire>=0.4.0
+myst-parser
+pandas>=1.2
+penquins>=2.1
+pre-commit
+pyyaml>=5.3.1
+sphinx<3
+sphinx_press_theme
+tdtax>=0.1.3

README.md

@@ -1,3 +1,3 @@
 # SCoPe: ZTF source classification project
 
-[Developer guidelines](docs/developer.md)
+The documentation is hosted at [https://scope.ztf.dev](https://scope.ztf.dev).

config.defaults.yaml

@@ -0,0 +1,53 @@
+kowalski:
+  host: kowalski.caltech.edu
+  port: 443
+  protocol: https
+  token:
+  collections:
+    gaia: Gaia_EDR3
+    sources: ZTF_sources_20201201
+taxonomy:
+  name: Class type
+  children:
+    - name: phenomenological
+      children:
+        - name: variable
+        - name: periodic
+        - name: long timescale
+        - name: irregular
+        - name: eclipsing
+          children:
+            - name: EA
+            - name: EB
+            - name: EW
+        - name: flaring
+        - name: bogus
+    - name: ontological
+      children:
+        - name: pulsator
+          children:
+            - name: Delta Scuti
+            - name: Cepheid
+            - name: RR Lyrae
+            - name: LPV
+              children:
+                - name: Mira
+                - name: SRV
+        - name: binary star
+          children:
+            - name: W Uma
+            - name: Beta Lyr
+            - name: RS CVn
+        - name: AGN
+        - name: YSO
+        - name: CV
+docs:
+  field_guide:
+    rr_lyr_ab:
+      coordinates: [254.9327, 30.8264]
+      period: 0.6299307435824886
+      title: RR Lyrae ab
+    rr_lyr_c:
+      coordinates: [254.7213, 32.8341]
+      period: 0.31087445597147706
+      title: RR Lyrae c

data/golden/rr_lyr.csv

@@ -0,0 +1,101 @@
+ra,dec
+6.6972,39.7288
+275.1411,25.8975
+265.0333,46.1262
+44.5261,8.2823
+202.9136,1.5778
+258.7869,-19.3287
+297.1027,-6.1235
+299.8918,42.2229
+2.1362,-1.3614
+191.1991,-2.0239
+282.4362,21.3963
+33.8971,8.2533
+290.2702,28.3179
+330.1756,18.2450
+238.4761,14.3448
+294.3752,10.8233
+140.2169,29.8618
+260.9076,17.3743
+227.0172,19.9154
+343.5809,15.7559
+313.3120,2.1652
+107.0656,19.8276
+246.9389,41.6731
+264.7615,8.1273
+272.8852,4.3133
+3.6658,44.7615
+259.6061,77.5957
+247.4274,17.4696
+144.2322,-27.2971
+201.7502,-0.9157
+355.0173,12.6335
+344.9450,11.1558
+279.4997,29.5152
+234.4930,13.1586
+12.4202,-17.9360
+224.9373,47.0693
+205.4918,28.3662
+189.1692,12.3411
+140.0693,17.5282
+136.0478,-5.2486
+297.6617,40.9226
+239.9427,1.0440
+153.2488,-7.6505
+45.3215,10.5618
+342.4765,-22.6354
+144.8815,41.4392
+327.8358,-3.8525
+0.0990,36.3283
+271.3058,1.3096
+284.5697,31.2490
+347.9852,-11.3169
+290.8039,27.9467
+247.3798,18.4955
+134.2435,-4.3487
+310.8780,17.1872
+98.3473,34.3550
+2.1380,-17.6860
+149.4252,-8.4441
+189.0454,4.9512
+223.8956,-20.2835
+227.2722,-11.4610
+11.3808,37.4950
+283.0494,29.3524
+294.2435,-17.6601
+287.2464,24.7713
+263.0360,27.1874
+315.1415,10.6395
+301.5676,-6.2720
+317.9104,23.3196
+2.2453,-15.2805
+5.2808,22.8316
+21.2562,-9.3134
+301.4226,58.0469
+315.6813,15.2190
+298.6058,17.3279
+227.0721,3.0619
+206.0271,0.1938
+183.8275,14.4614
+171.8987,26.6034
+339.3016,5.9743
+280.1734,30.9392
+273.1738,75.6190
+134.8732,18.3707
+239.1320,14.0647
+150.3709,0.1833
+41.4213,-19.1045
+302.3398,15.8138
+87.2335,41.9438
+229.2072,2.8119
+312.5179,-8.4091
+176.4899,25.7268
+39.9926,33.0617
+344.2298,-24.8992
+296.8557,1.2522
+178.5740,15.9890
+172.8428,24.7346
+210.7209,8.0693
+186.2641,6.5419
+104.8267,22.8503
+15.9448,-16.1852

doc/.gitignore

@@ -0,0 +1 @@
+_build

doc/Makefile

@@ -0,0 +1,26 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+html:
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	cp -r data _build/html
+	touch _build/html/.nojekyll
+	echo scope.ztf.dev > _build/html/CNAME

doc/conf.py

@@ -0,0 +1,58 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "ZTF Variable Source Classification Project"
+copyright = "2021, The SCoPe collaboration"
+author = "The SCoPe collaboration"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ["myst_parser"]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "press"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+
+def setup(app):
+    app.add_config_value(
+        "recommonmark_config", {"auto_toc_tree_section": "Contents"}, True
+    )
+    app.add_css_file("custom.css")