Minimal CLI

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    */tests/*

.github/workflows/main.yaml

@@ -0,0 +1,72 @@
+# Compare: https://github.com/pangeo-forge/pangeo-forge-recipes/blob/master/.github/workflows/main.yaml
+
+name: Tests
+
+on:
+  push:
+    branches: "*"
+    paths-ignore:
+    - 'docs/**'
+  pull_request:
+    branches: master
+    paths-ignore:
+    - 'docs/**'
+
+env:
+  PYTEST_ADDOPTS: "--color=yes"
+
+jobs:
+  test:
+    name: ${{ matrix.python-version }}-build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      # - name: Cache conda
+      #  uses: actions/cache@v1
+      #  env:
+          # Increase this value to reset cache if ci/py${{ matrix.python-version }}.yml has not changed
+      #    CACHE_NUMBER: 0
+      #  with:
+      #    path: ~/conda_pkgs_dir
+      #    key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/py${{ matrix.python-version }}.yml') }}
+      - name: setup miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: pfo
+      #    environment-file: ci/py${{ matrix.python-version }}.yml
+          python-version: ${{ matrix.python-version }}
+          auto-activate-base: false
+      #    use-only-tar-bz2: true
+      - name: install pangeo-forge-orchestrator plus deps
+        shell: bash -l {0}
+        run: |
+          pip install -e '.[dev]'
+      - name: print conda env
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - name: Run Tests
+        shell: bash -l {0}
+        run: |
+          pytest tests -v --cov=pangeo_forge_orchestrator \
+            --cov-config .coveragerc \
+            --cov-report term-missing \
+            --cov-report xml \
+            --durations=10 --durations-min=1.0
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          file: ./coverage.xml
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: false

.github/workflows/pre-commit.yaml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+# See: https://github.com/pangeo-forge/pangeo-forge-recipes/blob/master/.pre-commit-config.yaml
+
+repos:
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.1.0
+    hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-docstring-first
+    - id: check-json
+    - id: check-yaml
+    - id: pretty-format-json
+      args: ["--autofix", "--indent=2", "--no-sort-keys"]
+
+-   repo: https://github.com/ambv/black
+    rev: 19.10b0
+    hooks:
+    - id: black
+      args: ["--line-length", "100"]
+
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+    - id: flake8
+      args:
+        - "--max-line-length=100"
+
+-   repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
+    hooks:
+    - id: seed-isort-config
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.910'
+    hooks:
+    -   id: mypy
+        exclude: tests
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.5.4
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+
+-   repo: https://github.com/myint/rstcheck
+    rev: 3f92957478422df87bd730abde66f089cc1ee19b
+    hooks:
+    -   id: rstcheck

.readthedocs.yml

@@ -0,0 +1,10 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.8
+  install:
+    - requirements: docs/requirements.txt

codecov.yml

@@ -0,0 +1,20 @@
+codecov:
+  require_ci_to_pass: no
+  max_report_age: off
+
+comment: false
+
+coverage:
+  precision: 2
+  round: down
+  status:
+    project:
+      default:
+        target: 95
+        informational: true
+    patch: off
+    changes: off
+
+ignore:
+  - "tests/*"
+  - "**/__init__.py"

docs/Makefile

@@ -0,0 +1,22 @@
+# Compare: https://github.com/pangeo-forge/pangeo-forge-recipes/blob/master/docs/Makefile
+
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,47 @@
+# Compare: https://github.com/pangeo-forge/pangeo-forge-recipes/blob/master/docs/conf.py
+
+# -- Project information -----------------------------------------------------
+
+project = "Pangeo Forge Orchestrator"
+copyright = "2021, Pangeo Community"
+author = "Pangeo Community"
+
+# -- General configuration ---------------------------------------------------
+
+extensions = [
+    "myst_nb",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.extlinks",
+    # "numpydoc",
+    "sphinx_autodoc_typehints",
+    "sphinx_copybutton",
+]
+
+extlinks = {
+    "issue": ("https://github.com/pangeo-forge/pangeo-forge-orchestrator/issues/%s", "GH issue "),
+    "pull": ("https://github.com/pangeo-forge/pangeo-forge-orchestrator/pull/%s", "GH PR "),
+}
+
+exclude_patterns = ["_build", "**.ipynb_checkpoints"]
+master_doc = "index"
+
+# we always have to manually run the notebooks because they are slow / expensive
+# jupyter_execute_notebooks = "auto"
+# execution_excludepatterns = ["tutorials/xarray_zarr/*", "tutorials/hdf_reference/*"]
+
+# -- Options for HTML output -------------------------------------------------
+
+# https://sphinx-book-theme.readthedocs.io/en/latest/configure.html
+html_theme = "pangeo_sphinx_book_theme"
+html_theme_options = {
+    "repository_url": "https://github.com/pangeo-forge/pangeo-forge-orchestrator",
+    "repository_branch": "main",
+    "path_to_docs": "docs",
+    "use_repository_button": True,
+    "use_issues_button": True,
+    "use_edit_page_button": True,
+}
+html_logo = "_static/pangeo-forge-logo-blue.png"
+html_static_path = ["_static"]
+
+myst_heading_anchors = 2

docs/index.md

@@ -0,0 +1,19 @@
+# Pangeo Forge Orchestrator
+
+```{warning}
+The official Pangeo Forge docs can be found at [https://pangeo-forge.readthedocs.io/](https://pangeo-forge.readthedocs.io/en/latest/).
+
+You have found the documentation for for `pangeo-forge-orchestrator`, an unreleased package.
+```
+
+```{toctree}
+:maxdepth: 1
+
+motivation
+quick_start
+structural_view
+new_pydantic_types
+use_guide
+testing_strategy
+next_steps
+```

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/motivation.md

@@ -0,0 +1,13 @@
+# Motivation
+
+The first practical application of this package (even before we refactor other automation repos into it) is to orchestrate the process of generating STAC Items for datasets which have already been built with Pangeo Forge. All of the metadata required for cataloging already exists _somewhere_ in Pangeo Forge:
+
+- the datasets themselves are in a Bakery target somewhere
+- other metadata is available in the Feedstocks's `meta.yaml`
+
+...so a generalizable cataloging approach needs to know:
+
+- how to access a given Bakery target and open a dataset therein
+- which datasets reside at which paths at the given Bakery target
+- which Feedstocks (including Feedstock [version](https://github.com/pangeo-forge/roadmap/pull/34)) those paths were built from
+- how to read/parse a `meta.yaml` from a given versioned Feedstock

docs/new_pydantic_types.md

@@ -0,0 +1,66 @@
+# New pydantic types
+
+In updating `meta_types/bakery.py` (as mentioned in {doc}`structural_view`), `pangeo-forge-orchestrator` aims to implement Pydantic-based input validation with the lightest touch possible for each input type. The types were initially implemented as Python dataclasses, so for many of them, the edit was simply to use `pydantic.dataclasses` as a drop-in replacement (perhaps in combination with stricter type hints). In addition, three new dataclasses, two new Models, and some regex-constrained type functions are defined; these are described below.
+
+## `BakeryName`
+
+This dataclass validates a string against the [Bakery naming scheme defined in the Bakery database ADR](https://github.com/pangeo-forge/roadmap/blob/master/doc/adr/0004-use-yaml-file-for-bakery-database.md#bakery-name). As I understand it, the stipulation to "follow [java package syntax](https://docs.oracle.com/javase/tutorial/java/package/namingpkgs.html) to ensure unique bakery names" means that these names should all begin with a reversed organization url. Therefore, I'm making the assumption (reflected in this dataclass's type hinting) that un-reversing this portion of the Bakery name should yield a valid `pydantic.HttpUrl`. (More on this in {doc}`use_guide`.) In addition, following the ADR, an acceptable Bakery name input string must conclude with `".bakery.{region}"` where region conforms to an `"{provider}.{region}"` format.
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_orchestrator.meta_types.bakery.BakeryName
+    :members:
+```
+
+## `RunRecord`
+
+> This type (along with the next one, `BuildLogs`) provide the necessary link, mentioned in {doc}`motivation`, between datasets in a Bakery target and the Feedstocks from which they were built. They are new concepts for Pangeo Forge and certainly merit their own ADR. After we move through this PR (assuming we agree that some version of these concepts are useful), we can work on an ADR for them.
+
+This is a container for metadata describing the execution of a Pangeo Forge Recipe, including:
+- **timestamp**: The datetime at which the execution took place (not sure if this should be start, end, or maybe tuple of both).
+- **feedstock**: The name of the feedstock (including version). For this PR, I'm provisionally using the format `"{feedstock_name}@{major_version}.{minor_version}"`.
+- **recipe**: The name of the Python object within the Feedstock's `recipe.py` module which was used to build the dataset. In the case of a single-recipe module, this is the name of a `pangeo_forge_recipes.recipes` class instance (and therefore needs to be a valid Python identifier). In the case of a [`dict_object`](https://github.com/pangeo-forge/roadmap/blob/master/doc/adr/0002-use-meta-yaml-to-track-feedstock-metadata.md#recipes-section), this would follow the established convention for the `meta.yaml`: i.e., `"{dict_name}:{dict_key}"`.
+- **path**: The relative path to the dataset within the Bakery storage target.
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_orchestrator.meta_types.bakery.RunRecord
+    :members:
+```
+
+
+## `BuildLogs`
+
+This is a mapping between an execution run identifier and a `RunRecord`. Provisionally, I'm specifying a run identifier as a (five digit) integer string, i.e. `"00000"`, `"00001"` etc. These values would be sequentially assigned to each dataset as they are built to a given Bakery storage target. (And each Bakery target would keep its own tally.) There are other identifiers (i.e. the dataset path) which will be unique within a given Bakery storage target, but the idea here is to provide a short string for passing as a command line argument (an example of this is provided in the {doc}`use_guide`).
+
+> Currently, the `pangeo_forge_orchestrator.components.Bakery` object assumes that a `build_logs.json` (with entries parsable into `BuildLogs` objects) will exist at the Bakery target's root path. By opening and parsing this JSON, the `Bakery` instance knows exactly what dataset paths exist at the target and what Feedstocks they are tied to. In the future, these records could be ingested into a database instead of, or in addition to, keeping a copy in the Bakery target.
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_orchestrator.meta_types.bakery.BuildLogs
+    :members:
+```
+
+## Why is `StorageOptions` a Model?
+
+The Bakery database ADR [defines a `storage_options` field](https://github.com/pangeo-forge/roadmap/blob/master/doc/adr/0004-use-yaml-file-for-bakery-database.md#storage-options) for Bakery target access parameters. The reason we're defining the Python container for these options as a pydantic Model, rather than a dataclass, is for the [`.dict(exclude_none=True)` method](https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict), which allows us to define arbitrary numbers of optional type-checked fields for this object, but also succinctly export kwargs dictionaries representing only those fields which have been set on a given instance.
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_orchestrator.meta_types.bakery.StorageOptions
+    :members:
+```
+
+
+## ... and what about `BakeryDatabase`?
+
+The new `BakeryDatabase` object is a Model (instead of a dataclass) because we want to take advantage of Model features for `pangeo_forge_orchestrator.components.Bakery`, and it seemed to make sense to have `Bakery` inherit from `BakeryDatabase`.
+
+```{eval-rst}
+.. autoclass:: pangeo_forge_orchestrator.meta_types.bakery.BakeryDatabase
+    :members:
+```
+
+## What's with those regexes?
+
+There are certain string values (such as the Feedstock name, etc.) which we definitely want to ensure conform to a specified format, but for which it seemed excessive to define an entire class for. For [these cases](https://github.com/pangeo-forge/pangeo-forge-orchestrator/blob/620989215c8d191d55c3080d403d6454a895230b/pangeo_forge_orchestrator/meta_types/bakery.py#L110-L121), I opted to use pydantic's [Constrained Type function, `constr`](https://pydantic-docs.helpmanual.io/usage/types/#constrained-types). I (and I think most people) don't find regular expressions especially readable, so I wrote explanatory comments for each of these cases.
+
+## Full diff
+
+And finally, here's [the full diff](https://github.com/pangeo-forge/pangeo-forge-orchestrator/compare/de36c30070f249136a5eb3c0f54144f3eaafb428..620989215c8d191d55c3080d403d6454a895230b#diff-374b3112607d6019e80fa96dff3aec0f9159e803faf62a96ef35330f308bff9b) between Sean's existing types, and those proposed in this PR.

docs/next_steps.md

@@ -0,0 +1,6 @@
+# Next steps
+
+- Circle back to Orchestrator and BuildLogs ADR proposals
+- Bring the official Bakery database within spec, as described in the {doc}`use_guide`
+- Update the `catalog` CLI (and related pydantic types) to make it possible to build STAC Items for datasets built from un-merged Feedstocks
+- Begin to merge automation repositories (i.e. unmerged portions of `pangeo-forge-prefect`) and refactor related GitHub Actions accordingly

docs/quick_start.md

@@ -0,0 +1,10 @@
+# Quick start
+
+`pangeo-forge-orchestrator` uses Poetry for dependencies/packaging, because this is [recommended by Typer](https://typer.tiangolo.com/tutorial/package/). For a local install, you can follow [the steps defined by the CI tests](https://github.com/pangeo-forge/pangeo-forge-orchestrator/blob/620989215c8d191d55c3080d403d6454a895230b/.github/workflows/main.yaml#L41-L57), namely:
+
+1. Create & activate a new conda env named `pfo-poetry` with python >= 3.8
+    > "pfo" for "pangeo-forge-orchestrator"; the environment name doesn't matter if you don't plan to execute example notebooks with Papermill
+2. From the repo root run: `conda install -c conda-forge poetry && poetry install`
+3. (Optionally) Run `python -m ipykernel install --user --name pfo-poetry` if you want to execute example notebooks with Papermill
+
+The CLI entrypoint is `pangeo-forge`, so you can then start exploring with `pangeo-forge --help`. There are a limited number of commands that actually work now, they are detailed in {doc}`use_guide`.