Merge pull request #2 from lsst-dm/DM-31213

DM-31213: Add github actions-based CI
lsst-dm · Aug 9, 2021 · 4385145 · 4385145
2 parents e780649 + b9540fa
commit 4385145
Show file tree

Hide file tree

Showing 10 changed files with 228 additions and 36 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,69 @@
+name: ci
+
+on:
+  - push
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install package
+        run: pip install .[dev]
+
+      - name: Run commit-blocking lint checks
+        run: pre-commit run --all-files
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Pytest
+        run: pip install pytest
+
+      - name: Install package
+        run: pip install .
+
+      - name: Run unit tests
+        run: pytest
+
+  integration-test:
+    runs-on: ubuntu-latest
+    concurrency: integration-test
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Pytest
+        run: pip install pytest
+
+      - name: Install package
+        run: pip install .
+
+      - name: Set up cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Run integration tests
+        run: pytest
+        env:
+          ALERTDB_TEST_GCP_PROJECT: ${{ secrets.GCP_PROJECT_ID }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,33 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
+
+- repo: https://github.com/PyCQA/flake8
+  rev: 3.9.2
+  hooks:
+  - id: flake8
+
+- repo: https://github.com/PyCQA/isort
+  rev: 5.8.0
+  hooks:
+  - id: isort
+    additional_dependencies:
+      - toml
+
+- repo: https://github.com/psf/black
+  rev: 21.7b0
+  hooks:
+  - id: black
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.910
+  hooks:
+  - id: mypy
+    additional_dependencies:
+      - types-requests
diff --git a/LICENSE b/LICENSE
@@ -672,4 +672,3 @@ may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
 <https://www.gnu.org/licenses/why-not-lgpl.html>.
-
diff --git a/README.md b/README.md
@@ -46,7 +46,35 @@ optional arguments:
                         Google Cloud Storage bucket (default: None)
 ```
 
-## Running tests ##
+## Development Setup
+
+Clone as above, and then install with dev dependencies:
+
+```
+pip install --editable '.[dev]'
+```
+
+Then install the precommit hooks:
+
+```
+pre-commit install
+```
+
+### Running lint and mypy ###
+Linters and mypy checks should run automatically when you go to commit. To run
+them on-demand, you can use:
+
+```
+pre-commit run
+```
+
+That will only run on the files that you changed; to run on all files, use
+
+```
+pre-commit run --all-files
+```
+
+### Running tests ##
 
 The only test is an integration test against the Interim Data Facility on Google
 Cloud.
@@ -59,7 +87,7 @@ environment variable, and run the tests:
 
 ```
 % export ALERTDB_TEST_GCP_PROJECT=alert-stream
-% pytest .
+% pytest
 ============================= test session starts ==============================
 platform linux -- Python 3.8.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
 rootdir: /home/swnelson/code/rubin/alert_database_server

diff --git a/alertdb/bin/__init__.py b/alertdb/bin/__init__.py
diff --git a/alertdb/bin/alertdb.py b/alertdb/bin/alertdb.py
@@ -1,37 +1,52 @@
-import uvicorn
 import argparse
 
+import uvicorn
+
 from alertdb.server import create_server
 from alertdb.storage import FileBackend, GoogleObjectStorageBackend
 
 
 def main():
     parser = argparse.ArgumentParser(
-        "alertdb", formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        description="Run an alert database HTTP server."
+        "alertdb",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Run an alert database HTTP server.",
     )
     parser.add_argument(
-        "--listen-host", type=str, default="127.0.0.1",
+        "--listen-host",
+        type=str,
+        default="127.0.0.1",
         help="host address to listen on for requests",
     )
     parser.add_argument(
-        "--listen-port", type=int, default=5000,
+        "--listen-port",
+        type=int,
+        default=5000,
         help="host port to listen on for requests",
     )
     parser.add_argument(
-        "--backend", type=str, choices=("local-files", "google-cloud"), default="local-files",
+        "--backend",
+        type=str,
+        choices=("local-files", "google-cloud"),
+        default="local-files",
         help="backend to use to source alerts",
     )
     parser.add_argument(
-        "--local-file-root", type=str, default=None,
+        "--local-file-root",
+        type=str,
+        default=None,
         help="when using the local-files backend, the root directory where alerts should be found",
     )
     parser.add_argument(
-        "--gcp-project", type=str, default=None,
+        "--gcp-project",
+        type=str,
+        default=None,
         help="when using the google-cloud backend, the name of the GCP project",
     )
     parser.add_argument(
-        "--gcp-bucket", type=str, default=None,
+        "--gcp-bucket",
+        type=str,
+        default=None,
         help="when using the google-cloud backend, the name of the Google Cloud Storage bucket",
     )
     args = parser.parse_args()
@@ -48,8 +63,11 @@ def main():
             parser.error("--backend=google-cloud requires --gcp-bucket be set")
         backend = GoogleObjectStorageBackend(args.gcp_project, args.gcp_bucket)
     else:
-        # Shouldn't be possible if argparse is using the choices parameter as expected...
-        raise AssertionError("only valid --backend choices are local-files and google-cloud")
+        # Shouldn't be possible if argparse is using the choices parameter as
+        # expected...
+        raise AssertionError(
+            "only valid --backend choices are local-files and google-cloud"
+        )
 
     server = create_server(backend)
     uvicorn.run(server, host=args.listen_host, port=args.listen_port, log_level="info")
diff --git a/alertdb/server.py b/alertdb/server.py
@@ -4,12 +4,12 @@
 
 from alertdb.storage import AlertDatabaseBackend, NotFoundError
 
-
-# This Content-Type is described as the "preferred content type" for a Confluent
-# Schema Registry here:
+# This Content-Type is described as the "preferred content type" for a
+# Confluent Schema Registry here:
 # https://docs.confluent.io/platform/current/schema-registry/develop/api.html#content-types
-# We're not running a Confluent Schema Registry, and don't conform to the API of
-# one, but we do serve schemas, so this seems possibly appropriate.
+# We're not running a Confluent Schema Registry, and don't conform to the API
+# of one, but we do serve schemas, so this seems possibly appropriate.
+
 SCHEMA_CONTENT_TYPE = "application/vnd.schemaregistry.v1+json"
 
 # There's no consensus on an Avro content type. application/avro+binary is
@@ -33,13 +33,14 @@ def create_server(backend: AlertDatabaseBackend) -> FastAPI:
 
     Returns
     -------
-    FastAPI : A FastAPI application which routes HTTP requests to return schemas.
+    FastAPI : A FastAPI application which routes HTTP requests to return
+    schemas.
     """
 
-    # FastAPI documentation suggests that the application be a global singleton,
-    # with handlers defined as top-level functions, but this doesn't seem to
-    # permit any way of passing in a persistent backend. So, this little
-    # create_server closure exists to allow dependency injection.
+    # FastAPI documentation suggests that the application be a global
+    # singleton, with handlers defined as top-level functions, but this doesn't
+    # seem to permit any way of passing in a persistent backend. So, this
+    # little create_server closure exists to allow dependency injection.
 
     app = FastAPI()
 

diff --git a/alertdb/storage.py b/alertdb/storage.py
@@ -11,7 +11,8 @@
 
 class AlertDatabaseBackend(abc.ABC):
     """
-    An abstract interface representing a storage backend for alerts and schemas.
+    An abstract interface representing a storage backend for alerts and
+    schemas.
     """
 
     @abc.abstractmethod
@@ -22,7 +23,8 @@ def get_alert(self, alert_id: str) -> bytes:
         Confluent Wire Format is described here:
           https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#wire-format
 
-        To summarize, it is a 5-byte header, followed by binary-encoded Avro data.
+        To summarize, it is a 5-byte header, followed by binary-encoded Avro
+        data.
 
         The first header byte is magic byte, with a value of 0.
         The next 4 bytes are a 4-byte schema ID, which is an unsigned 32-bit
@@ -60,7 +62,9 @@ def get_alert(self, alert_id: str) -> bytes:
 
     @abc.abstractmethod
     def get_schema(self, schema_id: str) -> bytes:
-        """Retrieve a single alert schema JSON document in its JSON-serialized form.
+        """
+        Retrieve a single alert schema JSON document in its JSON-serialized
+        form.
 
         Parameters
         ----------
@@ -105,6 +109,7 @@ class FileBackend(AlertDatabaseBackend):
     This is provided as an example, to ensure that it's clear how to implement
     an AlertDatabaseBackend subclass.
     """
+
     def __init__(self, root_dir: str):
         self.root_dir = root_dir
 
@@ -129,6 +134,7 @@ class GoogleObjectStorageBackend(AlertDatabaseBackend):
 
     The path for alert and schema objects follows the scheme in DMTN-183.
     """
+
     def __init__(self, gcp_project: str, bucket_name: str):
         self.object_store_client = gcs.Client(project=gcp_project)
         self.bucket = self.object_store_client.bucket(bucket_name)
@@ -149,4 +155,6 @@ def get_schema(self, schema_id: str) -> bytes:
 
 
 class NotFoundError(Exception):
-    """Error which represents a failure to find an alert or schema in a backend."""
+    """
+    Error which represents a failure to find an alert or schema in a backend.
+    """
diff --git a/setup.cfg b/setup.cfg
@@ -28,3 +28,36 @@ packages =
 [options.entry_points]
 console_scripts =
     alertdb = alertdb.bin.alertdb:main
+
+[options.extras_require]
+dev =
+    pre-commit
+    pytest
+    mypy
+    flake8
+
+[flake8]
+max_line_length = 110
+max_doc_length = 79
+exclude =
+    bin
+    doc
+    **/*/__init__.py
+    **/*/version.py
+    tests/.tests
+ignore =
+    E133
+    E226
+    E228
+
+[mypy]
+exclude = virtualenv*
+
+[mypy-google.*]
+ignore_missing_imports = True
+
+[mypy-uvicorn.*]
+ignore_missing_imports = True
+
+[mypy-setuptools]
+ignore_missing_imports = True
Original file line number	Diff line number	Diff line change
Expand Up		@@ -672,4 +672,3 @@ may consider it more useful to permit linking proprietary applications with
		the library. If this is what you want to do, use the GNU Lesser General
		Public License instead of this License. But first, please read
		<https://www.gnu.org/licenses/why-not-lgpl.html>.