Added some examples of using the API, tweaked the API a bit, updated …

…versions and changelogs

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2022 Omnivector Solutions
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

examples/.gitignore

@@ -0,0 +1 @@
+*.token

examples/application-example/templates/dummy-script.py.j2

@@ -4,6 +4,6 @@
 #SBATCH -t 60
 
 print("I am a very, very dumb job script")
-print("foo='{{foo}}'")
-print("bar='{{bar}}'")
-print("baz='{{baz}}'")
+print("foo={{ data['foo'] }}")
+print("bar={{ data['bar'] }}")
+print("baz={{ data['baz'] }}")

examples/cli-login.py

@@ -0,0 +1,160 @@
+"""
+Demonstrate basic method of logging in via web portal and using the token to make a request to the Jobbegate API.
+
+To run this example::
+
+- Create a virtual environment to work in:
+  $ python -m venv env
+
+- Activate the virtual environment
+  $ source env/bin/activate
+
+- Install our two dependencies
+  $ pip install httpx python-jose
+
+- Set the environment variables for the auth client (request these from Omnivector)
+  $ export AUTH_CLIENT_ID=<insert-client-id-here>
+  $ export AUTH_CLIENT_SECRET=<insert-client-secret-here>
+
+- Run the demo
+  $ python cli-login.py
+
+Note that after logging in the first time, running this demo again will use the token saved in the same directory as
+"demo.token" until that token expires.
+"""
+
+import os
+import pathlib
+import time
+
+import httpx
+from jose import jwt
+from jose.exceptions import ExpiredSignatureError
+
+
+def is_token_expired(token):
+    """
+    Check a token to see if it is expired.
+
+    Do not check any other parts of a token including its signature. The API will validate the token for us, we only
+    want to see if we should first fetch a new token.
+    """
+    try:
+        jwt.decode(token, None, options=dict(verify_signature=False, verify_aud=False, verify_exp=True))
+        return False
+    except ExpiredSignatureError:
+        return True
+
+
+def login(domain, audience, client_id):
+    """
+    Get a new access token and refresh_token by logging into the web auth portal.
+
+    Returns a 2-tuple of (access-token, refresh-token)
+    """
+    response = httpx.post(
+        f"https://{domain}/oauth/device/code",
+        data=dict(
+            client_id=client_id,
+            audience=audience,
+            scope="offline_access",  # To get refresh token
+        ),
+    )
+    device_code_data = response.json()
+    verification_url = device_code_data["verification_uri_complete"]
+    wait_interval = device_code_data["interval"]
+    device_code = device_code_data["device_code"]
+
+    print(f"Login Here: {verification_url}")
+    while True:
+        time.sleep(wait_interval)
+        response = httpx.post(
+            f"https://{domain}/oauth/token",
+            data=dict(
+                grant_type="urn:ietf:params:oauth:grant-type:device_code",
+                device_code=device_code,
+                client_id=client_id,
+            ),
+        )
+        try:
+            response.raise_for_status()  # Will throw an exception if response isn't in 200s
+            response_data = response.json()
+            return (
+                response_data["access_token"],
+                response_data["refresh_token"],
+            )
+        except Exception:
+            pass
+
+
+def refresh(domain, client_id, client_secret, refresh_token):
+    """
+    Refresh an access token using a refresh token.
+    """
+    response = httpx.post(
+        f"https://{domain}/oauth/token",
+        data=dict(
+            grant_type="refresh_token",
+            client_id=client_id,
+            client_secret=client_secret,
+            refresh_token=refresh_token,
+        ),
+    )
+    refreshed_data = response.json()
+    access_token = refreshed_data["access_token"]
+    return access_token
+
+
+def get_saved_token(token_file, check_expiration=False):
+    """
+    Get a token that has been saved in a file and optionally check its expriation.
+    """
+    if not token_file.exists():
+        return None
+
+    token = token_file.read_text()
+    if check_expiration and is_token_expired(token):
+        return None
+
+    return token
+
+
+def acquire_token(
+    domain="login.omnivector.solutions",
+    client_id=os.environ["AUTH_CLIENT_ID"],
+    client_secret=os.environ["AUTH_CLIENT_SECRET"],
+    audience="https://armada.omnivector.solutions",
+    access_token_file=pathlib.Path("./access.token"),
+    refresh_token_file=pathlib.Path("./refresh.token"),
+):
+    """
+    Acquire an access token by attempting, in order::
+
+    * Retrieve from where it is cached in a local file
+    * Refresh it with a refresh token
+    * Retreive it from the auth API using client credentials
+    """
+    access_token = get_saved_token(access_token_file, check_expiration=True)
+    if access_token is None:
+        refresh_token = get_saved_token(refresh_token_file)
+
+        if refresh_token is None:
+            (access_token, refresh_token) = login(domain, audience, client_id)
+            refresh_token_file.write_text(refresh_token)
+        else:
+            access_token = refresh(domain, client_id, client_secret, refresh_token)
+        access_token_file.write_text(access_token)
+    return access_token
+
+if __name__ == '__main__':
+    token = acquire_token()
+
+    response = httpx.get(
+        "https://armada-k8s.staging.omnivector.solutions/jobbergate/job-submissions",
+        headers=dict(Authorization=f"Bearer {token}"),
+    )
+    try:
+        response.raise_for_status()
+        print("Successfully logged in!")
+    except Exception as err:
+        print(f"Login failed: {err}")

examples/create-submit-job.py

@@ -0,0 +1,123 @@
+"""
+Demonstrate creation of a job-script from an application and subsequent submission through API calls.
+
+To run this example::
+
+- Create a virtual environment to work in:
+  $ python -m venv env
+
+- Activate the virtual environment
+  $ source env/bin/activate
+
+- Install our one dependency
+  $ pip install httpx
+
+- Run the demo
+  $ python create-submit-job.py
+
+
+Note: Before running this demo, you will need::
+
+- An Auth token in the same directory named "access.token". You may use the ``cli-login.py`` for this.
+- An application created from the "application-example" directory. It may already exist.
+"""
+
+import pathlib
+
+import httpx
+
+
+def get_application_id(
+    base_api_url="https://armada-k8s.staging.omnivector.solutions/jobbergate",
+    access_token_file=pathlib.Path("./access.token"),
+    application_identifier="application-example",
+):
+    """
+    Get an application id for the application with the identifier "application-example".
+    """
+    token = access_token_file.read_text()
+    response = httpx.get(
+        f"{base_api_url}/applications",
+        headers=dict(Authorization=f"Bearer {token}"),
+        params=dict(search=application_identifier),
+    )
+    response.raise_for_status()  # Raise an exception if status is not in 200s
+    for result in response.json()["results"]:
+        if result["application_identifier"] == application_identifier:
+            application_id = result["id"]
+            print(f"Found application id {application_id}")
+            return application_id
+    raise RuntimeError(f"Couldn't find an application with identifier='{application_identifier}'")
+
+
+def create_job_script(
+    application_id,
+    base_api_url="https://armada-k8s.staging.omnivector.solutions/jobbergate",
+    access_token_file=pathlib.Path("./access.token"),
+    job_script_name="demo-job-script",
+):
+    """
+    Create a job-script from the "application-example".
+
+    Application config params and sbatch params are hard-coded but may be easily modified as desired.
+    """
+    token = access_token_file.read_text()
+    response = httpx.post(
+        f"{base_api_url}/job-scripts",
+        headers=dict(Authorization=f"Bearer {token}"),
+        json=dict(
+            application_id=application_id,
+            job_script_name=job_script_name,
+            job_script_description="A demonstration of job-script creation through the API",
+            sbatch_params=[
+                "--job-name Demooooooo",
+                "--time 30",
+            ],
+            param_dict=dict(
+                application_config=dict(
+                    foo="foofoofoo",
+                    bar="barbarbar",
+                    baz="bazbazbaz",
+                ),
+            ),
+        ),
+    )
+    response.raise_for_status()  # Raise an exception if status is not in 200s
+    result = response.json()
+    job_script_id = result["id"]
+    print(f"Created job-script {job_script_id}")
+    return job_script_id
+
+
+def create_job_submission(
+    job_script_id,
+    base_api_url="https://armada-k8s.staging.omnivector.solutions/jobbergate",
+    access_token_file=pathlib.Path("./access.token"),
+    job_submission_name="demo-job-sub",
+):
+    """
+    Create a job-submission from a provided job_script_id.
+
+    The job submission will be remotely submitted by the cluster-agent.
+    """
+    token = access_token_file.read_text()
+    response = httpx.post(
+        f"{base_api_url}/job-submissions",
+        headers=dict(Authorization=f"Bearer {token}"),
+        json=dict(
+            job_script_id=job_script_id,
+            job_submission_name=job_submission_name,
+            job_script_description="A demonstration of job-submission creation through the API",
+        ),
+    )
+    response.raise_for_status()  # Raise an exception if status is not in 200s
+    result = response.json()
+    job_submission_id = result["id"]
+    print(f"Created job-submission {job_submission_id}")
+    return job_submission_id
+
+
+if __name__ == '__main__':
+    application_id = get_application_id()
+    job_script_id = create_job_script(application_id)
+    job_submission_id = create_job_submission(job_script_id)

jobbergate-api/CHANGELOG.rst

@@ -7,6 +7,11 @@ This file keeps track of all notable changes to jobbergate-api
 Unreleased
 ----------
 
+3.0.4 -- 2022-04-11
+-------------------
+- Made supplying param_dict optional in job-scripts create (will use app defaults)
+- Included some example scripts for working with API directly.
+
 3.0.3 -- 2022-04-08
 -------------------
 - Restored jobberappslib in jobbergate CLI

jobbergate-api/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2022 Omnivector Solutions
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jobbergate-api/jobbergate_api/apps/job_scripts/routers.py

@@ -12,6 +12,7 @@
 from fastapi import APIRouter, Depends, HTTPException, Query, status
 from jinja2 import Template
 from loguru import logger
+from yaml import safe_load
 
 from jobbergate_api.apps.applications.models import applications_table
 from jobbergate_api.apps.applications.schemas import ApplicationResponse
@@ -169,8 +170,15 @@ async def job_script_create(
         job_script_owner_email=identity_claims.user_email,
     )
 
+    # Use application_config from the application as a baseline of defaults
+    print("APP CONFIG: ", application.application_config)
+    param_dict = safe_load(application.application_config)
+
+    # User supplied param dict is optional and may override defaults
+    param_dict.update(**job_script.param_dict)
+
     logger.debug("Rendering job_script data as string")
-    job_script_data_as_string = build_job_script_data_as_string(s3_application_tar, job_script.param_dict)
+    job_script_data_as_string = build_job_script_data_as_string(s3_application_tar, param_dict)
 
     sbatch_params = create_dict.pop("sbatch_params", [])
     create_dict["job_script_data_as_string"] = inject_sbatch_params(job_script_data_as_string, sbatch_params)

jobbergate-api/jobbergate_api/tests/apps/applications/test_routers.py

@@ -692,7 +692,9 @@ async def test_get_applications__with_pagination(
     assert count[0][0] == 5
 
     inject_security_header("[email protected]", Permissions.APPLICATIONS_VIEW)
-    response = await client.get("/jobbergate/applications/?start=0&limit=1&all=true")
+    response = await client.get(
+        "/jobbergate/applications/?start=0&limit=1&all=true&sort_field=application_identifier"
+    )
     assert response.status_code == status.HTTP_200_OK
 
     data = response.json()

jobbergate-api/jobbergate_api/tests/apps/conftest.py

@@ -15,7 +15,7 @@ def application_data():
         "application_owner_email": "[email protected]",
         "application_name": "test_name",
         "application_file": "the\nfile",
-        "application_config": "the configuration is here",
+        "application_config": "{}",
     }