From de8c43e477c54f69bfba8ce55a3dc617fd5cdaec Mon Sep 17 00:00:00 2001 From: Vanessasaurus <814322+vsoch@users.noreply.github.com> Date: Mon, 28 Feb 2022 18:18:31 -0800 Subject: [PATCH] preparing to add additional endpoints and start of tests (#4) * preparing to add additional endpoints and start of tests Signed-off-by: vsoch --- .github/workflows/main.yml | 20 +++++++- CHANGELOG.md | 1 + README.md | 3 +- docs/getting_started/index.rst | 9 ++-- docs/getting_started/user-guide.rst | 48 +++++++++++++++++-- usrse/client/__init__.py | 8 +++- usrse/client/listing.py | 10 ++++ usrse/main/client.py | 11 ++++- usrse/main/endpoints.py | 31 ++++++++++++- usrse/tests/__init__.py | 0 usrse/tests/helpers.sh | 26 +++++++++++ usrse/tests/test_client.py | 29 ++++++++++++ usrse/tests/test_client.sh | 45 ++++++++++++++++++ usrse/tests/test_utils.py | 72 +++++++++++++++++++++++++++++ usrse/version.py | 2 +- 15 files changed, 296 insertions(+), 19 deletions(-) create mode 100644 usrse/client/listing.py create mode 100644 usrse/tests/__init__.py create mode 100644 usrse/tests/helpers.sh create mode 100644 usrse/tests/test_client.py create mode 100755 usrse/tests/test_client.sh create mode 100644 usrse/tests/test_utils.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6689a27..9690fd8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,8 +11,7 @@ jobs: formatting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - + - uses: actions/checkout@v2 - name: Setup black linter run: conda create --quiet --name black pyflakes @@ -37,3 +36,20 @@ jobs: pyflakes usrse/*.py pyflakes usrse/utils/fileio.py pyflakes usrse/utils/terminal.py + + testing: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup black linter + run: conda create --quiet --name testing pytest + + - name: Run Tests + run: | + export PATH="/usr/share/miniconda/bin:$PATH" + source activate testing + pip install -e . + cd usrse/tests + /bin/bash test_client.sh + pytest -xsv test_*.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bb6088..a3f2724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,4 +14,5 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/USRSE/usrse-python/tree/main) (0.0.x) + - Adding support for jobs and member counts endpoint (0.0.11) - Initial creation of project (0.0.1) diff --git a/README.md b/README.md index d0c448c..0d93a33 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,9 @@ See our ⭐️ [Documentation](https://us-rse.org/usrse-python/) to get started! ## Coming Soon / TODO - - full tests for all endpoints - automated deployment to pypi on release - nightly test run to ensure US-RSE static API functioning as expected - - endpoints for jobs and then functions here + - other endpoints and functions here ## Contributors diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index e37af3c..afae756 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -4,12 +4,9 @@ Getting Started =============== -Singularity Registry (HPC) is a tool that makes it easy to install containers as -Lmod modules. You can create your own registry entries (e.g., a specification -to pull a particular container and expose some number of entrypoints) or -the library also provides you with a community set. - -If you have any questions or issues, please `let us know `_ +USRSE-Python provides easy ways to interact with content from the US-RSE website +and community! +If you have any questions or issues, please `let us know `_ .. toctree:: :maxdepth: 2 diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index ef2b563..5cbb9ee 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -28,10 +28,12 @@ Once you have usrse installed, you likely quickly want to get data .. code-block:: console - $ usrse get posts - $ usrse get newsletters - $ usrse get dei - $ usrse get events + $ usrse get posts + $ usrse get newsletters + $ usrse get dei + $ usrse get events + $ usrse get member-counts + $ usrse get jobs Or snazz things up a bit! @@ -41,6 +43,8 @@ Or snazz things up a bit! $ usrse get newsletters --live $ usrse get dei --live $ usrse get events --live + $ usrse get member-counts --live + $ usrse get jobs --live Or output as json or save to json for later. @@ -60,6 +64,36 @@ Commands The following commands are available! +.. _getting_started-commands-list: + +List +---- + +The first thing you might want to do is see what endpoints are available. +You can do that with ``list``: + +.. code-block:: console + + $ usrse list + dei + events + jobs + member-counts + newsletters + posts + + +This is also nice to pipe into a bash loop for doing something: + +.. code-block:: console + + for endpoint in $(usrse list); do + # do something here + echo $endpoint; + done + +Once you know endpoints, then you can ``get`` them, discussed next. + .. _getting_started-commands-get: Get @@ -79,6 +113,8 @@ The most basic functionality is to get content. Here are all the types we can as posts dei newsletters + member-counts + jobs optional arguments: -h, --help show this help message and exit @@ -133,6 +169,8 @@ Here are all the content types you can ask for: $ usrse get newsletters $ usrse get dei $ usrse get events + $ usrse get member-counts + $ usrse get jobs Want to have some fun? Try the live tables! @@ -143,6 +181,8 @@ Want to have some fun? Try the live tables! $ usrse get newsletters --live $ usrse get dei --live $ usrse get events --live + $ usrse get member-counts --live + $ usrse get jobs --live Changing the baseurl diff --git a/usrse/client/__init__.py b/usrse/client/__init__.py index 36aa586..0f280ef 100644 --- a/usrse/client/__init__.py +++ b/usrse/client/__init__.py @@ -5,6 +5,7 @@ __license__ = "MPL 2.0" import usrse +import usrse.main.endpoints as endpoints from usrse.logger import setup_logger import argparse import sys @@ -59,6 +60,9 @@ def get_parser(): # print version and exit subparsers.add_parser("version", description="show software version") + # List endpoints available + subparsers.add_parser("list", description="list endpoints available") + # Local shell with client loaded shell = subparsers.add_parser( "shell", @@ -82,7 +86,7 @@ def get_parser(): ) get.add_argument( "content_type", - help="content type\nevents\nposts\ndei\nnewsletters", + help="content type\n%s" % "\n".join(endpoints.register_names), ) get.add_argument("--json", help="output json", default=False, action="store_true") get.add_argument("--all", help="output json", default=False, action="store_true") @@ -151,6 +155,8 @@ def help(return_code=0): from .get import main elif args.command == "shell": from .shell import main + elif args.command == "list": + from .listing import main # Pass on to the correct parser return_code = 0 diff --git a/usrse/client/listing.py b/usrse/client/listing.py new file mode 100644 index 0000000..ad54f62 --- /dev/null +++ b/usrse/client/listing.py @@ -0,0 +1,10 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022, Vanessa Sochat" +__license__ = "MPL 2.0" + +import usrse.main.endpoints as endpoints + + +def main(args, parser, extra, subparser): + for endpoint in endpoints.register_names: + print(endpoint) diff --git a/usrse/main/client.py b/usrse/main/client.py index 0e31efb..650cf7a 100644 --- a/usrse/main/client.py +++ b/usrse/main/client.py @@ -27,9 +27,12 @@ class Result: """ def __init__(self, data, endpoint): - self.data = data or {} self.endpoint = endpoint + # Does the endpoint want to sort or otherwise order? + data = data or {} + self.data = self.endpoint.order(data) + # Keep track of the max length for each field not truncated self.max_widths = {} self.ensure_complete() @@ -42,7 +45,10 @@ def available_width(self, columns): Calculate available width based on fields we cannot truncate (urls) """ # We will determine column width based on terminal size - width = os.get_terminal_size().columns + try: + width = os.get_terminal_size().columns + except OSError: + width = 120 # Calculate column width column_width = int(width / len(columns)) @@ -270,4 +276,5 @@ def get(self, name): result = requests.get(endpoint.url) if result.status_code != 200: sys.exit("Issue retrieving %s:\n%s" % (endpoint.url, result.txt)) + return Result(result.json(), endpoint) diff --git a/usrse/main/endpoints.py b/usrse/main/endpoints.py index 16717a3..6d3a464 100644 --- a/usrse/main/endpoints.py +++ b/usrse/main/endpoints.py @@ -3,6 +3,7 @@ __license__ = "MPL 2.0" import usrse.defaults as defaults +from datetime import datetime import sys # Registered endpoints (populated on init) @@ -22,6 +23,10 @@ def __init__(self, baseurl): "Misconfigured endpoint %s missing %s attribute" % (self, attr) ) + # If needed, make sure endpoint data is sorted + def order(self, data): + return data + @property def url(self): return self.baseurl + self.path @@ -52,6 +57,28 @@ class Dei(Endpoint): emoji = "sparkles" +class Jobs(Endpoint): + name = "jobs" + path = "/api/jobs.json" + emoji = "briefcase" + + +class MemberCounts(Endpoint): + name = "member-counts" + path = "/api/member-counts.json" + emoji = "1234" + + def order(self, data): + """ + Sort by month and year + """ + return sorted( + data, + key=lambda entry: datetime.strptime(entry["date"], "%B %Y"), + reverse=True, + ) + + class Newsletters(Endpoint): name = "newsletters" path = "/api/newsletters.json" @@ -72,6 +99,8 @@ class Events(Endpoint): skips = ["repeated", "published", "description"] -for endpoint in [Dei, Newsletters, Posts, Events]: +for endpoint in [Dei, Newsletters, Posts, Events, Jobs, MemberCounts]: register_names.append(endpoint.name) register[endpoint.name] = endpoint + +register_names.sort() diff --git a/usrse/tests/__init__.py b/usrse/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/usrse/tests/helpers.sh b/usrse/tests/helpers.sh new file mode 100644 index 0000000..7982add --- /dev/null +++ b/usrse/tests/helpers.sh @@ -0,0 +1,26 @@ +runTest() { + + # The first argument is the code we should get + ERROR="${1:-}" + shift + OUTPUT=${1:-} + shift + echo "$@" + + "$@" > "${OUTPUT}" 2>&1 + RETVAL="$?" + + if [ "$ERROR" = "0" -a "$RETVAL" != "0" ]; then + echo "$@ (retval=$RETVAL) ERROR" + cat ${OUTPUT} + echo "Output in ${OUTPUT}" + exit 1 + elif [ "$ERROR" != "0" -a "$RETVAL" = "0" ]; then + echo "$@ (retval=$RETVAL) ERROR" + echo "Output in ${OUTPUT}" + cat ${OUTPUT} + exit 1 + else + echo "$@ (retval=$RETVAL) OK" + fi +} diff --git a/usrse/tests/test_client.py b/usrse/tests/test_client.py new file mode 100644 index 0000000..0c7e818 --- /dev/null +++ b/usrse/tests/test_client.py @@ -0,0 +1,29 @@ +#!/usr/bin/python + +# Copyright (C) 2022 Vanessa Sochat. + +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest +import os + +import usrse.main.endpoints as endpoints +from usrse.main import Client + +here = os.path.dirname(os.path.abspath(__file__)) +root = os.path.dirname(here) + +tests = [(ep, item) for ep in endpoints.register_names for item in [True, False]] + + +@pytest.mark.parametrize("endpoint,is_live", tests) +def test_install_get(tmp_path, endpoint, is_live): + """Test install and get""" + client = Client(quiet=False) + result = client.get(endpoint) + if is_live: + result.table_live() + else: + result.table() diff --git a/usrse/tests/test_client.sh b/usrse/tests/test_client.sh new file mode 100755 index 0000000..76eb8d4 --- /dev/null +++ b/usrse/tests/test_client.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +echo +echo "************** START: test_client.sh **********************" + +# Create temporary testing directory +echo "Creating temporary directory to work in." +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +shpc_root="$( dirname "${here}" )" + +. $here/helpers.sh + +# Create temporary testing directory +tmpdir=$(mktemp -d) +output=$(mktemp ${tmpdir:-/tmp}/usrse_test.XXXXXX) +printf "Created temporary directory to work in. ${tmpdir}\n" + +# Make sure it's installed +if ! command -v usrse &> /dev/null +then + printf "usrse is not installed\n" + exit 1 +else + printf "usrse is installed\n" +fi + +echo +echo "#### Testing base client " +runTest 0 $output usrse --version + +echo +echo "#### Testing list " +runTest 0 $output usrse list --help +runTest 0 $output usrse list + + +echo +echo "#### Testing get " +runTest 0 $output usrse get --help +for endpoint in $(usrse list); do + runTest 0 $output usrse get $endpoint + runTest 0 $output usrse get $endpoint --live +done + +rm -rf ${tmpdir} diff --git a/usrse/tests/test_utils.py b/usrse/tests/test_utils.py new file mode 100644 index 0000000..5b21d25 --- /dev/null +++ b/usrse/tests/test_utils.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +# Copyright (C) 2022 Vanessa Sochat. + +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import json +import pytest +import usrse.utils as utils + + +def test_write_read_files(tmp_path): + """test_write_read_files will test the functions write_file and read_file""" + print("Testing utils.write_file...") + + tmpfile = str(tmp_path / "written_file.txt") + assert not os.path.exists(tmpfile) + utils.write_file(tmpfile, "hello!") + assert os.path.exists(tmpfile) + + print("Testing utils.read_file...") + + content = utils.read_file(tmpfile) + assert content == "hello!" + + +def test_write_bad_json(tmp_path): + + bad_json = {"Wakkawakkawakka'}": [{True}, "2", 3]} + tmpfile = str(tmp_path / "json_file.txt") + assert not os.path.exists(tmpfile) + with pytest.raises(TypeError): + utils.write_json(bad_json, tmpfile) + + +def test_write_json(tmp_path): + + good_json = {"Wakkawakkawakka": [True, "2", 3]} + tmpfile = str(tmp_path / "good_json_file.txt") + + assert not os.path.exists(tmpfile) + utils.write_json(good_json, tmpfile) + with open(tmpfile, "r") as f: + content = json.loads(f.read()) + assert isinstance(content, dict) + assert "Wakkawakkawakka" in content + content = utils.read_json(tmpfile) + assert "Wakkawakkawakka" in content + + +def test_get_installdir(): + """get install directory should return the base of the install""" + print("Testing utils.get_installdir") + whereami = utils.get_installdir() + print(whereami) + assert whereami.endswith("usrse") + + +def test_run_command(): + print("Testing utils.run_command") + result = utils.run_command(["echo", "hello"]) + assert result["message"] == "hello\n" + assert result["return_code"] == 0 + + +def test_print_json(): + print("Testing utils.print_json") + result = utils.print_json({1: 1}) + assert result == '{\n "1": 1\n}' diff --git a/usrse/version.py b/usrse/version.py index afc254a..3c557e5 100644 --- a/usrse/version.py +++ b/usrse/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.1" +__version__ = "0.0.11" AUTHOR = "Vanessa Sochat" EMAIL = "vsoch@users.noreply.github.com" NAME = "usrse"