Create ocflib Tools for Announcements

Makefile

@@ -11,11 +11,12 @@ install-hooks: venv
 # after running tests
 .PHONY: test
 test:
-	tox
+	tox -e py37 -- $(TEST_FILE)
 ifneq ($(strip $(COVERALLS_REPO_TOKEN)),)
-	.tox/py37/bin/coveralls
+    .tox/py37/bin/coveralls
 endif
 
+
 .PHONY: release-pypi
 release-pypi: clean autoversion
 	python3 setup.py sdist bdist_wheel

README.md

@@ -57,6 +57,10 @@ The `tests` directory contains automated tests which you're encouraged to add
 to (and not break). The `tests-manual` directory contains scripts intended for
 testing.
 
+To run the a specific test file, run `make tests TEST_FILE=<folder>/<test.py>`.
+
+For example, to run the `announcements_test.py` file in the `lab` folder, run `make tests TEST_FILE=lab/announcements_test.py`
+
 
 #### Using pre-commit
 

ocflib/lab/announcements.py

@@ -0,0 +1,165 @@
+"""Announcements handling"""
+from datetime import datetime
+
+from requests import get
+from yaml import safe_load
+
+# The default branch is main
+ANNOUNCEMENTS_URL = (
+    'https://api.github.com/repos/ocf/announcements/contents/{folder}/{id}'
+)
+# 1 day in seconds
+TIME_TO_LIVE = 60 * 60 * 24
+
+
+class Metadata:
+    def __init__(self, title, date, author, tags, summary):
+        self.title = title
+        self.date = date
+        self.author = author
+        self.tags = tags
+        self.summary = summary
+
+
+class _AnnouncementCache:
+    def __init__(self) -> None:
+        # text_cache is a dict of {id: post content}
+        self.text_cache = {}
+        # id_cache is a list of ids, ordered by latest to oldest
+        self.id_cache = []
+        self.last_updated = datetime.now()
+
+    def clear_cache(self) -> None:
+        """Clear the cache if it's too old"""
+
+        if (datetime.now() - self.last_updated).total_seconds() > TIME_TO_LIVE:
+            self.text_cache.clear()
+            self.id_cache.clear()
+            self.last_updated = datetime.now()
+
+
+_announcement_cache_instance = _AnnouncementCache()
+
+
+def _check_id(id: str) -> bool:
+    """Check if the id is a valid date"""
+
+    try:
+        datetime.strptime(id, '%Y-%m-%d-%M')
+    except ValueError:
+        raise ValueError('Invalid id')
+
+
+def get_all_announcements(folder='announcements') -> [str]:
+    """
+    Get announcements from the announcements repo
+    The result is a list of IDs from latest to oldest
+    """
+
+    posts = get(
+        url=ANNOUNCEMENTS_URL.format(folder=folder, id=''),
+        headers={'Accept': 'application/vnd.github+json'},
+    )
+    posts.raise_for_status()
+
+    res = []
+
+    for post in posts.json():
+        res.append(get_id(post))
+
+    # Reverse the list so that the order is latest to oldest
+    res = res[::-1]
+
+    _announcement_cache_instance.id_cache = res
+
+    return res
+
+
+def get_announcement(id: str, folder='announcements') -> str:
+    """
+    Get one particular announcement from the announcements repo
+    The result is the post content
+    """
+
+    _check_id(id)
+
+    if id in _announcement_cache_instance.text_cache:
+        return _announcement_cache_instance.text_cache[id]
+
+    post = get(
+        url=ANNOUNCEMENTS_URL.format(folder=folder, id=id + '.md'),
+        headers={'Accept': 'application/vnd.github.raw'},
+    )
+    post.raise_for_status()
+
+    _announcement_cache_instance.text_cache[id] = post.text
+
+    return post.text
+
+
+def get_id(post_json: dict) -> str:
+    """Get announcement id based on the json response"""
+
+    # Since the id is the filename, remove the .md extension
+    try:
+        id = post_json['name'][:-3]
+    except KeyError:
+        raise KeyError('Missing id in announcement')
+
+    _check_id(id)
+
+    return id
+
+
+def get_metadata(post_text: str) -> Metadata:
+    """Get the metadata from one announcement"""
+
+    try:
+        meta_dict = safe_load(post_text.split('---')[1])
+
+        data = Metadata(
+            title=meta_dict['title'],
+            date=meta_dict['date'],
+            author=meta_dict['author'],
+            tags=meta_dict['tags'],
+            summary=meta_dict['summary'],
+        )
+    except (IndexError, KeyError) as e:
+        raise ValueError(f'Error parsing metadata: {e}')
+
+    return data
+
+
+def get_last_n_announcements(n: int) -> [dict]:
+    """Get the IDs of last n announcements"""
+
+    assert n > 0, 'n must be positive'
+
+    # check if the cache is too old
+    _announcement_cache_instance.clear_cache()
+
+    if _announcement_cache_instance.id_cache:
+        return _announcement_cache_instance.id_cache[:n]
+
+    return get_all_announcements()[:n]
+
+
+def get_last_n_announcements_text(n: int) -> [str]:
+    """Get the text of last n announcements"""
+
+    assert n > 0, 'n must be positive'
+
+    # check if the cache is too old
+    _announcement_cache_instance.clear_cache()
+
+    result = []
+
+    if _announcement_cache_instance.id_cache:
+        res = _announcement_cache_instance.id_cache[:n]
+    else:
+        res = get_all_announcements()[:n]
+
+    for id in res:
+        result.append(get_announcement(id))
+
+    return result

tests/lab/announcements_test.py

@@ -0,0 +1,119 @@
+import pytest
+from requests.exceptions import HTTPError
+
+from ocflib.lab.announcements import get_all_announcements
+from ocflib.lab.announcements import get_announcement
+from ocflib.lab.announcements import get_metadata
+
+TEST_FOLDER = 'tests'
+TEST_IDS = [
+    '2002-01-01-00',
+    '2002-01-01-01',
+    '2002-01-02-00',
+    '2023-09-01-00',
+    '2023-10-01-00',
+    '2023-11-01-00',
+]
+
+
+# scope = module means that the fixture is only run once per module
+@pytest.fixture(scope='module')
+def get_all() -> [str]:
+    return get_all_announcements(folder=TEST_FOLDER)
+
+
+# scope = module means that the fixture is only run once per module
+@pytest.fixture(scope='module')
+def announcement_data():
+    # Fetch data once for all tests in this module
+    return {id: get_announcement(id, folder=TEST_FOLDER) for id in TEST_IDS}
+
+
+# Health check
+@pytest.mark.parametrize(
+    'id',
+    TEST_IDS,
+)
+def test_get_announcement_pass(id):
+    assert 'testing' in get_announcement(id, folder=TEST_FOLDER)
+
+
+# Those ids are invalid
+@pytest.mark.parametrize(
+    'id',
+    [
+        '2002-01-00-00212',
+        '2002-01-01-aa',
+        '2002-01-02-21a',
+        '2002-223-02-00',
+        '2002-01-80-00',
+    ],
+)
+def test_get_announcement_bad_id(id):
+    with pytest.raises(ValueError):
+        get_announcement(id, folder=TEST_FOLDER)
+
+
+# Those announcements don't exist in the test folder
+@pytest.mark.parametrize(
+    'id',
+    [
+        '2002-01-01-10',
+        '2002-01-01-12',
+        '2002-01-02-30',
+    ],
+)
+def test_get_announcement_fail(id):
+    with pytest.raises(HTTPError):
+        get_announcement(id, folder=TEST_FOLDER)
+
+
+# Those ids are valid
+@pytest.mark.parametrize('id', TEST_IDS)
+def test_get_id_pass(id, get_all):
+    found = False
+    for post in get_all:
+        if id == post:
+            found = True
+            break
+    assert found, f'ID {id} not found in announcements'
+
+
+# Those ids don't exist in the test folder
+@pytest.mark.parametrize(
+    'id',
+    [
+        '2002-01-01-10',
+        '2002-01-01-12',
+        '2002-01-02-30',
+    ],
+)
+def test_get_id_fail(id, get_all):
+    for post in get_all:
+        assert id != post, f'Unexpected ID {id} found in announcements'
+
+
+@pytest.mark.parametrize('id', TEST_IDS)
+def test_get_metadata_pass(id, announcement_data):
+    content = announcement_data[id]
+    assert 'Victor' == get_metadata(content).author, 'author not found in metadata'
+
+
+def test_get_metadata_missing_metadata():
+    content = """
+    ---
+    title: test
+    date: 2020-01-01
+    ---
+    """
+    with pytest.raises(ValueError):
+        get_metadata(content)
+
+
+def test_get_metadata_bad_format():
+    content = """
+    title: test
+    date: 2020-01-01
+    """
+    with pytest.raises(ValueError):
+        get_metadata(content)

tox.ini

@@ -9,7 +9,7 @@ venv_update =
         install= -r {toxinidir}/requirements-dev.txt -e {toxinidir}
 commands =
     {[testenv]venv_update}
-    py.test --cov --cov-report=term-missing -n 1 -v tests -vv
+    py.test --cov --cov-report=term-missing -n 1 -v tests/{posargs} -vv
     pre-commit run --all-files
 
 [flake8]