From bbe6aa09aa72699c5f80213a42f5294795252608 Mon Sep 17 00:00:00 2001 From: Torben <59419684+entorb@users.noreply.github.com> Date: Wed, 20 Mar 2024 06:18:21 +0100 Subject: [PATCH 1/3] restructure to src/ --- .github/workflows/test-python-app.yml | 2 +- .gitignore | 4 +-- README.md | 8 ++--- cspell.config.yaml | 2 ++ src/__init__.py | 1 + 1auth.py => src/auth.py | 0 helper.py => src/helper.py | 31 +++++++++++-------- .../rememberthemilk.ini.example | 0 2tasks_completed.py => src/tasks_completed.py | 4 ++- 3tasks_overdue.py => src/tasks_overdue.py | 0 tests/helper_test.py | 13 +++++--- 11 files changed, 39 insertions(+), 26 deletions(-) create mode 100644 src/__init__.py rename 1auth.py => src/auth.py (100%) rename helper.py => src/helper.py (95%) rename rememberthemilk.ini.example => src/rememberthemilk.ini.example (100%) rename 2tasks_completed.py => src/tasks_completed.py (90%) rename 3tasks_overdue.py => src/tasks_overdue.py (100%) diff --git a/.github/workflows/test-python-app.yml b/.github/workflows/test-python-app.yml index eeea98b..ad99881 100644 --- a/.github/workflows/test-python-app.yml +++ b/.github/workflows/test-python-app.yml @@ -43,5 +43,5 @@ jobs: - name: Test with pytest run: | pip install pytest - cp rememberthemilk.ini.example rememberthemilk.ini + cp src/rememberthemilk.ini.example src/rememberthemilk.ini pytest diff --git a/.gitignore b/.gitignore index 53d7686..58eef7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .vscode/ __pycache__/ cache/ +output/ -*.ini rememberthemilk.ini -out-* .coverage +.DS_Store diff --git a/README.md b/README.md index 96ba553..b57c04a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ register for using the Remember The Milk API here: 3600: # noqa: PLR2004 file_path.unlink() config = ConfigParser() -config.read("rememberthemilk.ini") +config.read(Path(__file__).parent / "rememberthemilk.ini") API_KEY = config.get("settings", "api_key") SHARED_SECRET = config.get("settings", "shared_secret") TOKEN = config.get("settings", "token") @@ -135,7 +138,7 @@ def perform_rest_call(url: str) -> str: # pragma: no cover Return the response text. """ # rate limit: 1 request per second - for file_path in Path("cache/").glob("*.json"): + for file_path in cache_dir.glob("*.json"): if int(time.time()) == int(file_path.stat().st_mtime): print("sleeping for 1s to prevent rate limit") time.sleep(1) @@ -228,7 +231,7 @@ def get_lists_dict() -> dict[int, str]: def get_lists() -> list[dict[str, str]]: """Fetch lists from RTM or cache if recent.""" - cache_file = Path("cache/lists.json") + cache_file = cache_dir / "lists.json" if check_cache_file_available_and_recent(file_path=cache_file, max_age=3600): lists = json_read(cache_file) else: # pragma: no cover @@ -262,7 +265,7 @@ def get_tasks(my_filter: str) -> list[dict[str, str]]: # replace whitespaces by space my_filter = re.sub(r"\s+", " ", my_filter, flags=re.DOTALL) h = gen_md5_string(my_filter) - cache_file = Path(f"cache/tasks-{h}.json") + cache_file = cache_dir / f"tasks-{h}.json" if check_cache_file_available_and_recent(file_path=cache_file, max_age=3 * 3600): print(f"Using cache file: {cache_file}") tasks = json_read(cache_file) @@ -412,9 +415,9 @@ def task_add_fields( task["completed_week"] = None # type: ignore # add url - task[ - "url" - ] = f'https://www.rememberthemilk.com/app/#list/{task["list_id"]}/{task["task_id"]}' # type: ignore + task["url"] = ( + f'https://www.rememberthemilk.com/app/#list/{task["list_id"]}/{task["task_id"]}' # type: ignore + ) return task @@ -442,13 +445,15 @@ def df_name_url_to_html(df: pd.DataFrame) -> pd.DataFrame: return df -def df_to_html(df: pd.DataFrame, filename: str) -> None: # pragma: no cover +def df_to_html( + df: pd.DataFrame, filename: str, *, index: bool = False +) -> None: # pragma: no cover """Export DF to html.""" print(f"Exporting to {filename}") df.to_html( - filename, - index=False, + output_dir / filename, + index=index, render_links=False, escape=False, justify="center", diff --git a/rememberthemilk.ini.example b/src/rememberthemilk.ini.example similarity index 100% rename from rememberthemilk.ini.example rename to src/rememberthemilk.ini.example diff --git a/2tasks_completed.py b/src/tasks_completed.py similarity index 90% rename from 2tasks_completed.py rename to src/tasks_completed.py index 2dc8c34..116a088 100644 --- a/2tasks_completed.py +++ b/src/tasks_completed.py @@ -48,7 +48,7 @@ df[cols], "out-completed.html", ) - # df.to_excel("out-done-year.xlsx", index=False) + # df.to_excel(output_dir/"out-done-year.xlsx", index=False) df2 = df.groupby(["completed_week", "list"]).agg( count=("completed_week", "count"), @@ -57,3 +57,5 @@ sum_estimate=("estimate", "sum"), ) print(df2) + + df_to_html(df2, "out-completed-week.html", index=True) diff --git a/3tasks_overdue.py b/src/tasks_overdue.py similarity index 100% rename from 3tasks_overdue.py rename to src/tasks_overdue.py diff --git a/tests/helper_test.py b/tests/helper_test.py index 60a8b2d..b912250 100644 --- a/tests/helper_test.py +++ b/tests/helper_test.py @@ -12,7 +12,8 @@ import pytest # Add parent directory to the Python path, so we can run this file directly -sys.path.insert(0, Path(__file__).parent.parent.as_posix()) +sys.path.insert(0, (Path(__file__).parent.parent / "src").as_posix()) + from helper import ( convert_task_fields, @@ -30,6 +31,8 @@ tasks_to_df, ) +cache_dir = Path(__file__).parent.parent / "src" / "cache" + @pytest.fixture(autouse=True) def _setup_tests(): # noqa: ANN202 @@ -43,7 +46,7 @@ def _setup_tests(): # noqa: ANN202 def cache_prepare_lists() -> None: cache_source = Path("tests/test_data/lists.json") - cache_target = Path("cache/lists.json") + cache_target = cache_dir / "lists.json" shutil.copyfile(cache_source, cache_target) @@ -52,16 +55,16 @@ def cache_prepare_tasks() -> None: # l = get_tasks(my_filter=my_filter) h = gen_md5_string(my_filter) cache_source = Path(f"tests/test_data/tasks-{h}.json") - cache_target = Path(f"cache/tasks-{h}.json") + cache_target = cache_dir / f"tasks-{h}.json" shutil.copyfile(cache_source, cache_target) def cache_cleanup_test_data() -> None: - Path("cache/lists.json").unlink(missing_ok=True) + (cache_dir / "lists.json").unlink(missing_ok=True) my_filter = "list:unit-tests" h = gen_md5_string(my_filter) - cache_target = Path(f"cache/tasks-{h}.json") + cache_target = cache_dir / f"tasks-{h}.json" cache_target.unlink(missing_ok=True) From e861aaee0a565e559e0833cdfc6813ccabd5fca0 Mon Sep 17 00:00:00 2001 From: Torben <59419684+entorb@users.noreply.github.com> Date: Wed, 20 Mar 2024 06:18:50 +0100 Subject: [PATCH 2/3] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f5b1df..ad2baf2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.2.1" + rev: "v0.3.3" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -97,7 +97,7 @@ repos: # # code spell check via cspell - repo: https://github.com/streetsidesoftware/cspell-cli - rev: v8.3.0 + rev: v8.6.0 hooks: - id: cspell From 8c945938cab119afc37d1240e7b56eb8de5d7dee Mon Sep 17 00:00:00 2001 From: Torben <59419684+entorb@users.noreply.github.com> Date: Wed, 20 Mar 2024 06:35:28 +0100 Subject: [PATCH 3/3] proper html export --- src/helper.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/helper.py b/src/helper.py index 3576ea1..6939c8f 100644 --- a/src/helper.py +++ b/src/helper.py @@ -439,6 +439,9 @@ def df_name_url_to_html(df: pd.DataFrame) -> pd.DataFrame: name is html encoded first """ # html encoding of column "name" + df["name"] = df["name"].str.replace("&", "&") + df["name"] = df["name"].str.replace(">", ">") + df["name"] = df["name"].str.replace("<", "<") df["name"] = df["name"].str.encode("ascii", "xmlcharrefreplace").str.decode("utf-8") # add url link to name df["name"] = '' + df["name"] + "" @@ -451,10 +454,14 @@ def df_to_html( """Export DF to html.""" print(f"Exporting to {filename}") - df.to_html( - output_dir / filename, + html = df.to_html( index=index, render_links=False, escape=False, justify="center", ) + + html = html.replace("", "") + html = "\n" + html + + (output_dir / filename).write_text(html)