Some cleaning and pre-commit hooks

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+extend-ignore =  E131, E722, F401, F403

.github/workflows/codeql-analysis.yml

@@ -48,11 +48,11 @@ jobs:
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
-        
+
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
@@ -61,7 +61,7 @@ jobs:
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
     # - run: |

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+    -   id: check-added-large-files
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.4.0
+  hooks:
+    -   id: mypy
+        additional_dependencies:
+          - types-requests
+- repo: https://github.com/pylint-dev/pylint
+  rev: v3.0.0a6
+  hooks:
+    -   id: pylint
+        additional_dependencies:
+          - flask
+          - Flask-SQLAlchemy
+          - requests
+          - beautifulsoup4
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.0.0
+  hooks:
+    -   id: flake8
+        additional_dependencies:
+          - flake8-import-order

.pylintrc

@@ -1,8 +1,8 @@
 [GENERAL]
-fail-under=9.0
+fail-under=10.0
 
 [MESSAGES CONTROL]
-disable=R0903
+disable=C,R,W0511,W0702,W0718
 
 [TYPECHECK]
 generated-members=flask_sqlalchemy.SQLAlchemy.DateTime,

argostime.example.conf

@@ -6,4 +6,3 @@ user = argostime_user
 password = p@ssw0rd
 server = localhost
 database = argostime
-

argostime/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
     __init__.py
 
@@ -25,28 +24,32 @@
 logging.basicConfig(
     filename="argostime.log",
     level=logging.DEBUG,
-    format="%(asctime)s - %(processName)s - %(levelname)s - %(module)s - %(funcName)s - %(message)s"
+    format="%(asctime)s - %(processName)s - %(levelname)s - %(module)s - "
+           "%(funcName)s - %(message)s"
 )
 
-import os.path
+import configparser  # noqa: I100, I202, E402
+import os.path  # noqa: E402
 
-import configparser
+from flask import Flask  # noqa: E402
 
-from flask import Flask
-from flask_sqlalchemy import SQLAlchemy
+from flask_sqlalchemy import SQLAlchemy  # noqa: E402
 
 db: SQLAlchemy = SQLAlchemy()
 
+
 def get_current_commit() -> str:
     """Return the hexadecimal hash of the current running commit."""
-    git_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.git"))
-    with open(os.path.join(git_path, "HEAD"), "r") as file_head:
+    gp = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.git"))
+    with open(os.path.join(gp, "HEAD"), "r", encoding="utf-8") as file_head:
         hexsha = file_head.read().strip()
         while hexsha.startswith("ref: "):
-            with open(os.path.join(git_path, hexsha[5:])) as file_ref:
+            with open(os.path.join(gp, hexsha[5:]),
+                      "r", encoding="utf-8") as file_ref:
                 hexsha = file_ref.read().strip()
     return hexsha
 
+
 def create_app():
     """Return a flask object for argostime, initialize logger and db."""
     logging.getLogger("matplotlib.font_manager").disabled = True
@@ -58,7 +61,9 @@ def create_app():
     logging.debug("Found sections %s in config", config.sections())
 
     if "mariadb" in config:
-        app.config["SQLALCHEMY_DATABASE_URI"] = "mysql+pymysql://{user}:{password}@{server}/{database}?charset=utf8mb4".format(
+        app.config["SQLALCHEMY_DATABASE_URI"] = \
+            "mysql+pymysql://{user}:{password}@{server}/{database}" \
+            "?charset=utf8mb4".format(
             user=config["mariadb"]["user"],
             password=config["mariadb"]["password"],
             server=config["mariadb"]["server"],
@@ -74,6 +79,6 @@ def create_app():
     db.init_app(app)
 
     with app.app_context():
-        from . import routes
+        from . import routes  # pylint: disable=W0611
         db.create_all()
         return app

argostime/crawler/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
     crawler/__init__.py
 
@@ -22,6 +21,6 @@
     along with Argostimè. If not, see <https://www.gnu.org/licenses/>.
 """
 
-from argostime.crawler.crawl_utils import CrawlResult, enabled_shops
 from argostime.crawler.crawl_url import crawl_url
+from argostime.crawler.crawl_utils import CrawlResult, enabled_shops
 from argostime.crawler.shop import *

argostime/crawler/crawl_url.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python3
 """
     crawler/crawl_url.py
 
-    Crawler function exposed to the rest of the system to get pricing and product
-    information from a given URL.
+    Crawler function exposed to the rest of the system to get pricing and
+    product information from a given URL.
 
     Copyright (c) 2022 Martijn <martijn [at] mrtijn.nl>
     Copyright (c) 2022 Kevin <kevin [at] 2sk.nl>
@@ -27,9 +26,8 @@
 import logging
 import urllib.parse
 
-from argostime.exceptions import WebsiteNotImplementedException
-
 from argostime.crawler.crawl_utils import CrawlResult, enabled_shops
+from argostime.exceptions import WebsiteNotImplementedException
 
 
 def crawl_url(url: str) -> CrawlResult:
@@ -47,8 +45,9 @@ def crawl_url(url: str) -> CrawlResult:
     if hostname not in enabled_shops:
         raise WebsiteNotImplementedException(url)
 
-    # Note: This is a function call! The called function is the corresponding crawler
-    # registered using the "@register_crawler" decorator in the "shop" directory.
+    # Note: This is a function call! The called function is the corresponding
+    # crawler registered using the "@register_crawler" decorator in the "shop"
+    # directory.
     result: CrawlResult = enabled_shops[hostname]["crawler"](url)
     result.check()
 

argostime/crawler/crawl_utils.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
     crawler/crawl_utils.py
 
@@ -36,7 +35,9 @@
 
 
 class CrawlResult:
-    """Data structure for returning the results of a crawler in a uniform way."""
+    """
+    Data structure for returning the results of a crawler in a uniform way.
+    """
 
     url: Optional[str]
     product_name: Optional[str]
@@ -50,15 +51,15 @@ class CrawlResult:
 
     def __init__(
         self,
-        url: Optional[str]=None,
-        product_name: Optional[str]=None,
-        product_description: Optional[str]=None,
-        product_code: Optional[str]=None,
-        normal_price: float=-1.0,
-        discount_price: float=-1.0,
-        on_sale: bool=False,
-        ean: Optional[int]=None,
-        ):
+        url: Optional[str] = None,
+        product_name: Optional[str] = None,
+        product_description: Optional[str] = None,
+        product_code: Optional[str] = None,
+        normal_price: float = -1.0,
+        discount_price: float = -1.0,
+        on_sale: bool = False,
+        ean: Optional[int] = None,
+    ):
         self.url = url
         self.product_name = product_name
         self.product_description = product_description
@@ -72,7 +73,8 @@ def __str__(self) -> str:
         string = f"CrawlResult(product_name={self.product_name},"\
             f"product_description={self.product_description},"\
             f"product_code={self.product_code},price={self.normal_price},"\
-            f"discount={self.discount_price},sale={self.on_sale},ean={self.ean}"
+            f"discount={self.discount_price},sale={self.on_sale}," \
+            f"ean={self.ean}"
 
         return string
 
@@ -100,24 +102,28 @@ def check(self) -> None:
         if self.discount_price < 0 and self.on_sale:
             raise CrawlerException("No discount price given for item on sale!")
         if self.normal_price < 0 and not self.on_sale:
-            raise CrawlerException("No normal price given for item not on sale!")
+            raise CrawlerException(
+                "No normal price given for item not on sale!")
 
 
 CrawlerFunc = Callable[[str], CrawlResult]
-ShopDict = TypedDict("ShopDict", {"name": str, "hostname": str, "crawler": CrawlerFunc})
+ShopDict = TypedDict("ShopDict", {"name": str, "hostname": str,
+                                  "crawler": CrawlerFunc})
 enabled_shops: Dict[str, ShopDict] = {}
 
 
-def register_crawler(name: str, host: str, use_www: bool = True) -> Callable[[CrawlerFunc], None]:
+def register_crawler(name: str, host: str, use_www: bool = True) \
+        -> Callable[[CrawlerFunc], None]:
     """Decorator to register a new crawler function."""
 
     def decorate(func: Callable[[str], CrawlResult]) -> None:
         """
-        This function will be called when you put the "@register_crawler" decorator above
-        a function defined in a file in the "shop" directory! The argument will be the
-        function above which you put the decorator.
+        This function will be called when you put the "@register_crawler"
+        decorator above a function defined in a file in the "shop" directory!
+        The argument will be the function above which you put the decorator.
         """
-        if "argostime" in __config and "disabled_shops" in __config["argostime"]:
+        if "argostime" in __config and \
+                "disabled_shops" in __config["argostime"]:
             if host in __config["argostime"]["disabled_shops"]:
                 logging.debug("Shop %s is disabled", host)
                 return
@@ -137,7 +143,9 @@ def decorate(func: Callable[[str], CrawlResult]) -> None:
 
 
 def parse_promotional_message(message: str, price: float) -> float:
-    """Parse a given promotional message, and returns the calculated effective price.
+    """
+    Parse a given promotional message, and returns the calculated effective
+    price.
 
     For example "1+1 GRATIS" will be parsed to meaning a 50% discount.
     "2+1 GRATIS" will be parsed to mean a 33% discount, and will return 2/3.
@@ -152,7 +160,8 @@ def parse_promotional_message(message: str, price: float) -> float:
 
     message_no_whitespace = message_no_whitespace.lower()
 
-    logging.debug("Promotion yielded sanitized input %s", message_no_whitespace)
+    logging.debug("Promotion yielded sanitized input %s",
+                  message_no_whitespace)
 
     if message_no_whitespace == "1+1gratis":
         return 1/2 * price
@@ -187,8 +196,9 @@ def parse_promotional_message(message: str, price: float) -> float:
                 return float(msg_split[1])
             return float(msg_split[1]) / float(msg_split[0])
         except ArithmeticError as exception:
-            logging.error("Calculation error parsing %s %s", message_no_whitespace, exception)
-        except IndexError as exception:
+            logging.error("Calculation error parsing %s %s",
+                          message_no_whitespace, exception)
+        except IndexError:
             logging.error("IndexError in message %s", message_no_whitespace)
 
     logging.error("Promotion text did not match any known promotion")

argostime/crawler/shop/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """
     crawler/shop/__init__.py
 
@@ -22,10 +21,11 @@
     along with Argostimè. If not, see <https://www.gnu.org/licenses/>.
 """
 
-from os.path import dirname, basename, isfile, join
 import glob
+from os.path import basename, dirname, isfile, join
 
-# Load all modules in the current directory, based on the answer from Anurag Uniyal:
-# https://stackoverflow.com/questions/1057431/how-to-load-all-modules-in-a-folder
+# Load all modules in the current directory, based on the answer from
+# Anurag Uniyal: https://stackoverflow.com/q/1057431
 modules = glob.glob(join(dirname(__file__), "*.py"))
-__all__ = [basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')]
+__all__ = [basename(f)[:-3] for f in modules if isfile(f) and
+           not f.endswith('__init__.py')]