Restructure project (#18)

Dockerfile

@@ -10,19 +10,26 @@ ENV PYTHONUNBUFFERED=1
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # Install pip requirements
-COPY requirements.txt .
+COPY . .
 
-RUN python -m pip install --user -r requirements.txt --no-warn-script-location
+RUN python3.11 -m venv /app/venv
+ENV PATH=/app/venv/bin:$PATH
+RUN /app/venv/bin/pip3 install build
+RUN /app/venv/bin/python -m build --wheel
+RUN /app/venv/bin/pip3 install dist/*.whl
 
 FROM python:3.11-slim-bullseye as build-image
 WORKDIR /app
-COPY --from=compile-image /root/.local /app/.local
-
-COPY . /app
+COPY --from=compile-image /app/venv /app/venv
 
 # Switching to a non-root user, please refer to https://aka.ms/vscode-docker-python-user-rights
 RUN useradd -d /app appuser && chown -R appuser /app
 USER appuser
 
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV VIRTUAL_ENV=/app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
 # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
-CMD ["python", "retriever.py"]
+CMD ["retriever"]

README.md

@@ -17,13 +17,9 @@ This script retrieves channel information from a cable modem and stores time ser
 
 ## Installation
 
-Install the following packages with pip
+Build and install with pip
 
-`pip3 install requests beautifulsoup4 influxdb schedule`
-
-Optionally, for log collection:
-
-`pip3 install python-logging-loki`
+`pip3 install -e .`
 
 ## Configuration
 
@@ -40,13 +36,13 @@ Valid strings for `ModemType`:
 
 Test the script by executing it manually and verify it completes without error messages.
 
-`python3 retriever.py`
+`retriever`
 
 ## Set up recurring execution
 
 Create a cron job (executes every 2 minutes):
 
-`*/2 * * * * /usr/bin/python3 /opt/cm-status/retriever.py 2>&1 > /dev/null`
+`*/2 * * * * /usr/local/bin/retriever 2>&1 > /dev/null`
 
 ## Docker
 

docsismodem/__init__.py

@@ -0,0 +1,3 @@
+from . import retriever
+
+__version__ = '0.0.28'

collectionJob.py → docsismodem/collectionJob.py

@@ -2,8 +2,7 @@
 import functools
 from logging import Logger
 from sched import scheduler
-
-from modems.observable_modem import ObservableModem
+from docsismodem.modems.observable_modem import ObservableModem
 
 class collectionJob():
 

probe.py → docsismodem/probe.py

@@ -1,6 +1,6 @@
 import datetime
-from collectionJob import collectionJob
 from flask_healthz import HealthError
+from docsismodem.collectionJob import collectionJob
 
 
 class Probe():

docsismodem/retriever.py

@@ -0,0 +1,121 @@
+import configparser
+import threading
+from influxdb_client import InfluxDBClient
+import logging
+import logging_loki
+from requests.packages import urllib3
+import schedule
+import time
+from docsismodem.modems.netgear_cm2000 import NetgearCM2000
+from docsismodem.modems.technicolor_xb7 import TechnicolorXB7
+from docsismodem.modems.motorola_mb8600 import MotorolaMB8600
+from docsismodem.modems.touchstone_tg3492_upc_ch import TouchstoneTG3492UPCCH
+
+from flask import Flask
+from flask_healthz import healthz
+
+from docsismodem.probe import Probe
+from docsismodem.collectionJob import collectionJob
+
+def main():
+    consoleLogger.info("Connecting to InfluxDB")
+
+    handler = logging_loki.LokiHandler(
+        url=lokiUrl + "/loki/api/v1/push", 
+        tags={'application': "cablemodem-status-logs"},
+        auth=(lokiUsername, lokiPassword),
+        version="1",
+    )
+
+    filter = CustomTimestampFilter()
+
+    logger = logging.getLogger("modem")
+    logger.addHandler(handler)
+    logger.addFilter(filter)
+
+    influxUrl =  ("https" if influxUseTLS else "http") + "://" + influxHost + ":" + influxPort
+
+    dbClient = InfluxDBClient(url=influxUrl, org=influxOrg, token=influxToken, ssl=influxUseTLS)
+
+    modems = {
+        "MotorolaMB8600": MotorolaMB8600(config, dbClient, consoleLogger),
+        "NetgearCM2000": NetgearCM2000(config, dbClient, consoleLogger),
+        "TechnicolorXB7": TechnicolorXB7(config, dbClient, consoleLogger),
+        "TouchstoneTG3492UPCCH": TouchstoneTG3492UPCCH(config, dbClient, consoleLogger)
+    }
+
+    jobRunner = collectionJob(modems[config['General']['ModemType']], config['Modem'].getboolean('CollectLogs'), consoleLogger)
+
+    # Because the modem uses a self-signed certificate and this is expected, disabling the warning to reduce noise.
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    def runDaemon():
+        jobRunner.collectionJob()
+        consoleLogger.info("Running as daemon")
+        schedule.every(runEveryMinutes).minutes.do(jobRunner.collectionJob)
+
+        while 1:
+            schedule.run_pending()
+            time.sleep(1)
+
+    if runAsDaemon:
+        runnerThread = threading.Thread(target=runDaemon, daemon=True)
+        runnerThread.start()
+        create_flask_app(jobRunner)
+    else:
+        consoleLogger.info("One-time execution")
+        jobRunner.collectionJob()
+
+
+
+def create_flask_app(runner):
+
+    if enableHealthProbe is True:
+        app = Flask(__name__)
+        app.register_blueprint(healthz, url_prefix="/healthz")
+
+        probe = Probe(runner, runEveryMinutes)
+
+        app.config.update(
+            HEALTHZ = {
+                "live": probe.liveness,
+                "ready": lambda: None,
+            }
+        )
+
+        app.run(host='0.0.0.0', port=80, debug=False, use_reloader=False) 
+
+class CustomTimestampFilter(logging.Filter):
+    def filter(self, record):
+        if hasattr(record, 'timestamp'):
+            record.created = record.timestamp
+        return True
+
+# Init logger
+logging.basicConfig(level=logging.INFO)
+consoleLogger = logging.getLogger("app")
+
+# Read configuration
+consoleLogger.info("Reading configuration")
+
+config = configparser.ConfigParser()
+config.read('data/configuration.ini')
+
+influxOrg = config['Database']['Org']
+influxHost = config['Database']['Host']
+influxPort = config['Database']['Port']
+influxToken = config['Database']['Token']
+influxUseTLS = bool(config['Database']['UseTls'])
+
+lokiUrl = config['Loki']['Url']
+lokiUsername = config['Loki']['Username']
+lokiPassword = config['Loki']['Password']
+
+runAsDaemon = config['General'].getboolean('Daemon')
+enableHealthProbe = config['General'].getboolean('EnableK8sProbe')
+runEveryMinutes = int(config['General']['RunEveryMinutes'])
+
+lastUpdated = 0
+
+if __name__ == '__main__':
+    main()

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"