From ee3a59d72a654e14fb6a29a539fa085d4203217e Mon Sep 17 00:00:00 2001 From: istrator Date: Thu, 23 Nov 2023 02:44:18 +0100 Subject: [PATCH 01/15] fix typo in readme.md fix broken links in readme.md add several ToDos --- .gitignore | 4 +++- README.md | 6 +++--- entrypoint | 4 ++-- requirements.txt | 17 +++++++++-------- wgkex.yaml.example | 2 +- wgkex/config/config.py | 6 +++--- wgkex/config/config_test.py | 9 ++++++--- wgkex/worker/app.py | 2 +- wgkex/worker/app_test.py | 2 +- wgkex/worker/mqtt.py | 2 +- 10 files changed, 30 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index c2847fb..4ffb7f2 100644 --- a/.gitignore +++ b/.gitignore @@ -132,7 +132,6 @@ dmypy.json bazel-* # docker-compose -.env docker-compose.override.yaml # docker-compose volumes /volumes @@ -141,3 +140,6 @@ docker-compose.override.yaml # config file wgkex.yaml + +# pycharm project metadata +.idea/ diff --git a/README.md b/README.md index 88a466c..47fda7a 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ - [WireGuard Key Exchange](#wireguard-key-exchange) * [Overview](#overview) + [Frontend broker](#frontend-broker) - - [POST /api/v1/wg/key/exchange](#post--api-v1-wg-key-exchange) + - [POST /api/v1/wg/key/exchange](#post-apiv1wgkeyexchange) + [Backend worker](#backend-worker) * [Installation](#installation) * [Configuration](#configuration) - * [Running the broker](#running-the-broker) + * [Running the broker](#running-the-broker-and-worker) * [Client usage](#client-usage) * [Contact](#contact) @@ -143,7 +143,7 @@ import requests key_data = {"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUSitVag9mS7pSXUesNM0ESnvj/wwehkg="} broker_url = "http://127.0.0.1:5000" push_key = requests.get(f'{broker_url}/api/v1/wg/key/exchange', json=key_data) -print(f'Key push was: {push_key.json().get("Message")]}') +print(f'Key push was: {push_key.json().get("Message")}') ``` ### Worker diff --git a/entrypoint b/entrypoint index 6741032..b842b1c 100755 --- a/entrypoint +++ b/entrypoint @@ -2,7 +2,7 @@ set -e : ${WGKEX_DOMAINS:="ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt"} -: ${WGKEX_DOMAIN_PREFIX:="ffmuc_"} +: ${WGKEX_DOMAIN_PREFIX:="ffmuc_"} # ToDo : ${WGKEX_DEBUG:="DEBUG"} : ${MQTT_BROKER_URL:="mqtt"} : ${MQTT_BROKER_PORT:="1883"} @@ -21,7 +21,7 @@ for i in $WGKEX_DOMAINS; do done cat < "Config": @@ -79,7 +79,7 @@ def from_dict(cls, cfg: Dict[str, str]) -> "Config": return cls( domains=cfg["domains"], mqtt=mqtt_cfg, - domain_prefix=cfg["domain_prefix"], + domain_prefix=cfg["domain_prefix"], # probably ToDo ) diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py index 58427c8..15dd480 100644 --- a/wgkex/config/config_test.py +++ b/wgkex/config/config_test.py @@ -3,8 +3,11 @@ import config import yaml -_VALID_CFG = "domain_prefix: ffmuc_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n" -_INVALID_LINT = "domain_prefix: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n" +# ToDo +_VALID_CFG = ("domain_prefix: ffmuc_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " + "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") +_INVALID_LINT = ("domain_prefix: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " + "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") _INVALID_CFG = "asdasdasdasd" @@ -52,7 +55,7 @@ def test_fetch_from_config_success(self): self.assertListEqual(["a", "b"], config.fetch_from_config("domains")) def test_fetch_from_config_no_key_in_config(self): - """Test fetch non existent key from configuration.""" + """Test fetch non-existent key from configuration.""" mock_open = mock.mock_open(read_data=_VALID_CFG) with mock.patch("builtins.open", mock_open): self.assertIsNone(config.fetch_from_config("key_does_not_exist")) diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index 911fd8b..51b18ca 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -35,7 +35,7 @@ def clean_up_worker(domains: List[Text]) -> None: domains: list of domains. """ logger.debug("Cleaning up the following domains: %s", domains) - prefix = config.load_config().get("domain_prefix") + prefix = config.load_config().get("domain_prefix") # ToDo for domain in domains: logger.info("Scheduling cleanup task for %s, ", domain) try: diff --git a/wgkex/worker/app_test.py b/wgkex/worker/app_test.py index 1fbea9d..66dee4d 100644 --- a/wgkex/worker/app_test.py +++ b/wgkex/worker/app_test.py @@ -13,7 +13,7 @@ def setUp(self) -> None: def test_main_success(self, connect_mock, config_mock): """Ensure we can execute main.""" connect_mock.return_value = None - test_prefix = "TEST_PREFIX_" + test_prefix = "TEST_PREFIX_" # probably ToDo config_mock.return_value = dict( domains=[f"{test_prefix}domain.one"], domain_prefix=test_prefix ) diff --git a/wgkex/worker/mqtt.py b/wgkex/worker/mqtt.py index 995d49c..491f472 100644 --- a/wgkex/worker/mqtt.py +++ b/wgkex/worker/mqtt.py @@ -84,7 +84,7 @@ def on_message(client: mqtt.Client, userdata: Any, message: mqtt.MQTTMessage) -> """ # TODO(ruairi): Check bounds and raise exception here. logger.debug("Got message %s from MTQQ", message) - domain_prefix = load_config().get("domain_prefix") + domain_prefix = load_config().get("domain_prefix") # ToDo domain = re.search(r"/.*" + domain_prefix + "(\w+)/", message.topic) if not domain: raise ValueError( From 2357bb943efa5d0db9698d7faf8c63d06f5f2429 Mon Sep 17 00:00:00 2001 From: DB3342 Date: Fri, 24 Nov 2023 12:36:54 +0100 Subject: [PATCH 02/15] enabled config stuff multiprefixable --- entrypoint | 25 +++++++++++++++---------- env.example | 6 ++---- wgkex.yaml.example | 28 +++++++++++++++++----------- wgkex/config/config.py | 15 +++++++++++---- wgkex/config/config_test.py | 11 ++++++++--- 5 files changed, 53 insertions(+), 32 deletions(-) diff --git a/entrypoint b/entrypoint index b842b1c..3650bfc 100755 --- a/entrypoint +++ b/entrypoint @@ -1,15 +1,15 @@ #!/bin/bash set -e -: ${WGKEX_DOMAINS:="ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt"} -: ${WGKEX_DOMAIN_PREFIX:="ffmuc_"} # ToDo -: ${WGKEX_DEBUG:="DEBUG"} -: ${MQTT_BROKER_URL:="mqtt"} -: ${MQTT_BROKER_PORT:="1883"} -: ${MQTT_USERNAME:=""} -: ${MQTT_PASSWORD:=""} -: ${MQTT_KEEPALIVE:="5"} -: ${MQTT_TLS:="False"} +: "${WGKEX_DOMAINS:='ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events'}" +: "${WGKEX_DOMAIN_PREFIXES:='ffmuc_, ffdon_, ffwert_'}" +: "${WGKEX_DEBUG:='DEBUG'}" +: "${MQTT_BROKER_URL:='mqtt'}" +: "${MQTT_BROKER_PORT:='1883'}" +: "${MQTT_USERNAME:=''}" +: "${MQTT_PASSWORD:=''}" +: "${MQTT_KEEPALIVE:='5'}" +: "${MQTT_TLS:='False'}" mk_config() { if [ ! -e /etc/wgkex.yaml ] ; then @@ -21,7 +21,12 @@ for i in $WGKEX_DOMAINS; do done cat < "MQTT": + """seems to generate a mqtt config object from dictionary + + Args: + mqtt_cfg (): + + Returns: + mqtt config object + """ return cls( broker_url=mqtt_cfg["broker_url"], username=mqtt_cfg["username"], @@ -60,12 +68,11 @@ class Config: Attributes: domains: The list of domains to listen for. mqtt: The MQTT configuration. - domain_prefix: The prefix to pre-pend to a given domain. # ToDo - """ + domain_prefixes: The list of prefixes to pre-pend to a given domain.""" domains: List[str] mqtt: MQTT - domain_prefix: str # ToDo + domain_prefixes: List[str] @classmethod def from_dict(cls, cfg: Dict[str, str]) -> "Config": @@ -79,7 +86,7 @@ def from_dict(cls, cfg: Dict[str, str]) -> "Config": return cls( domains=cfg["domains"], mqtt=mqtt_cfg, - domain_prefix=cfg["domain_prefix"], # probably ToDo + domain_prefixes=cfg["domain_prefixes"], ) diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py index 15dd480..997a4b8 100644 --- a/wgkex/config/config_test.py +++ b/wgkex/config/config_test.py @@ -3,10 +3,15 @@ import config import yaml -# ToDo -_VALID_CFG = ("domain_prefix: ffmuc_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " +# env for windows dev +import os + +os.environ["WG_CONFIG_OS_ENV"] = "env.example" + +_VALID_CFG = ( + "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") -_INVALID_LINT = ("domain_prefix: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " +_INVALID_LINT = ("domain_prefixes: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") _INVALID_CFG = "asdasdasdasd" From f5c3d1807d2f17ed76c433ac427c0130c8e4c59b Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 17:11:19 +0100 Subject: [PATCH 03/15] enabled config stuff multiprefixable --- wgkex/config/config_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py index 997a4b8..fd1050e 100644 --- a/wgkex/config/config_test.py +++ b/wgkex/config/config_test.py @@ -1,3 +1,4 @@ +"""Tests for configuration handling class.""" import unittest import mock import config @@ -9,8 +10,8 @@ os.environ["WG_CONFIG_OS_ENV"] = "env.example" _VALID_CFG = ( - "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " - "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") + "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883" + "\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") _INVALID_LINT = ("domain_prefixes: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") _INVALID_CFG = "asdasdasdasd" From 031f3f54572067bb1173b9d8df36c166e94e27c5 Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 19:21:22 +0100 Subject: [PATCH 04/15] make multiprefix-able fix some autoformatting --- .github/buildkitd.toml | 2 +- .github/dependabot.yml | 18 ++++---- .github/workflows/bazel.yml | 2 +- .github/workflows/black.yml | 2 +- .github/workflows/pylint.yml | 12 +++--- README.md | 7 +-- docker-compose.yml | 6 +-- wgkex/broker/app.py | 6 +-- wgkex/broker/templates/index.html | 2 +- wgkex/config/config_test.py | 5 --- wgkex/worker/app.py | 72 +++++++++++++++++++++++++------ wgkex/worker/app_test.py | 8 ++-- wgkex/worker/mqtt.py | 12 ++++-- wgkex/worker/netlink.py | 2 +- 14 files changed, 101 insertions(+), 55 deletions(-) diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml index ff6039f..0af5d20 100644 --- a/.github/buildkitd.toml +++ b/.github/buildkitd.toml @@ -1,2 +1,2 @@ [worker.oci] - max-parallelism = 4 +max-parallelism = 4 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 92bfa4c..feea51e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,18 +5,18 @@ version: 2 updates: - package-ecosystem: pip directory: / - schedule: {interval: monthly} - reviewers: [freifunkMUC/salt-stack] - assignees: [freifunkMUC/salt-stack] + schedule: { interval: monthly } + reviewers: [ freifunkMUC/salt-stack ] + assignees: [ freifunkMUC/salt-stack ] - package-ecosystem: github-actions directory: / - schedule: {interval: monthly} - reviewers: [freifunkMUC/salt-stack] - assignees: [freifunkMUC/salt-stack] + schedule: { interval: monthly } + reviewers: [ freifunkMUC/salt-stack ] + assignees: [ freifunkMUC/salt-stack ] - package-ecosystem: docker directory: / - schedule: {interval: monthly} - reviewers: [freifunkMUC/salt-stack] - assignees: [freifunkMUC/salt-stack] + schedule: { interval: monthly } + reviewers: [ freifunkMUC/salt-stack ] + assignees: [ freifunkMUC/salt-stack ] diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1513680..52a7fdd 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,6 +1,6 @@ name: Bazel tests -on: [push, pull_request] +on: [ push, pull_request ] jobs: bazel-run: diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index e91693a..01bee2b 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,6 +1,6 @@ name: Lint -on: [push, pull_request] +on: [ push, pull_request ] jobs: lint: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 642de39..68d2fa8 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,12 +1,12 @@ -on: [push, pull_request] +on: [ push, pull_request ] name: pylint jobs: gitHubActionForPylint: name: GitHub Action for pylint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: GitHub Action for pylint - uses: cclauss/GitHub-Action-for-pylint@master - with: - args: pylint **/*.py + - uses: actions/checkout@v4 + - name: GitHub Action for pylint + uses: cclauss/GitHub-Action-for-pylint@master + with: + args: pylint **/*.py diff --git a/README.md b/README.md index 47fda7a..6694f7e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ * [Client usage](#client-usage) * [Contact](#contact) - # WireGuard Key Exchange wgkex is a WireGuard key exchange and management tool designed and run by FFMUC. @@ -75,7 +74,8 @@ Each worker must run on a machine with a unique hostname, as it is used for sepa This tool is intended to facilitate running BATMAN over VXLAN over WireGuard as a means to create encrypted high-performance mesh links. -For further information, please see this [presentation on the architecture](https://www.slideshare.net/AnnikaWickert/ffmuc-goes-wild-infrastructure-recap-2020-rc3) +For further information, please see +this [presentation on the architecture](https://www.slideshare.net/AnnikaWickert/ffmuc-goes-wild-infrastructure-recap-2020-rc3) ## Installation @@ -130,6 +130,7 @@ python3 -c 'from wgkex.worker.app import main; main()' ## Client usage The client can be used via CLI: + ``` $ wget -q -O- --post-data='{"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUSitVag9mS7pSXUesNM0ESnvj/wwehkg="}' --header='Content-Type:application/json' 'http://127.0.0.1:5000/api/v1/wg/key/exchange' { @@ -138,6 +139,7 @@ $ wget -q -O- --post-data='{"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUS ``` Or via python: + ```python import requests key_data = {"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUSitVag9mS7pSXUesNM0ESnvj/wwehkg="} @@ -169,7 +171,6 @@ sudo ip link set wg-welt up sudo ip link set vx-welt up ``` - ## Contact [Freifunk Munich Mattermost](https://chat.ffmuc.net) diff --git a/docker-compose.yml b/docker-compose.yml index 0f96056..4e94b32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: - ./volumes/mosquitto/data:/mosquitto/data - ./volumes/mosquitto/log:/mosquitto/log ports: - - "9001:9001" + - "9001:9001" broker: image: ghcr.io/freifunkmuc/wgkex:latest @@ -17,7 +17,7 @@ services: restart: unless-stopped ports: - "5000:5000" - #volumes: + #volumes: #- ./config/broker/wgkex.yaml:/etc/wgkex.yaml environment: WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt} @@ -35,7 +35,7 @@ services: command: worker restart: unless-stopped #volumes: - #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml + #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml environment: WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt} WGKEX_DOMAIN_PREFIX: ${WGKEX_DOMAIN_PREFIX-ffmuc_} diff --git a/wgkex/broker/app.py b/wgkex/broker/app.py index 5001dc3..5a1fd86 100644 --- a/wgkex/broker/app.py +++ b/wgkex/broker/app.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""wgkex broker""" import re import dataclasses import logging @@ -17,7 +18,6 @@ from wgkex.config import config from wgkex.common import logger - WG_PUBKEY_PATTERN = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw480]=$") @@ -99,7 +99,7 @@ def wg_key_exchange() -> Tuple[str, int]: @mqtt.on_connect() def handle_mqtt_connect( - client: mqtt_client.Client, userdata: bytes, flags: Any, rc: Any + client: mqtt_client.Client, userdata: bytes, flags: Any, rc: Any ) -> None: """Prints status of connect message.""" # TODO(ruairi): Clarify current usage of this function. @@ -113,7 +113,7 @@ def handle_mqtt_connect( @mqtt.on_message() def handle_mqtt_message( - client: mqtt_client.Client, userdata: bytes, message: mqtt_client.MQTTMessage + client: mqtt_client.Client, userdata: bytes, message: mqtt_client.MQTTMessage ) -> None: """Prints message contents.""" # TODO(ruairi): Clarify current usage of this function. diff --git a/wgkex/broker/templates/index.html b/wgkex/broker/templates/index.html index cf56495..4bd1383 100644 --- a/wgkex/broker/templates/index.html +++ b/wgkex/broker/templates/index.html @@ -1,6 +1,6 @@ -wgkex + wgkex

WGKEX

diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py index fd1050e..95a08e8 100644 --- a/wgkex/config/config_test.py +++ b/wgkex/config/config_test.py @@ -4,11 +4,6 @@ import config import yaml -# env for windows dev -import os - -os.environ["WG_CONFIG_OS_ENV"] = "env.example" - _VALID_CFG = ( "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883" "\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index 51b18ca..ef99e46 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -20,6 +20,14 @@ class DomainsNotInConfig(Error): """If no domains exist in configuration file.""" +class PrefixesNotInConfig(Error): + """If no prefixes exist in configuration file.""" + + +class DomainsAreNotUnique(Error): + """If non-unique domains exist in configuration file.""" + + def flush_workers(domain: Text) -> None: """Calls peer flush every _CLEANUP_TIME interval.""" while True: @@ -35,20 +43,55 @@ def clean_up_worker(domains: List[Text]) -> None: domains: list of domains. """ logger.debug("Cleaning up the following domains: %s", domains) - prefix = config.load_config().get("domain_prefix") # ToDo + prefixes = config.load_config().get("domain_prefix") + cleanup_counter = 0 + # ToDo: do we need a check if every domain got gleaned? + for prefix in prefixes: + for domain in domains: + if prefix in domain: + logger.info("Scheduling cleanup task for %s, ", domain) + try: + cleaned_domain = domain.split(prefix)[1] + cleanup_counter += 1 + except IndexError: + logger.error( + "Cannot strip domain with prefix %s from passed value %s. Skipping cleanup operation", + prefix, + domain, + ) + continue + thread = threading.Thread(target=flush_workers, args=(cleaned_domain,)) + thread.start() + if cleanup_counter < len(domains): + logger.error( + "Not every domain got cleaned. Check domains for missing prefixes", repr(domains), repr(prefixes), ) + + +def check_all_domains_unique(domains): + """strips off prefixes and checks if domains are unique + + Args: + domains: [str] + Returns: + boolean + """ + prefixes = config.load_config().get("prefixes") + if not prefixes: + raise PrefixesNotInConfig("Could not locate prefixes in configuration.") + unique_domains = [] for domain in domains: - logger.info("Scheduling cleanup task for %s, ", domain) - try: - cleaned_domain = domain.split(prefix)[1] - except IndexError: - logger.error( - "Cannot strip domain with prefix %s from passed value %s. Skipping cleanup operation", - prefix, - domain, - ) - continue - thread = threading.Thread(target=flush_workers, args=(cleaned_domain,)) - thread.start() + for prefix in prefixes: + if prefix in domain: + stripped_domain = domain.split(prefix)[1] + if stripped_domain in unique_domains: + logger.error( + "We have a non-unique domain here", + domain, + ) + return False + else: + unique_domains.append(stripped_domain) + return True def main(): @@ -56,10 +99,13 @@ def main(): Raises: DomainsNotInConfig: If no domains were found in configuration file. + DomainsAreNotUnique: If there were non-unique domains after stripping prefix """ domains = config.load_config().get("domains") if not domains: raise DomainsNotInConfig("Could not locate domains in configuration.") + if not check_all_domains_unique(domains): + raise DomainsAreNotUnique("There are non-unique domains! Check config.") clean_up_worker(domains) watch_queue() mqtt.connect() diff --git a/wgkex/worker/app_test.py b/wgkex/worker/app_test.py index 66dee4d..7ae0ed6 100644 --- a/wgkex/worker/app_test.py +++ b/wgkex/worker/app_test.py @@ -13,9 +13,9 @@ def setUp(self) -> None: def test_main_success(self, connect_mock, config_mock): """Ensure we can execute main.""" connect_mock.return_value = None - test_prefix = "TEST_PREFIX_" # probably ToDo + test_prefixes = "TEST_PREFIX_, TEST_PREFIX2_" config_mock.return_value = dict( - domains=[f"{test_prefix}domain.one"], domain_prefix=test_prefix + domains=[f"{test_prefixes[1]}domain.one"], domain_prefix=test_prefixes ) with mock.patch("app.flush_workers", return_value=None): app.main() @@ -34,9 +34,9 @@ def test_main_fails_no_domain(self, connect_mock, config_mock): @mock.patch.object(app.mqtt, "connect", autospec=True) def test_main_fails_bad_domain(self, connect_mock, config_mock): """Ensure we fail when domains are badly formatted.""" - test_prefix = "TEST_PREFIX_" + test_prefixes = "TEST_PREFIX_, TEST_PREFIX2_" config_mock.return_value = dict( - domains=[f"cant_split_domain"], domain_prefix=test_prefix + domains=[f"cant_split_domain"], domain_prefix=test_prefixes ) connect_mock.return_value = None with mock.patch("app.flush_workers", return_value=None): diff --git a/wgkex/worker/mqtt.py b/wgkex/worker/mqtt.py index 491f472..1c1cf31 100644 --- a/wgkex/worker/mqtt.py +++ b/wgkex/worker/mqtt.py @@ -84,15 +84,19 @@ def on_message(client: mqtt.Client, userdata: Any, message: mqtt.MQTTMessage) -> """ # TODO(ruairi): Check bounds and raise exception here. logger.debug("Got message %s from MTQQ", message) - domain_prefix = load_config().get("domain_prefix") # ToDo - domain = re.search(r"/.*" + domain_prefix + "(\w+)/", message.topic) + domain_prefixes = load_config().get("domain_prefixes") + domain = None + for domain_prefix in domain_prefixes: + domain = re.search(r"/.*" + domain_prefix + "(\w+)/", message.topic) + if domain: + break if not domain: raise ValueError( - "Could not find a match for %s on %s", domain_prefix, message.topic + "Could not find a match for %s on %s", repr(domain_prefixes), message.topic ) + # this will not work, if we have non-unique prefix stripped domains domain = domain.group(1) logger.debug("Found domain %s", domain) - logger.info( f"Received create message for key {str(message.payload.decode('utf-8'))} on domain {domain} adding to queue" ) diff --git a/wgkex/worker/netlink.py b/wgkex/worker/netlink.py index d4f0656..9f44776 100644 --- a/wgkex/worker/netlink.py +++ b/wgkex/worker/netlink.py @@ -203,6 +203,6 @@ def find_stale_wireguard_clients(wg_interface: str) -> List: client.get_attr("WGPEER_A_PUBLIC_KEY").decode("utf-8") for client in all_clients if client.get_attr("WGPEER_A_LAST_HANDSHAKE_TIME").get("tv_sec", int()) - < three_hrs_in_secs + < three_hrs_in_secs ] return ret From 3242276277b1015177de434c27ef9330a404b4ee Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 20:10:09 +0100 Subject: [PATCH 05/15] change quoting --- entrypoint | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/entrypoint b/entrypoint index 3650bfc..a6364c5 100755 --- a/entrypoint +++ b/entrypoint @@ -1,15 +1,15 @@ #!/bin/bash set -e -: "${WGKEX_DOMAINS:='ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events'}" -: "${WGKEX_DOMAIN_PREFIXES:='ffmuc_, ffdon_, ffwert_'}" -: "${WGKEX_DEBUG:='DEBUG'}" -: "${MQTT_BROKER_URL:='mqtt'}" -: "${MQTT_BROKER_PORT:='1883'}" -: "${MQTT_USERNAME:=''}" -: "${MQTT_PASSWORD:=''}" -: "${MQTT_KEEPALIVE:='5'}" -: "${MQTT_TLS:='False'}" +: "${WGKEX_DOMAINS:=ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events}" +: "${WGKEX_DOMAIN_PREFIXES:=ffmuc_, ffdon_, ffwert_}" +: "${WGKEX_DEBUG:=DEBUG}" +: "${MQTT_BROKER_URL:=mqtt}" +: "${MQTT_BROKER_PORT:=1883}" +: "${MQTT_USERNAME:=}" +: "${MQTT_PASSWORD:=}" +: "${MQTT_KEEPALIVE:=5}" +: "${MQTT_TLS:=False}" mk_config() { if [ ! -e /etc/wgkex.yaml ] ; then From 1603982558f03ad5793e70dedd08b91da1129d0e Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 20:12:22 +0100 Subject: [PATCH 06/15] fix entrypoint --- entrypoint | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/entrypoint b/entrypoint index a6364c5..8242236 100755 --- a/entrypoint +++ b/entrypoint @@ -19,10 +19,9 @@ IFS=", " for i in $WGKEX_DOMAINS; do echo " - $i" done -cat < Date: Fri, 24 Nov 2023 20:14:12 +0100 Subject: [PATCH 07/15] fix entrypoint --- entrypoint | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/entrypoint b/entrypoint index 8242236..1b34612 100755 --- a/entrypoint +++ b/entrypoint @@ -19,13 +19,12 @@ IFS=", " for i in $WGKEX_DOMAINS; do echo " - $i" done -# cat < Date: Fri, 24 Nov 2023 20:14:57 +0100 Subject: [PATCH 08/15] fix entrypoint --- entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint b/entrypoint index 1b34612..7c4b1a6 100755 --- a/entrypoint +++ b/entrypoint @@ -20,7 +20,7 @@ for i in $WGKEX_DOMAINS; do echo " - $i" done echo "domain_prefixes:" -for i in WGKEX_DOMAIN_PREFIXES; do +for i in $WGKEX_DOMAIN_PREFIXES; do echo " - $i" done cat < Date: Fri, 24 Nov 2023 20:24:14 +0100 Subject: [PATCH 09/15] fix config --- wgkex/worker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index ef99e46..fa59c2a 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -75,7 +75,7 @@ def check_all_domains_unique(domains): Returns: boolean """ - prefixes = config.load_config().get("prefixes") + prefixes = config.load_config().get("domain_prefixes") if not prefixes: raise PrefixesNotInConfig("Could not locate prefixes in configuration.") unique_domains = [] From 52aadf308223449578f7fb32eb5b75af9610b86a Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 20:24:14 +0100 Subject: [PATCH 10/15] fix config --- wgkex/worker/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index ef99e46..92be6fa 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -43,7 +43,7 @@ def clean_up_worker(domains: List[Text]) -> None: domains: list of domains. """ logger.debug("Cleaning up the following domains: %s", domains) - prefixes = config.load_config().get("domain_prefix") + prefixes = config.load_config().get("domain_prefixes") cleanup_counter = 0 # ToDo: do we need a check if every domain got gleaned? for prefix in prefixes: @@ -75,7 +75,7 @@ def check_all_domains_unique(domains): Returns: boolean """ - prefixes = config.load_config().get("prefixes") + prefixes = config.load_config().get("domain_prefixes") if not prefixes: raise PrefixesNotInConfig("Could not locate prefixes in configuration.") unique_domains = [] From 78c03d0e402990fa6db8e953914c1633ee4555b4 Mon Sep 17 00:00:00 2001 From: istrator Date: Fri, 24 Nov 2023 21:13:47 +0100 Subject: [PATCH 11/15] edit docker-compose.yml --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4e94b32..825ae68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: #volumes: #- ./config/broker/wgkex.yaml:/etc/wgkex.yaml environment: - WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt} - WGKEX_DOMAIN_PREFIX: ${WGKEX_DOMAIN_PREFIX-ffmuc_} + WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events} + WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_} WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG} MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt} MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883} @@ -37,8 +37,8 @@ services: #volumes: #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml environment: - WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt} - WGKEX_DOMAIN_PREFIX: ${WGKEX_DOMAIN_PREFIX-ffmuc_} + WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events} + WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_} WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG} MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt} MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883} From 4203cc987239f1fa48ed6ebe77113c2c2e60fa87 Mon Sep 17 00:00:00 2001 From: istrator Date: Sat, 25 Nov 2023 23:37:36 +0100 Subject: [PATCH 12/15] running black file reformat --- wgkex/broker/app.py | 4 ++-- wgkex/config/config.py | 2 ++ wgkex/config/config_test.py | 9 ++++++--- wgkex/worker/app.py | 5 ++++- wgkex/worker/netlink.py | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/wgkex/broker/app.py b/wgkex/broker/app.py index 5a1fd86..f01ec3f 100644 --- a/wgkex/broker/app.py +++ b/wgkex/broker/app.py @@ -99,7 +99,7 @@ def wg_key_exchange() -> Tuple[str, int]: @mqtt.on_connect() def handle_mqtt_connect( - client: mqtt_client.Client, userdata: bytes, flags: Any, rc: Any + client: mqtt_client.Client, userdata: bytes, flags: Any, rc: Any ) -> None: """Prints status of connect message.""" # TODO(ruairi): Clarify current usage of this function. @@ -113,7 +113,7 @@ def handle_mqtt_connect( @mqtt.on_message() def handle_mqtt_message( - client: mqtt_client.Client, userdata: bytes, message: mqtt_client.MQTTMessage + client: mqtt_client.Client, userdata: bytes, message: mqtt_client.MQTTMessage ) -> None: """Prints message contents.""" # TODO(ruairi): Clarify current usage of this function. diff --git a/wgkex/config/config.py b/wgkex/config/config.py index c548543..0659d69 100644 --- a/wgkex/config/config.py +++ b/wgkex/config/config.py @@ -1,4 +1,5 @@ """Configuration handling class.""" +import logging import os import sys import yaml @@ -131,6 +132,7 @@ def fetch_config_from_disk() -> str: The file contents as string. """ config_file = os.environ.get(WG_CONFIG_OS_ENV, WG_CONFIG_DEFAULT_LOCATION) + logging.debug("getting config_file: %s", repr(config_file)) try: with open(config_file, "r") as stream: return stream.read() diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py index 95a08e8..3c33148 100644 --- a/wgkex/config/config_test.py +++ b/wgkex/config/config_test.py @@ -6,9 +6,12 @@ _VALID_CFG = ( "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n broker_port: 1883" - "\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") -_INVALID_LINT = ("domain_prefixes: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " - "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n") + "\n broker_url: mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n" +) +_INVALID_LINT = ( + "domain_prefixes: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n broker_port: 1883\n broker_url: " + "mqtt://broker\n keepalive: 5\n password: pass\n tls: true\n username: user\n" +) _INVALID_CFG = "asdasdasdasd" diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index 92be6fa..9424469 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -64,7 +64,10 @@ def clean_up_worker(domains: List[Text]) -> None: thread.start() if cleanup_counter < len(domains): logger.error( - "Not every domain got cleaned. Check domains for missing prefixes", repr(domains), repr(prefixes), ) + "Not every domain got cleaned. Check domains for missing prefixes", + repr(domains), + repr(prefixes), + ) def check_all_domains_unique(domains): diff --git a/wgkex/worker/netlink.py b/wgkex/worker/netlink.py index 9f44776..d4f0656 100644 --- a/wgkex/worker/netlink.py +++ b/wgkex/worker/netlink.py @@ -203,6 +203,6 @@ def find_stale_wireguard_clients(wg_interface: str) -> List: client.get_attr("WGPEER_A_PUBLIC_KEY").decode("utf-8") for client in all_clients if client.get_attr("WGPEER_A_LAST_HANDSHAKE_TIME").get("tv_sec", int()) - < three_hrs_in_secs + < three_hrs_in_secs ] return ret From 82ca95c242b755659b9d9a9c6595d71bc9253ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?istrator=C2=B2?= Date: Sun, 26 Nov 2023 14:20:29 +0100 Subject: [PATCH 13/15] revert .github/ whitespace changes --- .github/buildkitd.toml | 2 +- .github/dependabot.yml | 18 +++++++++--------- .github/workflows/bazel.yml | 2 +- .github/workflows/black.yml | 2 +- .github/workflows/pylint.yml | 12 ++++++------ 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml index 0af5d20..ff6039f 100644 --- a/.github/buildkitd.toml +++ b/.github/buildkitd.toml @@ -1,2 +1,2 @@ [worker.oci] -max-parallelism = 4 + max-parallelism = 4 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index feea51e..92bfa4c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,18 +5,18 @@ version: 2 updates: - package-ecosystem: pip directory: / - schedule: { interval: monthly } - reviewers: [ freifunkMUC/salt-stack ] - assignees: [ freifunkMUC/salt-stack ] + schedule: {interval: monthly} + reviewers: [freifunkMUC/salt-stack] + assignees: [freifunkMUC/salt-stack] - package-ecosystem: github-actions directory: / - schedule: { interval: monthly } - reviewers: [ freifunkMUC/salt-stack ] - assignees: [ freifunkMUC/salt-stack ] + schedule: {interval: monthly} + reviewers: [freifunkMUC/salt-stack] + assignees: [freifunkMUC/salt-stack] - package-ecosystem: docker directory: / - schedule: { interval: monthly } - reviewers: [ freifunkMUC/salt-stack ] - assignees: [ freifunkMUC/salt-stack ] + schedule: {interval: monthly} + reviewers: [freifunkMUC/salt-stack] + assignees: [freifunkMUC/salt-stack] diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 52a7fdd..1513680 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,6 +1,6 @@ name: Bazel tests -on: [ push, pull_request ] +on: [push, pull_request] jobs: bazel-run: diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 01bee2b..e91693a 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,6 +1,6 @@ name: Lint -on: [ push, pull_request ] +on: [push, pull_request] jobs: lint: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 68d2fa8..642de39 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,12 +1,12 @@ -on: [ push, pull_request ] +on: [push, pull_request] name: pylint jobs: gitHubActionForPylint: name: GitHub Action for pylint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: GitHub Action for pylint - uses: cclauss/GitHub-Action-for-pylint@master - with: - args: pylint **/*.py + - uses: actions/checkout@v4 + - name: GitHub Action for pylint + uses: cclauss/GitHub-Action-for-pylint@master + with: + args: pylint **/*.py From 627bfddfb6c1c46cf521f37a43100b4cec6545cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?istrator=C2=B2?= Date: Sun, 26 Nov 2023 14:32:28 +0100 Subject: [PATCH 14/15] revert changes on README.md --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6694f7e..88a466c 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,15 @@ - [WireGuard Key Exchange](#wireguard-key-exchange) * [Overview](#overview) + [Frontend broker](#frontend-broker) - - [POST /api/v1/wg/key/exchange](#post-apiv1wgkeyexchange) + - [POST /api/v1/wg/key/exchange](#post--api-v1-wg-key-exchange) + [Backend worker](#backend-worker) * [Installation](#installation) * [Configuration](#configuration) - * [Running the broker](#running-the-broker-and-worker) + * [Running the broker](#running-the-broker) * [Client usage](#client-usage) * [Contact](#contact) + # WireGuard Key Exchange wgkex is a WireGuard key exchange and management tool designed and run by FFMUC. @@ -74,8 +75,7 @@ Each worker must run on a machine with a unique hostname, as it is used for sepa This tool is intended to facilitate running BATMAN over VXLAN over WireGuard as a means to create encrypted high-performance mesh links. -For further information, please see -this [presentation on the architecture](https://www.slideshare.net/AnnikaWickert/ffmuc-goes-wild-infrastructure-recap-2020-rc3) +For further information, please see this [presentation on the architecture](https://www.slideshare.net/AnnikaWickert/ffmuc-goes-wild-infrastructure-recap-2020-rc3) ## Installation @@ -130,7 +130,6 @@ python3 -c 'from wgkex.worker.app import main; main()' ## Client usage The client can be used via CLI: - ``` $ wget -q -O- --post-data='{"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUSitVag9mS7pSXUesNM0ESnvj/wwehkg="}' --header='Content-Type:application/json' 'http://127.0.0.1:5000/api/v1/wg/key/exchange' { @@ -139,13 +138,12 @@ $ wget -q -O- --post-data='{"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUS ``` Or via python: - ```python import requests key_data = {"domain": "ffmuc_welt","public_key": "o52Ge+Rpj4CUSitVag9mS7pSXUesNM0ESnvj/wwehkg="} broker_url = "http://127.0.0.1:5000" push_key = requests.get(f'{broker_url}/api/v1/wg/key/exchange', json=key_data) -print(f'Key push was: {push_key.json().get("Message")}') +print(f'Key push was: {push_key.json().get("Message")]}') ``` ### Worker @@ -171,6 +169,7 @@ sudo ip link set wg-welt up sudo ip link set vx-welt up ``` + ## Contact [Freifunk Munich Mattermost](https://chat.ffmuc.net) From c9d87165ddbb157f3aea2794cf05c8bb69006156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?istrator=C2=B2?= Date: Mon, 27 Nov 2023 22:15:08 +0100 Subject: [PATCH 15/15] add unittests --- docker-compose.yml | 8 +++---- entrypoint | 2 +- env.example | 2 +- wgkex.yaml.example | 8 ------- wgkex/worker/app.py | 11 ++++----- wgkex/worker/app_test.py | 48 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 825ae68..39ba4f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: #volumes: #- ./config/broker/wgkex.yaml:/etc/wgkex.yaml environment: - WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events} - WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_} + WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city} + WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_, ffdon_, ffwert_} WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG} MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt} MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883} @@ -37,8 +37,8 @@ services: #volumes: #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml environment: - WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events} - WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_} + WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city} + WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_, ffdon_, ffwert_} WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG} MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt} MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883} diff --git a/entrypoint b/entrypoint index 7c4b1a6..4975f35 100755 --- a/entrypoint +++ b/entrypoint @@ -1,7 +1,7 @@ #!/bin/bash set -e -: "${WGKEX_DOMAINS:=ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events}" +: "${WGKEX_DOMAINS:=ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city}" : "${WGKEX_DOMAIN_PREFIXES:=ffmuc_, ffdon_, ffwert_}" : "${WGKEX_DEBUG:=DEBUG}" : "${MQTT_BROKER_URL:=mqtt}" diff --git a/env.example b/env.example index ddb01e3..074413e 100644 --- a/env.example +++ b/env.example @@ -1,6 +1,6 @@ # Copy or rename this file to .env and modify if for your needs -#WGKEX_DOMAINS="ffmuc_augsburg, ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt, ffwert_city, ffwert_events" +#WGKEX_DOMAINS="ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city" #WGKEX_DOMAIN_PREFIXES="ffmuc_, ffdon_, ffwert_" #WGKEX_DEBUG="DEBUG" diff --git a/wgkex.yaml.example b/wgkex.yaml.example index 3a4daf2..7b82c71 100644 --- a/wgkex.yaml.example +++ b/wgkex.yaml.example @@ -1,19 +1,11 @@ domains: - - ffmuc_augsburg - - ffmuc_freising - - ffmuc_gauting - ffmuc_muc_cty - ffmuc_muc_nord - ffmuc_muc_ost - ffmuc_muc_sued - ffmuc_muc_west - - ffmuc_uml_nord - - ffmuc_uml_ost - - ffmuc_uml_sued - - ffmuc_uml_west - ffmuc_welt - ffwert_city - - ffwert_events mqtt: broker_url: broker.hivemq.com broker_port: 1883 diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py index 9424469..70aa8fc 100644 --- a/wgkex/worker/app.py +++ b/wgkex/worker/app.py @@ -70,7 +70,7 @@ def clean_up_worker(domains: List[Text]) -> None: ) -def check_all_domains_unique(domains): +def check_all_domains_unique(domains, prefixes): """strips off prefixes and checks if domains are unique Args: @@ -78,9 +78,10 @@ def check_all_domains_unique(domains): Returns: boolean """ - prefixes = config.load_config().get("domain_prefixes") if not prefixes: raise PrefixesNotInConfig("Could not locate prefixes in configuration.") + if not isinstance(prefixes, list): + raise TypeError("prefixes is not a list") unique_domains = [] for domain in domains: for prefix in prefixes: @@ -92,8 +93,7 @@ def check_all_domains_unique(domains): domain, ) return False - else: - unique_domains.append(stripped_domain) + unique_domains.append(stripped_domain) return True @@ -105,9 +105,10 @@ def main(): DomainsAreNotUnique: If there were non-unique domains after stripping prefix """ domains = config.load_config().get("domains") + prefixes = config.load_config().get("domain_prefixes") if not domains: raise DomainsNotInConfig("Could not locate domains in configuration.") - if not check_all_domains_unique(domains): + if not check_all_domains_unique(domains, prefixes): raise DomainsAreNotUnique("There are non-unique domains! Check config.") clean_up_worker(domains) watch_queue() diff --git a/wgkex/worker/app_test.py b/wgkex/worker/app_test.py index 7ae0ed6..111590b 100644 --- a/wgkex/worker/app_test.py +++ b/wgkex/worker/app_test.py @@ -5,17 +5,57 @@ class AppTest(unittest.TestCase): + """unittest.TestCase class""" + def setUp(self) -> None: + """set up unittests""" app._CLEANUP_TIME = 0 + def test_unique_domains_success(self): + """Ensure domain suffixes are unique.""" + test_prefixes = ["TEST_PREFIX_", "TEST_PREFIX2_"] + test_domains = [ + "TEST_PREFIX_DOMAINSUFFIX1", + "TEST_PREFIX_DOMAINSUFFIX2", + "TEST_PREFIX2_DOMAINSUFFIX3", + ] + self.assertTrue( + app.check_all_domains_unique(test_domains, test_prefixes), + "unique domains are not detected unique", + ) + + def test_unique_domains_fail(self): + """Ensure domain suffixes are not unique.""" + test_prefixes = ["TEST_PREFIX_", "TEST_PREFIX2_"] + test_domains = [ + "TEST_PREFIX_DOMAINSUFFIX1", + "TEST_PREFIX_DOMAINSUFFIX2", + "TEST_PREFIX2_DOMAINSUFFIX1", + ] + self.assertFalse( + app.check_all_domains_unique(test_domains, test_prefixes), + "non-unique domains are detected as unique", + ) + + def test_unique_domains_not_list(self): + """Ensure domain prefixes are a list.""" + test_prefixes = "TEST_PREFIX_, TEST_PREFIX2_" + test_domains = [ + "TEST_PREFIX_DOMAINSUFFIX1", + "TEST_PREFIX_DOMAINSUFFIX2", + "TEST_PREFIX2_DOMAINSUFFIX1", + ] + with self.assertRaises(TypeError): + app.check_all_domains_unique(test_domains, test_prefixes) + @mock.patch.object(app.config, "load_config") @mock.patch.object(app.mqtt, "connect", autospec=True) def test_main_success(self, connect_mock, config_mock): """Ensure we can execute main.""" connect_mock.return_value = None - test_prefixes = "TEST_PREFIX_, TEST_PREFIX2_" + test_prefixes = ["TEST_PREFIX_", "TEST_PREFIX2_"] config_mock.return_value = dict( - domains=[f"{test_prefixes[1]}domain.one"], domain_prefix=test_prefixes + domains=[f"{test_prefixes[1]}domain.one"], domain_prefixes=test_prefixes ) with mock.patch("app.flush_workers", return_value=None): app.main() @@ -34,9 +74,9 @@ def test_main_fails_no_domain(self, connect_mock, config_mock): @mock.patch.object(app.mqtt, "connect", autospec=True) def test_main_fails_bad_domain(self, connect_mock, config_mock): """Ensure we fail when domains are badly formatted.""" - test_prefixes = "TEST_PREFIX_, TEST_PREFIX2_" + test_prefixes = ["TEST_PREFIX_", "TEST_PREFIX2_"] config_mock.return_value = dict( - domains=[f"cant_split_domain"], domain_prefix=test_prefixes + domains=[f"cant_split_domain"], domain_prefixes=test_prefixes ) connect_mock.return_value = None with mock.patch("app.flush_workers", return_value=None):