From eef42fb1c0eaf75d9fa2d66da51bdce6cb77083b Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 5 Jul 2023 16:00:19 +0200 Subject: [PATCH 1/5] draft --- packages/helpermodules/measurement_log.py | 46 +++++++++++++++++++++-- packages/helpermodules/update_config.py | 36 +++++++++++++++++- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/packages/helpermodules/measurement_log.py b/packages/helpermodules/measurement_log.py index e3011f8f65..cb229ac697 100644 --- a/packages/helpermodules/measurement_log.py +++ b/packages/helpermodules/measurement_log.py @@ -3,9 +3,12 @@ import logging import pathlib from pathlib import Path +import re +from paho.mqtt.client import Client as MqttClient, MQTTMessage from typing import Dict, List from control import data +from helpermodules.broker import InternalBrokerClient from helpermodules.pub import Pub from helpermodules import timecheck from control.bat import Bat @@ -13,6 +16,7 @@ from control.counter import Counter from control.ev import Ev from control.pv import Pv +from helpermodules.utils.topic_parser import decode_payload, get_index log = logging.getLogger(__name__) @@ -78,8 +82,8 @@ def save_log(folder): } ... (dynamisch, je nach konfigurierter Anzahl) } - "smarthome_devices": { - "device1": { + "sh": { + "sh1": { "counter": Wh, wenn konfiguriert: "temp1": int in °C, @@ -172,7 +176,8 @@ def save_log(folder): "ev": ev_dict, "counter": counter_dict, "pv": pv_dict, - "bat": bat_dict + "bat": bat_dict, + "sh": LegacySmarthomeLogdata().update() } # json-Objekt in Datei einfügen @@ -206,7 +211,7 @@ def save_log(folder): def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict: # beim Jahreslog werden die Summen aus den Monatssummen berechnet, bei allen anderen aus den absoluten Zählerwerten - totals: Dict[str, Dict] = {"cp": {}, "counter": {}, "pv": {}, "bat": {}} + totals: Dict[str, Dict] = {"cp": {}, "counter": {}, "pv": {}, "bat": {}, "sh": {}} prev_entry: Dict = {} for group in totals.keys(): for entry in entries: @@ -314,3 +319,36 @@ def update_exported(daily_exported: float) -> None: if module == "cp" and m == "all": module_data = data.data.cp_all_data update_imported_exported(totals[module][m]["imported"], totals[module][m]["exported"]) + + +class LegacySmarthomeLogdata: + def __init__(self) -> None: + self.all_received_topics: Dict = {} + + def update(self): + sh_dict = {} + try: + InternalBrokerClient("smarthome-logging", self.on_connect, self.on_message).start_finite_loop() + for topic, payload in self.all_received_topics.items(): + if re.search("openWB/LegacySmartHome/config/get/Devices/[1-9]/device_configured", topic) is not None: + if decode_payload(payload) == 1: + index = get_index(topic) + sh_dict.update({f"sh{index}": {}}) + for topic, payload in self.all_received_topics.items(): + if f"openWB/LegacySmartHome/Devices/{index}/Wh" == topic: + sh_dict[f"sh{index}"].update({"imported": decode_payload(payload)}) + for sensor_id in range(0, 3): + if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic: + sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)}) + except Exception: + log.exception("Fehler im Werte-Logging-Modul für Smarthome") + finally: + return sh_dict + + def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int): + """ connect to broker and subscribe to set topics + """ + client.subscribe("openWB/LegacySmartHome/#", 2) + + def on_message(self, client: MqttClient, userdata, msg: MQTTMessage): + self.all_received_topics.update({msg.topic: msg.payload}) diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 2bf182695a..7c9d3ba5d5 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -24,7 +24,7 @@ class UpdateConfig: - DATASTORE_VERSION = 15 + DATASTORE_VERSION = 16 valid_topic = [ "^openWB/bat/config/configured$", "^openWB/bat/set/charging_power_left$", @@ -499,7 +499,7 @@ def __solve_breaking_changes(self) -> None: try: getattr(self, f"upgrade_datastore_{version}")() except AttributeError: - log.error("missing upgrade function! $version$") + log.error(f"missing upgrade function! {version}") def upgrade_datastore_0(self) -> None: # prevent_switch_stop auf zwei Einstellungen prevent_phase_switch und prevent_charge_stop aufteilen @@ -759,3 +759,35 @@ def upgrade_datastore_14(self) -> None: payload.configuration.pop("ip_adress") Pub().pub(topic.replace("openWB/", "openWB/set/"), payload) Pub().pub("openWB/system/datastore_version", 15) + + def upgrade_datastore_15(self) -> None: + files = glob.glob("/var/www/html/openWB/data/daily_log/*") + files.extend(glob.glob("/var/www/html/openWB/data/monthly_log/*")) + for file in files: + with open(file, "r+") as jsonFile: + try: + content = json.load(jsonFile) + if isinstance(content, List): + try: + new_content = {"entries": content, "totals": measurement_log.get_totals(content)} + jsonFile.seek(0) + json.dump(new_content, jsonFile) + jsonFile.truncate() + log.debug(f"Format der Logdatei {file} aktualisiert.") + except Exception: + log.exception(f"Logfile {file} entspricht nicht dem Dateiformat von Alpha 3.") + except json.decoder.JSONDecodeError: + log.exception( + f"Logfile {file} konnte nicht konvertiert werden, da es keine gültigen json-Daten enthält.") + with open(file, "r+") as jsonFile: + try: + content = json.load(jsonFile) + for e in content["entries"]: + e.update({"sh": {}}) + content["totals"].update({"sh": {}}) + jsonFile.seek(0) + json.dump(content, jsonFile) + jsonFile.truncate() + except Exception: + log.exception(f"Logfile {file} konnte nicht konvertiert werden.") + # Pub().pub("openWB/system/datastore_version", 16) From ae556a3148cd6d169ab541d145886e1ff133adcc Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 6 Jul 2023 09:38:24 +0200 Subject: [PATCH 2/5] Add Smarthome to Logging --- packages/helpermodules/measurement_log.py | 4 ++-- packages/helpermodules/measurement_log_test.py | 7 ++++++- packages/helpermodules/update_config.py | 18 ++---------------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/packages/helpermodules/measurement_log.py b/packages/helpermodules/measurement_log.py index cb229ac697..49cdf27117 100644 --- a/packages/helpermodules/measurement_log.py +++ b/packages/helpermodules/measurement_log.py @@ -223,7 +223,7 @@ def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict: totals[group][module] = {"exported": 0} if group == "pv" else {"imported": 0, "exported": 0} else: for key, value in entry[group][module].items(): - if key != "soc": + if key != "soc" and "temp" not in key: if sum_up_diffs: value = (Decimal(str(value)) + Decimal(str(totals[group][module][key]))) @@ -336,7 +336,7 @@ def update(self): sh_dict.update({f"sh{index}": {}}) for topic, payload in self.all_received_topics.items(): if f"openWB/LegacySmartHome/Devices/{index}/Wh" == topic: - sh_dict[f"sh{index}"].update({"imported": decode_payload(payload)}) + sh_dict[f"sh{index}"].update({"imported": decode_payload(payload), "exported": 0}) for sensor_id in range(0, 3): if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic: sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)}) diff --git a/packages/helpermodules/measurement_log_test.py b/packages/helpermodules/measurement_log_test.py index f3d7b45573..5a79a4d93b 100644 --- a/packages/helpermodules/measurement_log_test.py +++ b/packages/helpermodules/measurement_log_test.py @@ -72,6 +72,7 @@ def test_get_daily_yields(mock_pub): 'date': '13:41', 'ev': {'ev0': {'soc': 0}}, 'pv': {'all': {'exported': 88}, 'pv1': {'exported': 92}}, + "sh": {}, 'timestamp': 1654861269}, {'bat': {'all': {'exported': 0, 'imported': 146.108, 'soc': 53}, 'bat2': {'exported': 0, 'imported': 149.099, 'soc': 53}}, @@ -83,6 +84,7 @@ def test_get_daily_yields(mock_pub): 'date': '13:46', 'ev': {'ev0': {'soc': 4}}, 'pv': {'all': {'exported': 214}, 'pv1': {'exported': 214}}, + "sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 100, "exported": 0}}, 'timestamp': 1654861569}, {'bat': {'all': {'exported': 0, 'imported': 234.308, 'soc': 55}, 'bat2': {'exported': 0, 'imported': 234.308, 'soc': 55}}, @@ -96,6 +98,7 @@ def test_get_daily_yields(mock_pub): 'date': '13:51', 'ev': {'ev0': {'soc': 6}}, 'pv': {'all': {'exported': 339}, 'pv1': {'exported': 339}}, + "sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 200, "exported": 0}}, 'timestamp': 1654861869}, {'bat': {'all': {'exported': 0, 'imported': 234.308, 'soc': 55}, 'bat2': {'exported': 0, 'imported': 234.308, 'soc': 55}}, @@ -107,6 +110,7 @@ def test_get_daily_yields(mock_pub): 'date': '13:51', 'ev': {'ev0': {'soc': 6}}, 'pv': {'all': {'exported': 339}, 'pv1': {'exported': 339}}, + "sh": {"sh1": {"temp0": 300, "temp1": 300, "temp2": 300, "imported": 400, "exported": 0}}, 'timestamp': 1654862069}] TOTALS = {'bat': {'all': {'exported': 0, 'imported': 175.534}, @@ -117,4 +121,5 @@ def test_get_daily_yields(mock_pub): 'cp4': {'exported': 0, 'imported': 85}, 'cp5': {'exported': 0, 'imported': 0}, 'cp6': {'exported': 0, 'imported': 2}}, - 'pv': {'all': {'exported': 251}, 'pv1': {'exported': 247}}} + 'pv': {'all': {'exported': 251}, 'pv1': {'exported': 247}}, + "sh": {"sh1": {"imported": 300, "exported": 0}}} diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 7c9d3ba5d5..85ebcb0307 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -764,21 +764,6 @@ def upgrade_datastore_15(self) -> None: files = glob.glob("/var/www/html/openWB/data/daily_log/*") files.extend(glob.glob("/var/www/html/openWB/data/monthly_log/*")) for file in files: - with open(file, "r+") as jsonFile: - try: - content = json.load(jsonFile) - if isinstance(content, List): - try: - new_content = {"entries": content, "totals": measurement_log.get_totals(content)} - jsonFile.seek(0) - json.dump(new_content, jsonFile) - jsonFile.truncate() - log.debug(f"Format der Logdatei {file} aktualisiert.") - except Exception: - log.exception(f"Logfile {file} entspricht nicht dem Dateiformat von Alpha 3.") - except json.decoder.JSONDecodeError: - log.exception( - f"Logfile {file} konnte nicht konvertiert werden, da es keine gültigen json-Daten enthält.") with open(file, "r+") as jsonFile: try: content = json.load(jsonFile) @@ -788,6 +773,7 @@ def upgrade_datastore_15(self) -> None: jsonFile.seek(0) json.dump(content, jsonFile) jsonFile.truncate() + log.debug(f"Format der Logdatei {file} aktualisiert.") except Exception: log.exception(f"Logfile {file} konnte nicht konvertiert werden.") - # Pub().pub("openWB/system/datastore_version", 16) + Pub().pub("openWB/system/datastore_version", 16) From a852bb42c0a85208dda82264a59cee973f63c238 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 6 Jul 2023 10:00:19 +0200 Subject: [PATCH 3/5] clean up --- packages/helpermodules/measurement_log.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/helpermodules/measurement_log.py b/packages/helpermodules/measurement_log.py index 49cdf27117..31e04e44b1 100644 --- a/packages/helpermodules/measurement_log.py +++ b/packages/helpermodules/measurement_log.py @@ -346,8 +346,6 @@ def update(self): return sh_dict def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int): - """ connect to broker and subscribe to set topics - """ client.subscribe("openWB/LegacySmartHome/#", 2) def on_message(self, client: MqttClient, userdata, msg: MQTTMessage): From abadff2c4fb60a7e5ddcd757770ced88c5b99f8b Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 6 Jul 2023 12:28:16 +0200 Subject: [PATCH 4/5] add names to logging --- packages/helpermodules/measurement_log.py | 41 ++++++++++++++++--- .../helpermodules/measurement_log_test.py | 37 +++++++++++++---- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/packages/helpermodules/measurement_log.py b/packages/helpermodules/measurement_log.py index 31e04e44b1..9c22c61c98 100644 --- a/packages/helpermodules/measurement_log.py +++ b/packages/helpermodules/measurement_log.py @@ -4,8 +4,9 @@ import pathlib from pathlib import Path import re +import string from paho.mqtt.client import Client as MqttClient, MQTTMessage -from typing import Dict, List +from typing import Dict, List, Tuple from control import data from helpermodules.broker import InternalBrokerClient @@ -17,6 +18,7 @@ from control.ev import Ev from control.pv import Pv from helpermodules.utils.topic_parser import decode_payload, get_index +from modules.common.utils.component_parser import get_component_name_by_id log = logging.getLogger(__name__) @@ -103,6 +105,11 @@ def save_log(folder): 'cp5': {'exported': 0, 'imported': 0}, 'cp6': {'exported': 0, 'imported': 64}}, 'pv': {'all': {'imported': 251}, 'pv1': {'imported': 247}}} + }, + "names": { + "counter0": "Mein EVU-Zähler", + "bat1": "Mein toller Speicher", + ... } } @@ -169,6 +176,8 @@ def save_log(folder): except Exception: log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat)) + sh_dict, sh_names = LegacySmarthomeLogdata().update() + new_entry = { "timestamp": current_timestamp, "date": date, @@ -177,7 +186,7 @@ def save_log(folder): "counter": counter_dict, "pv": pv_dict, "bat": bat_dict, - "sh": LegacySmarthomeLogdata().update() + "sh": sh_dict } # json-Objekt in Datei einfügen @@ -204,6 +213,7 @@ def save_log(folder): entries = content["entries"] entries.append(new_entry) content["totals"] = get_totals(entries) + content["names"] = get_names(content["totals"], sh_names) with open(filepath, "w") as jsonFile: json.dump(content, jsonFile) return content["totals"] @@ -245,6 +255,23 @@ def get_totals(entries: List, sum_up_diffs: bool = False) -> Dict: return totals +def get_names(totals: Dict, sh_names: Dict) -> Dict: + names = sh_names + for group in totals.items(): + if group[0] == "sh": + continue + for entry in group[1]: + try: + if "cp" in entry: + names.update({entry: data.data.cp_data[entry].data.config.name}) + elif "all" != entry: + id = entry.strip(string.ascii_letters) + names.update({entry: get_component_name_by_id(int(id))}) + except (ValueError, KeyError): + names.update({entry: entry}) + return names + + def get_daily_log(date: str): try: with open(str(Path(__file__).resolve().parents[2] / "data"/"daily_log"/(date+".json")), "r") as jsonFile: @@ -325,8 +352,9 @@ class LegacySmarthomeLogdata: def __init__(self) -> None: self.all_received_topics: Dict = {} - def update(self): - sh_dict = {} + def update(self) -> Tuple[Dict, Dict]: + sh_dict: Dict = {} + sh_names: Dict = {} try: InternalBrokerClient("smarthome-logging", self.on_connect, self.on_message).start_finite_loop() for topic, payload in self.all_received_topics.items(): @@ -340,10 +368,13 @@ def update(self): for sensor_id in range(0, 3): if f"openWB/LegacySmartHome/Devices/{index}/TemperatureSensor{sensor_id}" == topic: sh_dict[f"sh{index}"].update({f"temp{sensor_id}": decode_payload(payload)}) + for topic, payload in self.all_received_topics.items(): + if f"openWB/LegacySmartHome/config/get/Devices/{index}/device_name" == topic: + sh_names.update({f"sh{index}": decode_payload(payload)}) except Exception: log.exception("Fehler im Werte-Logging-Modul für Smarthome") finally: - return sh_dict + return sh_dict, sh_names def on_connect(self, client: MqttClient, userdata, flags: dict, rc: int): client.subscribe("openWB/LegacySmartHome/#", 2) diff --git a/packages/helpermodules/measurement_log_test.py b/packages/helpermodules/measurement_log_test.py index 5a79a4d93b..88e32f3d92 100644 --- a/packages/helpermodules/measurement_log_test.py +++ b/packages/helpermodules/measurement_log_test.py @@ -1,4 +1,5 @@ import threading +from unittest.mock import Mock import pytest from control.chargepoint import chargepoint @@ -8,14 +9,6 @@ from control import data -def test_get_totals(): - # execution - totals = measurement_log.get_totals(SAMPLE) - - # evaluation - assert totals == TOTALS - - @pytest.fixture(autouse=True) def data_module() -> None: data.data_init(threading.Event()) @@ -27,6 +20,25 @@ def data_module() -> None: data.data.pv_data.update({"all": pv_all.PvAll(), "pv1": pv.Pv(1)}) +def test_get_totals(monkeypatch): + # execution + totals = measurement_log.get_totals(SAMPLE) + + # evaluation + assert totals == TOTALS + + +def test_get_names(monkeypatch): + # setup + component_names_mock = Mock(side_effect=["Speicher", "Zähler", "Wechselrichter"]) + monkeypatch.setattr(measurement_log, "get_component_name_by_id", component_names_mock) + # execution + names = measurement_log.get_names(TOTALS, {"sh1": "Smarthome1"}) + + # evaluation + assert names == NAMES + + def test_get_daily_yields(mock_pub): # setup and execution [measurement_log.update_module_yields(type, TOTALS) for type in ("bat", "counter", "cp", "pv")] @@ -123,3 +135,12 @@ def test_get_daily_yields(mock_pub): 'cp6': {'exported': 0, 'imported': 2}}, 'pv': {'all': {'exported': 251}, 'pv1': {'exported': 247}}, "sh": {"sh1": {"imported": 300, "exported": 0}}} + +NAMES = {'bat2': "Speicher", + 'counter0': "Zähler", + 'cp3': "cp3", + 'cp4': "Standard-Ladepunkt", + 'cp5': "Standard-Ladepunkt", + 'cp6': "Standard-Ladepunkt", + 'pv1': "Wechselrichter", + "sh1": "Smarthome1"} From 468ecce8c92348fc0e973f1cd038c8132ad859ac Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 6 Jul 2023 15:03:48 +0200 Subject: [PATCH 5/5] review --- packages/helpermodules/measurement_log.py | 5 +++-- packages/helpermodules/update_config.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/helpermodules/measurement_log.py b/packages/helpermodules/measurement_log.py index 9c22c61c98..c8346a743c 100644 --- a/packages/helpermodules/measurement_log.py +++ b/packages/helpermodules/measurement_log.py @@ -86,7 +86,8 @@ def save_log(folder): } "sh": { "sh1": { - "counter": Wh, + "exported": Wh, + "imported": Wh, wenn konfiguriert: "temp1": int in °C, "temp2": int in °C, @@ -108,7 +109,7 @@ def save_log(folder): }, "names": { "counter0": "Mein EVU-Zähler", - "bat1": "Mein toller Speicher", + "bat2": "Mein toller Speicher", ... } } diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 85ebcb0307..2e757ec4e7 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -770,6 +770,7 @@ def upgrade_datastore_15(self) -> None: for e in content["entries"]: e.update({"sh": {}}) content["totals"].update({"sh": {}}) + content["names"] = measurement_log.get_names(content["totals"], {}) jsonFile.seek(0) json.dump(content, jsonFile) jsonFile.truncate()