diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3a913be..95a037a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,8 +33,10 @@ jobs: run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + + # don't care about coding standards # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | pytest diff --git a/.gitignore b/.gitignore index fce9f27..af7ee80 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,13 @@ *.pyc .vscode settings.json + +# Ignore Python virtual environment directories virtualenv/* +venv/ +env/ +.venv/ +.env/ growatt2mqtt.cfg growatt2mqtt.service diff --git a/README.md b/README.md index 814a18d..d2fb31f 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,13 @@ protocol_version = {{version}} v0.14 = growatt inverters 2020+ sigineer_v0.11 = sigineer inverters growatt_2020_v1.24 = alt protocol for large growatt inverters - currently untested -eg4_v58 = eg4 inverters ( EG4-6000XP ) - confirmed working srne_v3.9 = SRNE inverters - Untested victron_gx_3.3 = Victron GX Devices - Untested solark_v1.1 = SolarArk 8/12K Inverters - Untested hdhk_16ch_ac_module = some chinese current monitoring device :P + +eg4_v58 = eg4 inverters ( EG4-6000XP ) - confirmed working +eg4_3000ehv_v1 = eg4 inverters ( EG4_3000EHV ) ``` more details on these protocols can be found in the wiki diff --git a/classes/protocol_settings.py b/classes/protocol_settings.py index 8a9db75..6549109 100644 --- a/classes/protocol_settings.py +++ b/classes/protocol_settings.py @@ -1,6 +1,7 @@ import csv from dataclasses import dataclass from enum import Enum +from typing import Union from defs.common import strtoint import itertools import json @@ -22,6 +23,8 @@ class Data_Type(Enum): '''32 bit signed int''' _16BIT_FLAGS = 7 _8BIT_FLAGS = 8 + _32BIT_FLAGS = 9 + ASCII = 84 ''' 2 characters ''' @@ -104,7 +107,9 @@ def getSize(cls, data_type : 'Data_Type'): Data_Type.UINT : 32, Data_Type.SHORT : 16, Data_Type.INT : 32, - Data_Type._16BIT_FLAGS : 16 + Data_Type._8BIT_FLAGS : 8, + Data_Type._16BIT_FLAGS : 16, + Data_Type._32BIT_FLAGS : 32 } if data_type in sizes: @@ -143,6 +148,7 @@ def fromString(cls, name : str): "READDISABLED" : "READDISABLED", "DISABLED" : "READDISABLED", "D" : "READDISABLED", + "R/W" : "WRITE", "RW" : "WRITE", "W" : "WRITE", "YES" : "WRITE" @@ -337,7 +343,7 @@ def determine_delimiter(first_row) -> str: if first_row.count(';') < first_row.count(','): delimeter = ',' - first_row = re.sub(r"\s+" + re.escape(delimeter) +"|" + re.escape(delimeter) +"\s+", delimeter, first_row) #trim values + first_row = re.sub(r"\s+" + re.escape(delimeter) +"|" + re.escape(delimeter) +r"\s+", delimeter, first_row) #trim values csvfile = itertools.chain([first_row], csvfile) #add clean header to begining of iterator @@ -357,7 +363,7 @@ def determine_delimiter(first_row) -> str: character_part = row['unit'] else: # Use regular expressions to extract numeric and character parts - matches = re.findall(r'([0-9.]+)|(.*?)$', row['unit']) + matches = re.findall(r'(\-?[0-9.]+)|(.*?)$', row['unit']) # Iterate over the matches and assign them to appropriate variables for match in matches: @@ -373,7 +379,7 @@ def determine_delimiter(first_row) -> str: variable_name = row['variable name'] if row['variable name'] else row['documented name'] variable_name = variable_name = variable_name.strip().lower().replace(' ', '_').replace('__', '_') #clean name - if re.search("[^a-zA-Z0-9\_]", variable_name) : + if re.search(r"[^a-zA-Z0-9\_]", variable_name) : print("WARNING Invalid Name : " + str(variable_name) + " reg: " + str(row['register']) + " doc name: " + str(row['documented name']) + " path: " + str(path)) #convert to float @@ -659,15 +665,21 @@ def process_register_bytes(self, registry : dict[int,bytes], entry : registry_ma value = int.from_bytes(register[:2], byteorder='big', signed=False) elif entry.data_type == Data_Type.SHORT: value = int.from_bytes(register[:2], byteorder='big', signed=True) - elif entry.data_type == Data_Type._16BIT_FLAGS or entry.data_type == Data_Type._8BIT_FLAGS: + elif entry.data_type == Data_Type._16BIT_FLAGS or entry.data_type == Data_Type._8BIT_FLAGS or entry.data_type == Data_Type._32BIT_FLAGS: #16 bit flags start_bit : int = 0 - if entry.data_type == Data_Type._8BIT_FLAGS: - start_bit = 8 + end_bit : int = 16 #default 16 bit + flag_size : int = Data_Type.getSize(entry.data_type) + + if entry.register_bit > 0: #handle custom bit offset + start_bit = entry.register_bit + + #handle custom sizes, less than 1 register + end_bit = flag_size + start_bit if entry.documented_name+'_codes' in self.protocolSettings.codes: flags : list[str] = [] - for i in range(start_bit, 16): # Iterate over each bit position (0 to 15) + for i in range(start_bit, end_bit): # Iterate over each bit position (0 to 15) byte = i // 8 bit = i % 8 val = register[byte] @@ -680,7 +692,7 @@ def process_register_bytes(self, registry : dict[int,bytes], entry : registry_ma value = ",".join(flags) else: flags : list[str] = [] - for i in range(start_bit, 16): # Iterate over each bit position (0 to 15) + for i in range(start_bit, end_bit): # Iterate over each bit position (0 to 15) # Check if the i-th bit is set if (val >> i) & 1: flags.append("1") @@ -762,32 +774,58 @@ def process_register_ushort(self, registry : dict[int, int], entry : registry_ma value = -value #value = struct.unpack(' 0: #handle custom bit offset + start_bit = entry.register_bit + + #handle custom sizes, less than 1 register + end_bit = flag_size + start_bit + + offset : int = 0 + #calculate current offset for mutliregiter values, were assuming concatenate registers is in order, 0 being the first / lowest + #offset should always be >= 0 + if entry.concatenate: + offset : int = entry.register - entry.concatenate_registers[0] + + #compensate for current offset + end_bit = end_bit - (offset * 16) + + val = registry[entry.register] if entry.documented_name+'_codes' in self.codes: flags : list[str] = [] - for i in range(start_bit, 16): # Iterate over each bit position (0 to 15) - # Check if the i-th bit is set - if (val >> i) & 1: - flag_index = "b"+str(i) - if flag_index in self.codes[entry.documented_name+'_codes']: - flags.append(self.codes[entry.documented_name+'_codes'][flag_index]) + offset : int = 0 + + if end_bit > 0: + end : int = 16 if end_bit >= 16 else end_bit + for i in range(start_bit, end): # Iterate over each bit position (0 to 15) + # Check if the i-th bit is set + if (val >> i) & 1: + flag_index = "b"+str(i+offset) + if flag_index in self.codes[entry.documented_name+'_codes']: + flags.append(self.codes[entry.documented_name+'_codes'][flag_index]) + value = ",".join(flags) else: flags : list[str] = [] - for i in range(start_bit, 16): # Iterate over each bit position (0 to 15) - # Check if the i-th bit is set - if (val >> i) & 1: - flags.append("1") - else: - flags.append("0") + if end_bit > 0: + end : int = 16 if end_bit >= 16 else end_bit + for i in range(start_bit, end): # Iterate over each bit position (0 to 15) + # Check if the i-th bit is set + if (val >> i) & 1: + flags.append("1") + else: + flags.append("0") + value = ''.join(flags) + elif entry.data_type.value > 200 or entry.data_type == Data_Type.BYTE: #bit types bit_size = Data_Type.getSize(entry.data_type) bit_mask = (1 << bit_size) - 1 # Create a mask for extracting X bits @@ -823,7 +861,7 @@ def process_register_ushort(self, registry : dict[int, int], entry : registry_ma return value - def process_registery(self, registry : dict[int,int] | dict[int,bytes] , map : list[registry_map_entry]) -> dict[str,str]: + def process_registery(self, registry : Union[dict[int, int], dict[int, bytes]] , map : list[registry_map_entry]) -> dict[str,str]: '''process registry into appropriate datatypes and names -- maybe add func for single entry later?''' concatenate_registry : dict = {} @@ -856,6 +894,10 @@ def process_registery(self, registry : dict[int,int] | dict[int,bytes] , map : l concatenated_value = concatenated_value + str(concatenate_registry[key]) del concatenate_registry[key] + #replace null characters with spaces and trim + if entry.data_type == Data_Type.ASCII: + concatenated_value = concatenated_value.replace("\x00", " ").strip() + info[entry.variable_name] = concatenated_value else: info[entry.variable_name] = value @@ -871,7 +913,7 @@ def validate_registry_entry(self, entry : registry_map_entry, val) -> int: return 0 if entry.data_type == Data_Type.ASCII: - if val and not re.match('[^a-zA-Z0-9\_\-]', val): #validate ascii + if val and not re.match(r'[^a-zA-Z0-9\_\-]', val): #validate ascii if entry.value_regex: #regex validation if re.match(entry.value_regex, val): if entry.concatenate: diff --git a/classes/transports/transport_base.py b/classes/transports/transport_base.py index 702c1af..048c62e 100644 --- a/classes/transports/transport_base.py +++ b/classes/transports/transport_base.py @@ -33,8 +33,12 @@ class transport_base: def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_settings' = None) -> None: + #apply log level to logger + self._log_level = logging.getLevelName(settings.get('log_level', fallback='INFO')) self._log : logging.Logger = logging.getLogger(__name__) - logging.basicConfig(level=logging.DEBUG) + + self._log.setLevel(self._log_level) + logging.basicConfig(level=self._log_level) self.transport_name = settings.name #section name diff --git a/config.cfg.example b/config.cfg.example index bdac533..74474c7 100644 --- a/config.cfg.example +++ b/config.cfg.example @@ -2,6 +2,9 @@ log_level = DEBUG [transport.0] #name must be unique, ie: transport.modbus +#logging level for transport +log_level = DEBUG + #rs485 / modbus device #protocol config files are located in protocols/ protocol_version = v0.14 diff --git a/defs/common.py b/defs/common.py index 4428b94..f3e2c85 100644 --- a/defs/common.py +++ b/defs/common.py @@ -16,6 +16,10 @@ def strtobool (val): def strtoint(val : str) -> int: ''' converts str to int, but allows for hex string input, identified by x prefix''' + + if isinstance(val, int): #is already int. + return val + if val and val[0] == 'x': return int.from_bytes(bytes.fromhex(val[1:]), byteorder='big') diff --git a/docs/EG4-3000-EHV - MODBUS Communication Protocol.pdf b/docs/EG4-3000-EHV - MODBUS Communication Protocol.pdf new file mode 100644 index 0000000..2d6106c Binary files /dev/null and b/docs/EG4-3000-EHV - MODBUS Communication Protocol.pdf differ diff --git a/protocol_gateway.py b/protocol_gateway.py index 7869020..b992d48 100644 --- a/protocol_gateway.py +++ b/protocol_gateway.py @@ -96,7 +96,7 @@ class Protocol_Gateway: def __init__(self, config_file : str): self.__log = logging.getLogger('invertermodbustomqqt_log') handler = logging.StreamHandler(sys.stdout) - self.__log.setLevel(logging.DEBUG) + #self.__log.setLevel(logging.DEBUG) formatter = logging.Formatter('[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s') handler.setFormatter(formatter) self.__log.addHandler(handler) diff --git a/protocols/eg4_3000ehv_v1.holding_registry_map.csv b/protocols/eg4_3000ehv_v1.holding_registry_map.csv new file mode 100644 index 0000000..de29619 --- /dev/null +++ b/protocols/eg4_3000ehv_v1.holding_registry_map.csv @@ -0,0 +1,62 @@ +variable name,data type,register,documented name,unit,values,writable,note +,32BIT_FLAGS,100~101,Fault code,,"{ ""b0"": ""Reserve"", ""b1"": ""Over temperature of DCDC module"", ""b2"": ""Battery over voltage"", ""b3"": ""Reserve"", ""b4"": ""Output short circuited"", ""b5"": ""Over Inverter voltage"", ""b6"": ""Output over load"", ""b7"": ""Bus over voltage"", ""b8"": ""Bus soft start times out"", ""b9"": ""PV over current"", ""b10"": ""PV over voltage"", ""b11"": ""Battery over current"", ""b12"": ""Inverter over current"", ""b13"": ""Bus low voltage"", ""b14"": ""Reserve"", ""b15"": ""Inverter DC component is too high"", ""b16"": ""Reserve"", ""b17"": ""The zero bias of Output current is too large"", ""b18"": ""The zero bias of inverter current is too large"", ""b19"": ""The zero bias of battery current is too large"", ""b20"": ""The zero bias of PV current is too large"", ""b21"": ""Inverter low voltage"", ""b22"": ""Inverter negative power protection"", ""b23"": ""The host in the parallel system is lost"", ""b24"": ""Synchronization signal abnormal in the parallel system"", ""b25"": ""Reserve"", ""b26"": ""Parallel versions are incompatible"" }",R,"32-bit fault code, each bit corresponds to a fault code, see the fault code table for details, fault code 1 corresponds to bit1, fault code 2 corresponds to bit2, and so on" +,32BIT_FLAGS,108~109,Obtain warning code,,"{""b0"": ""Zero crossing loss of mains power"", ""b1"": ""Mains waveform abnormal"", ""b2"": ""Mains over voltage"", ""b3"": ""Mains low voltage"", ""b4"": ""Mains over frequency"", ""b5"": ""Mains low frequency"", ""b6"": ""PV low voltage"", ""b7"": ""Over temperature"", ""b8"": ""Battery low voltage"", ""b9"": ""Battery is not connected"", ""b10"": ""Overload"", ""b11"": ""Battery Eq charging"", ""b12"": ""Battery is discharged at a low voltage and it has not been charged back to the recovery point"", ""b13"": ""Output power derating"", ""b14"": ""Fan blocked"", ""b15"": ""PV energy is too low to be used"", ""b16"": ""Parallel communication interrupted"", ""b17"": ""Output mode of Single and Parallel systems is inconsistent"", ""b18"": ""Battery voltage difference of parallel system is too large""}",R/W,32-bit warning code see the warning code description for details +Serial Number,ASCII,186~197,Series NO,,,R,12 registers +,USHORT,201,Working Mode,,"{""0"": ""Power On Mode"", ""1"": ""Standby mode"", ""2"": ""Mains mode"", ""3"": ""Off-Grid mode"", ""4"": ""Bypass mode"", ""5"": ""Charging mode"", ""6"": ""Fault mode""}",R, +,SHORT,202,Effective mains voltage,-0.1V,,R, +,SHORT,203,Mains Frequency,-0.01Hz,,R, +,SHORT,204,Average mains power,-1w,,R, +,SHORT,205,Affective inverter voltage,-0.1V,,R, +,SHORT,206,Affective inverter current,-0.1A,,R, +,SHORT,207,Inverter frequency,-0.01Hz,,R, +,SHORT,208,Average inverter power,-1w,,R,"Positive numbers indicate inverter output, negative numbers indicate inverter input" +,SHORT,209,Inverter charging power,-1w,,R, +,SHORT,210,Output effective voltage,-0.1V,,R, +,SHORT,211,Output effective Current,-0.1A,,R, +,SHORT,212,Output frequency,-0.01Hz,,R, +,SHORT,213,Output active power,-1w,,R, +,SHORT,214,Output apparent power,-1VA,,R, +,SHORT,215,Battery average voltage,-0.1V,,R, +,SHORT,216,Battery average Current,-0.1A,,R, +,SHORT,217,Battery average power,-1w,,R, +,SHORT,219,PV average voltage,-0.1V,,R, +,SHORT,220,PV average Current,-0.1A,,R, +,SHORT,223,PV average power,-1w,,R, +,SHORT,224,PV charging average power,-1w,,R, +,SHORT,225,load percentage,-1%,,R, +,SHORT,226,DCDC Temperature,-1 C,,R, +,SHORT,227,Inverter Temperature,-1 C,,R, +,USHORT,229,Battery percentage,1%,,R, +,SHORT,232,Battery average current,-0.1A,,R,"Positive number means charging, negative number means discharging" +,SHORT,233,Inverter charging average current,-0.1A,,R, +,SHORT,234,PV charging average current,-0.1A,,R, +,USHORT,300,Output Mode,,"{""0"": ""Single"", ""1"": ""Parallel"", ""2"": ""3 Phase-P1"", ""3"": ""3 Phase-P2"", ""4"": ""3 Phase-P3"", ""5"": ""Split Phase-P1"", ""6"": ""Split Phase-P2""} ",R/W, +,USHORT,301,Output priority,,"{""0"": ""Utility-PV-Battery"", ""1"": ""PV-Utility-Battery"", ""2"": ""PV-Battery-Utility""} ",R/W,0: Utility-PV-Battery 1:PV-Utility-Battery 2: PV-Battery-Utility +,USHORT,302,Input voltage range,,"{""0"": ""Wide range"", ""1"": ""Narrow range""}",R/W,0: Wide range 1: Narrow range +,USHORT,303,Buzzer mode,,"{""0"": ""Mute in all situations"", ""1"": ""Sound when the input source is changed or there is a specific warning or fault"", ""2"": ""Sound when there is a specific warning or fault"", ""3"": ""Sound when fault occurs""}",R/W,0: Mute in all situations; 1: Sound when the input source is changed or there is a specific warning or fault; 2: Sound when there is a specific warning or fault; 3: Sound when fault occurs; +,USHORT,305,LCD backlight,,"{""0"": ""Timed off"", ""1"": ""Always on""} ",R/W,0: Timed off; 1: Always on; +,USHORT,306,LCD automatically returns to the homepage,,"{""0"": ""Do not return automatically;"", ""1"": ""Automatically return after 1 minute;""}",R/W,0: Do not return automatically; 1: Automatically return after 1 minute; +,USHORT,307,Energy_saving mode,,"{""0"": ""Energy-saving mode is off;"", ""1"": ""Energy-saving mode is on;""}",R/W,0: Energy-saving mode is off; 1: Energy-saving mode is on; +,USHORT,308,Overload automatic restart,,"{""0"": ""Overload failure will not restart;"", ""1"": ""Automatic restart after overload failure;""}",R/W,0: Overload failure will not restart; 1: Automatic restart after overload failure; +,USHORT,309,Over temperature automatic restart,,"{""0"": ""Over temperature failure will not restart;"", ""1"": ""Automatic restart after over-temperature fault;""}",R/W,0: Over temperature failure will not restart; 1: Automatic restart after over- temperature fault +,USHORT,310,Overload transfer to bypass enabled,,"{""0"": ""Disable;"", ""1"": ""Enable;""}",R/W,0: Disable; 1: Enable; +,USHORT,313,Battery Eq mode is enabled,,"{""0"": ""Disable;"", ""1"": ""Enable;""} ",R/W,0: Disable; 1: Enable; +,USHORT,320,Output voltage,0.1v,,R/W, +,USHORT,321,Output frequency,0.01Hz,,R/W, +,USHORT,323,Battery overvoltage protection point,0.1V,,R/W, +,USHORT,324,Max charging voltage,0.1V,,R/W, +,USHORT,325,Floating charging voltage,0.1V,,R/W, +,USHORT,326,Battery discharge recovery point in mains mode,0.1V,,R/W, +,USHORT,327,Battery low voltage protection point in mains mode,0.1V,,R/W, +,USHORT,329,Battery low voltage protection point in off_grid mode,0.1V,,R/W, +,USHORT,331,Battery charging priority,,"{""0"": ""Utility priority;"", ""1"": ""PV priority;"", ""2"": ""PV is at the same level as the Utility;"", ""3"": ""Only PV charging is allowed;""}",R/W,0: Utility priority; 1: PV priority; 2: PV is at the same level as the Utility; 3: Only PV charging is allowed +,USHORT,332,Maximum charging current,0.1A,,R/W, +,USHORT,333,Maximum mains charging current,0.1A,,R/W, +,USHORT,334,Eq Charging voltage,0.1V,,R/W, +,USHORT,335,bat_eq_time,min,0~900,R/W,Range: 0~900 +,USHORT,336,Eq Timeout exit,min,0~900,R/W,Range: 0~900 +,USHORT,337,Two Eq charging intervals,day,1~90,R/W,Range:1~90 +,USHORT,406,Turn on mode,,"{""0"": ""Can be turn-on locally or remotely;"", ""1"": ""Only local turn-on;"", ""2"": ""Only remote turn-on;""}",R/W,0: Can be turn-on locally or remotely 1: Only local turn-on 2: Only remote turn-on +,USHORT,420,Remote switch,,"{""0"": ""Remote shutdown"", ""1"": ""Remote turn-on""}",R/W,0: Remote shutdown 1: Remote turn-on +,USHORT,426,Exit the fault mode,,,W,"1: Exit the fault state(only when the inverter enters the fault mode , it could be available )" +,USHORT,643,Rated Power,W,,R, diff --git a/protocols/eg4_3000ehv_v1.json b/protocols/eg4_3000ehv_v1.json new file mode 100644 index 0000000..d98f3b4 --- /dev/null +++ b/protocols/eg4_3000ehv_v1.json @@ -0,0 +1,7 @@ +{ + "transport" : "modbus_rtu", + "baud" : 9600, + "batch_size" : 40, + "send_holding_register": true, + "send_input_register" : false +} \ No newline at end of file diff --git a/pytests/README.md b/pytests/README.md new file mode 100644 index 0000000..946c2f7 --- /dev/null +++ b/pytests/README.md @@ -0,0 +1 @@ +this folder contains tests for github / git \ No newline at end of file diff --git a/pytests/test_example_config.py b/pytests/test_example_config.py new file mode 100644 index 0000000..37bf4b3 --- /dev/null +++ b/pytests/test_example_config.py @@ -0,0 +1,12 @@ +import sys +import os +import pytest + +#move up a folder for tests +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from protocol_gateway import CustomConfigParser + +def test_example_cfg(): + parser = CustomConfigParser() + parser.read("config.cfg.example") \ No newline at end of file diff --git a/pytests/test_protocol_settings.py b/pytests/test_protocol_settings.py new file mode 100644 index 0000000..df8cd34 --- /dev/null +++ b/pytests/test_protocol_settings.py @@ -0,0 +1,18 @@ +import sys +import os +import pytest + +#move up a folder for tests +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from classes.protocol_settings import protocol_settings + +# List of protocols to test +protocols = [os.path.splitext(f)[0] for f in os.listdir("protocols") if f.endswith('.json')] + + +# Parameterized test function +@pytest.mark.parametrize("protocol", protocols) +def test_protocol_setting(protocol : str): + protocolSettings : protocol_settings = protocol_settings(protocol) +