From fc0507836c74ae94278ebd4001a7d95de86b3d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Br=C3=B6cker?= Date: Mon, 18 Jan 2021 10:55:31 +0100 Subject: [PATCH] prepare release 0.9.3 (#543) --- requirements.test.py2.txt | 4 +-- requirements.test.py3.txt | 4 +-- requirements.tox.txt | 1 + requirements.txt | 2 +- src/canmatrix/canmatrix.py | 37 +++++++++----------------- src/canmatrix/formats/__init__.py | 2 +- src/canmatrix/formats/arxml.py | 18 ++++++++----- src/canmatrix/formats/dbc.py | 8 +++--- src/canmatrix/formats/json.py | 2 ++ src/canmatrix/formats/sym.py | 5 +++- src/canmatrix/tests/test_dbc.py | 27 +++++++++++++++++++ src/canmatrix/tests/test_json.py | 4 +++ src/canmatrix/tests/test_sym.py | 44 +++++++++++++++++++++++++++++++ src/canmatrix/utils.py | 20 ++++++++++++++ test/test.py | 3 +++ 15 files changed, 139 insertions(+), 42 deletions(-) diff --git a/requirements.test.py2.txt b/requirements.test.py2.txt index cb6f54941..f50451304 100644 --- a/requirements.test.py2.txt +++ b/requirements.test.py2.txt @@ -16,7 +16,7 @@ tox==3.2.1 typing==3.6.6 virtualenv==16.0.0 xlwt==1.3.0 -xlrd==1.1.0 +xlrd==1.2.0 click==7.0 -lxml==4.5.2 +lxml==4.6.2 enum34==1.1.10 diff --git a/requirements.test.py3.txt b/requirements.test.py3.txt index c820f4cc4..5c289a6ca 100644 --- a/requirements.test.py3.txt +++ b/requirements.test.py3.txt @@ -16,8 +16,8 @@ tox==3.2.1 typing==3.6.6; python_version < '3.5' virtualenv==16.0.0 xlwt==1.3.0 -xlrd==1.1.0 -lxml==4.5.2 +xlrd==1.2.0 +lxml==4.6.2 click==7.0 xlsxwriter==1.2.8 pyaml==20.4.0 diff --git a/requirements.tox.txt b/requirements.tox.txt index b397e90a8..351470739 100644 --- a/requirements.tox.txt +++ b/requirements.tox.txt @@ -8,3 +8,4 @@ typing==3.6.6; python_version < '3.5' virtualenv==16.4.1 click==7.0 enum34==1.1.10; python_version <= '2.7' +xlrd==1.2.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 57f096786..92396cf0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ click lxml xlwt -xlrd +xlrd==1.2.0 xlsxwriter pyyaml future diff --git a/src/canmatrix/canmatrix.py b/src/canmatrix/canmatrix.py index 87e33fe65..4bd401012 100644 --- a/src/canmatrix/canmatrix.py +++ b/src/canmatrix/canmatrix.py @@ -1351,35 +1351,18 @@ def unpack(self, data, report_error=True): return returnDict - def _has_sub_multiplexer(self, parent_multiplexer_name): + def _get_sub_multiplexer(self, parent_multiplexer_name, parent_multiplexer_value): """ - check if any sub-multiplexer in frame - used for complex-multiplexed frame decoding - - :param parent_multiplexer_name: string with name of parent multiplexer - :return: True or False - """ - for signal in self.signals: - if signal.is_multiplexer and signal.muxer_for_signal == parent_multiplexer_name: - return True - return False - - def _get_sub_multiplexer(self, parent_multiplexer_name, parent_multiplexer_value, decoded): - """ - get any sub-multiplexer in frame for decoded data - return multiplexers name and value - used for complex-multiplexed frame decoding + get any sub-multiplexer in frame used + for complex-multiplexed frame decoding :param parent_multiplexer_name: string with name of parent multiplexer :param parent_multiplexer_value: raw_value (int) of parent multiplexer - :param decoded: OrderedDictionary which is returned from canmatrix.Frame.unpack - :return: muxer_name and muxer_value + :return: muxer signal or None """ for signal in self.signals: if signal.is_multiplexer and signal.muxer_for_signal == parent_multiplexer_name and signal.multiplexer_value_in_range(parent_multiplexer_value): - muxer_value = decoded[signal.name].raw_value - muxer_name = signal.name - return muxer_name, muxer_value + return signal def _filter_signals_for_multiplexer(self, multiplexer_name, multiplexer_value): """ @@ -1417,11 +1400,15 @@ def decode(self, data): multiplex_name = None multiplex_value = None - while self._has_sub_multiplexer(multiplex_name): - multiplex_name, multiplex_value = self._get_sub_multiplexer(multiplex_name, multiplex_value, decoded) - decoded_values[multiplex_name] = decoded[multiplex_name] + sub_multiplexer = self._get_sub_multiplexer(multiplex_name, multiplex_value) + while sub_multiplexer is not None: + multiplex_name = sub_multiplexer.name + multiplex_signal = decoded_values[multiplex_name] = decoded[multiplex_name] + multiplex_value = multiplex_signal.raw_value filtered_signals += self._filter_signals_for_multiplexer(multiplex_name, multiplex_value) + sub_multiplexer = self._get_sub_multiplexer(multiplex_name, multiplex_value) + for signal in filtered_signals: decoded_values[signal.name] = decoded[signal.name] return decoded_values diff --git a/src/canmatrix/formats/__init__.py b/src/canmatrix/formats/__init__.py index 718c32fb3..5e74e3f9b 100644 --- a/src/canmatrix/formats/__init__.py +++ b/src/canmatrix/formats/__init__.py @@ -27,7 +27,7 @@ importlib.import_module("canmatrix.formats." + module) loadedFormats.append(module) except ImportError: - logger.exception("%s is not supported", module) + logger.warning("%s is not supported", module) for loadedModule in loadedFormats: supportedFormats[loadedModule] = [] diff --git a/src/canmatrix/formats/arxml.py b/src/canmatrix/formats/arxml.py index 1a12510d7..5ad5874d8 100644 --- a/src/canmatrix/formats/arxml.py +++ b/src/canmatrix/formats/arxml.py @@ -1732,6 +1732,7 @@ def decode_ethernet_helper(ea, float_factory): pdu_sig_mapping = ea.findall("I-SIGNAL-TO-I-PDU-MAPPING", ipdu) get_signals(pdu_sig_mapping, target_frame, ea, None, float_factory) + target_frame.update_receiver() db.add_frame(target_frame) return found_matrixes @@ -1750,14 +1751,14 @@ def decode_flexray_helper(ea, float_factory): found_matrixes[channel_name] = db frames = ea.findall("FLEXRAY-FRAME-TRIGGERING", pc) - for frame in frames: + for frame_element in frames: frame_counter += 1 - slot_id = int(ea.get_child(frame, "SLOT-ID").text) - base_cycle = ea.get_child(frame, "BASE-CYCLE").text - ipdu_triggerings = ea.get_children(frame, "I-PDU-TRIGGERING") - frame_repetition_cycle = ea.find_children_by_path(frame, "CYCLE-REPETITION/CYCLE-REPETITION")[0].text - network_endpoints = pc.findall('.//' + ns + "NETWORK-ENDPOINT") - frame_size = int(ea.find_children_by_path(frame, "FRAME/FRAME-LENGTH")[0].text) + slot_id = int(ea.get_child(frame_element, "SLOT-ID").text) + base_cycle = ea.get_child(frame_element, "BASE-CYCLE").text + ipdu_triggerings = ea.get_children(frame_element, "I-PDU-TRIGGERING") + frame_repetition_cycle = ea.find_children_by_path(frame_element, "CYCLE-REPETITION/CYCLE-REPETITION")[0].text + network_endpoints = pc.findall('.//' + ea.ns + "NETWORK-ENDPOINT") + frame_size = int(ea.find_children_by_path(frame_element, "FRAME/FRAME-LENGTH")[0].text) frame = canmatrix.Frame(size = frame_size, arbitration_id = frame_counter) frame.slot_id = slot_id frame.base_cycle = base_cycle @@ -1888,7 +1889,10 @@ def decode_can_helper(ea, float_factory, ignore_cluster_info): sig_value_hash[sig.name] = 0 frame_data = frame.encode(sig_value_hash) frame.add_attribute("GenMsgStartValue", "".join(["%02x" % x for x in frame_data])) + frame.update_receiver() found_matrixes[bus_name] = db + + return found_matrixes def load(file, **options): diff --git a/src/canmatrix/formats/dbc.py b/src/canmatrix/formats/dbc.py index 0150806e4..9fa555ed0 100644 --- a/src/canmatrix/formats/dbc.py +++ b/src/canmatrix/formats/dbc.py @@ -35,7 +35,7 @@ from builtins import * import canmatrix - +import canmatrix.utils logger = logging.getLogger(__name__) @@ -755,13 +755,15 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None if temp: frame_id = temp.group(1) signal_name = temp.group(2) - temp_list = temp.group(3).split('"') + temp_list = list(canmatrix.utils.escape_aware_split(temp.group(3), '"')) + if frame_id.isnumeric(): # value for Frame try: frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(frame_id))) sg = frame.signal_by_name(signal_name) for i in range(math.floor(len(temp_list) / 2)): val = temp_list[i * 2 + 1] + val = val.replace('\\"', '"') if sg: sg.add_values(temp_list[i * 2], val) except: @@ -930,7 +932,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None ecu.name = ecu.attributes.get("SystemNodeLongSymbol")[1:-1] ecu.del_attribute("SystemNodeLongSymbol") for frame in db.frames: - frame.cycle_time = int(frame.attributes.get("GenMsgCycleTime", 0)) + frame.cycle_time = int(float(frame.attributes.get("GenMsgCycleTime", 0))) if frame.attributes.get("SystemMessageLongSymbol", None) is not None: frame.name = frame.attributes.get("SystemMessageLongSymbol")[1:-1] frame.del_attribute("SystemMessageLongSymbol") diff --git a/src/canmatrix/formats/json.py b/src/canmatrix/formats/json.py index f383fd2d1..1c8c2eb6a 100644 --- a/src/canmatrix/formats/json.py +++ b/src/canmatrix/formats/json.py @@ -93,6 +93,7 @@ def dump(db, f, **options): symbolic_frame = {"name": frame.name, "id": int(frame.arbitration_id.id), "is_extended_frame": frame.arbitration_id.extended, + "is_fd": frame.is_fd, "signals": symbolic_signals} frame_attributes = { attr: frame.attribute(attr) @@ -146,6 +147,7 @@ def dump(db, f, **options): {"name": frame.name, "id": int(frame.arbitration_id.id), "is_extended_frame": frame.arbitration_id.extended, + "is_fd": frame.is_fd, "signals": symbolic_signals, "attributes": frame_attributes, "comment": frame.comment, diff --git a/src/canmatrix/formats/sym.py b/src/canmatrix/formats/sym.py index cc95e1837..b7f71f7f6 100644 --- a/src/canmatrix/formats/sym.py +++ b/src/canmatrix/formats/sym.py @@ -442,7 +442,10 @@ class Mode(object): is_float = False is_ascii = False enumeration = None - if index_offset != 1: + + if tmp_mux == "Mux": + is_signed = False + elif index_offset != 1 : is_signed = True else: is_signed = False diff --git a/src/canmatrix/tests/test_dbc.py b/src/canmatrix/tests/test_dbc.py index 55dbe7ea9..d291a3837 100644 --- a/src/canmatrix/tests/test_dbc.py +++ b/src/canmatrix/tests/test_dbc.py @@ -478,4 +478,31 @@ def test_missing_space(): matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8") assert matrix.frames[0].signals[0].name == "sig1" +def test_escaped_quotes(): + dbc = io.BytesIO(textwrap.dedent(r''' + BO_ 17 Frame_1: 8 Vector__XXX + SG_ Signal : 0|8@1-(1,0)[0|0] "" Vector__XXX + + VAL_ 17 Signal 0 "zero" 1 "one " 2 "string with \"escaped\" double quotes"; + ''').encode('utf-8')) + matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8") + assert matrix.frames[0].signals[0].values[2] == r'string with "escaped" double quotes' + + +def test_float_cycle_time(): + dbc = io.BytesIO(textwrap.dedent(u'''\ + BO_ 17 Frame_1: 8 Vector__XXX + SG_ sig2 : 8|8@1- (1,0) [0|0] "" Vector__XXX + SG_ sig1 : 0|8@1- (1,0) [0|0] "" Vector__XXX + + + BA_DEF_ BO_ "GenMsgCycleTime" INT 10 3600000; + BA_ "GenMsgCycleTime" BO_ 17 100.0; +''').encode('utf-8')) + + + matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8") + + assert matrix.frames[0].cycle_time == 100 + diff --git a/src/canmatrix/tests/test_json.py b/src/canmatrix/tests/test_json.py index 784fdd324..7345d5b0e 100644 --- a/src/canmatrix/tests/test_json.py +++ b/src/canmatrix/tests/test_json.py @@ -88,6 +88,7 @@ def test_import_min_max(): "comment": "", "id": 10, "is_extended_frame": false, + "is_fd": false, "length": 6, "name": "test_frame", "signals": [ @@ -122,6 +123,7 @@ def test_import_native(): "comment": "", "id": 10, "is_extended_frame": false, + "is_fd": false, "length": 6, "name": "test_frame", "signals": [ @@ -167,6 +169,7 @@ def test_import_export_enums(): "comment": "", "id": 10, "is_extended_frame": false, + "is_fd": false, "length": 6, "name": "test_frame", "signals": [ @@ -218,3 +221,4 @@ def test_export_all_native(): assert (data['messages'][0]['signals'][0]['max'] == 42) assert (data['messages'][0]['signals'][0]['factor'] == 0.123) assert (data['messages'][0]['signals'][0]['offset'] == 1) + assert (data['messages'][0]['is_fd'] is False) diff --git a/src/canmatrix/tests/test_sym.py b/src/canmatrix/tests/test_sym.py index ab7354e44..79048f6cd 100644 --- a/src/canmatrix/tests/test_sym.py +++ b/src/canmatrix/tests/test_sym.py @@ -3,6 +3,7 @@ import sys import textwrap from itertools import chain +from pprint import pprint import pytest @@ -353,4 +354,47 @@ def test_types_read(): "Double", "Float", ]) +@pytest.mark.parametrize( + 'var_name,data,raw_value', + ( + ('VarMux1', bytearray([1, 12, 0, 0, 0, 0, 0, 0]), 12), + ('VarMux2', bytearray([2, 0, 0, 0, 23, 0, 0, 0]), 23), + ('VarMux200', bytearray([200, 0, 0, 0, 0, 0, 34, 0]), 34), + ) +) +def test_mux_decode(var_name,data,raw_value): + f = io.BytesIO('''\ + FormatVersion=5.0 // Do not edit this line! + Title="Types Test" + +FormatVersion=5.0 // Do not edit this line! +Title="Test Symbols File" + +{SENDRECEIVE} + +[MuxTestFrame] +ID=002h +DLC=8 +Mux=Mux1 0,8 1 +Var=VarMux1 unsigned 8,8 + +[MuxTestFrame] +DLC=8 +Mux=Mux2 0,8 2 +Var=VarMux2 unsigned 32,8 + +[MuxTestFrame] +DLC=8 +Mux=Mux200 0,8 C8h +Var=VarMux200 unsigned 48,8 + '''.encode('utf-8'), + ) + + matrix = canmatrix.formats.sym.load(f) + # Check no errors loading the matrix + assert matrix.load_errors == [] + frame = matrix.frame_by_name("MuxTestFrame") + r = frame.decode(data) + assert var_name in r.keys(), "Signal {}, not decoded. Only : {}".format(var_name, ','.join(r for r in r.keys())) + assert r[var_name].raw_value == raw_value \ No newline at end of file diff --git a/src/canmatrix/utils.py b/src/canmatrix/utils.py index e523f8777..48bb0b06e 100644 --- a/src/canmatrix/utils.py +++ b/src/canmatrix/utils.py @@ -20,6 +20,26 @@ def quote_aware_space_split(in_line): # type: (str) -> typing.List[str] return [item.decode('utf-8') for item in shlex.split(in_line.strip().encode('utf-8'))] +# https://stackoverflow.com/questions/18092354/python-split-string-without-splitting-escaped-character +def escape_aware_split(string, delimiter): + if len(delimiter) != 1: + raise ValueError('Invalid delimiter: ' + delimiter) + ln = len(string) + i = 0 + j = 0 + while j < ln: + if string[j] == '\\': + if j + 1 >= ln: + yield string[i:j] + return + j += 1 + elif string[j] == delimiter: + yield string[i:j] + i = j + 1 + j += 1 + yield string[i:j] + + def quote_aware_comma_split(string): # type: (str) -> typing.List[str] """ Split a string containing comma separated list of fields. diff --git a/test/test.py b/test/test.py index c75e128e9..90f0983a5 100755 --- a/test/test.py +++ b/test/test.py @@ -44,6 +44,9 @@ def run_tests(): export_types.sort() # TODO: support testing of xlsx # export_types.remove('xlsx') + if "xlsx" in import_types: + # todo issue #541 + import_types.remove("xlsx") if "fibex" in export_types: export_types.remove('fibex')