From 4f12f7ce7869cd24bdc0acd224881bf26ec35de2 Mon Sep 17 00:00:00 2001 From: Michael Stovenour Date: Wed, 4 Sep 2019 20:26:22 -0500 Subject: [PATCH 01/10] Resolve issue nutechsoftware/alarmdecoder#issue_40 Solution adds previous leftover to front of buf so leftovers can be searched for newline. It also will process leftover data even if there is no more serial data arriving. --- alarmdecoder/devices/serial_device.py | 49 +++++++++++++++++---------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/alarmdecoder/devices/serial_device.py b/alarmdecoder/devices/serial_device.py index 96bc636..69311fc 100644 --- a/alarmdecoder/devices/serial_device.py +++ b/alarmdecoder/devices/serial_device.py @@ -210,8 +210,14 @@ def timeout_event(): timeout_event.reading = False timeout_event.reading = True - if purge_buffer: - self._buffer = b'' + buf = b'' + if not purge_buffer: + #Add previous leftovers to front of buf so leftovers can be searched for newlines + #If there were no leftovers then self._buffer will be empty + buf = self._buffer + + self._buffer = b'' + got_line, data = False, '' @@ -222,17 +228,27 @@ def timeout_event(): leftovers = b'' try: while timeout_event.reading and not got_line: - read_ready, _, _ = select.select([self._device.fileno()], [], [], 0.5) - if len(read_ready) == 0: + #If there is leftover data then only perform a poll to set the read_ready flag + #otherwise block here until short timeout waiting for more data + select_timeout = 0.5 + if len(self._buffer) > 0: + select_timeout = 0.1 + read_ready, _, _ = select.select([self._device.fileno()], [], [], select_timeout) + if len(read_ready) == 0 and len(self._buffer) == 0: continue - bytes_avail = 0 - if hasattr(self._device, "in_waiting"): - bytes_avail = self._device.in_waiting - else: - bytes_avail = self._device.inWaiting() + #Only read data if there is some waiting + if len(read_ready) > 0: + bytes_avail = 0 + if hasattr(self._device, "in_waiting"): + bytes_avail = self._device.in_waiting + else: + bytes_avail = self._device.inWaiting() + + device_data = self._device.read(bytes_avail) - buf = self._device.read(bytes_avail) + #Append new data to previous leftovers, if any + buf = buf + device_data for idx in range(len(buf)): c = buf[idx] @@ -242,15 +258,14 @@ def timeout_event(): ub = bytes([ub]) # NOTE: AD2SERIAL and AD2PI apparently sends down \xFF on boot. - if ub != b'' and ub != b"\xff": - self._buffer += ub - - if ub == b"\n": - self._buffer = self._buffer.strip(b"\r\n") - + if ub != b'' and ub != b"\xff" and ub != b"\r": + if ub != b"\n": + self._buffer += ub + else: + #found end of line if len(self._buffer) > 0: got_line = True - leftovers = buf[idx:] + leftovers = buf[idx+1:] break except (OSError, serial.SerialException) as err: From 66752a6c8843f5cfaf896a2766d7daa9b1234df8 Mon Sep 17 00:00:00 2001 From: KRKeegan Date: Thu, 25 Jun 2020 16:15:06 -0700 Subject: [PATCH 02/10] Improved Handling of New Zone Faults Avoids some inaccuracies where a newly faulted zone could cause other faulted zones to appear as ready temporarily. Fixes #50 --- alarmdecoder/zonetracking.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/alarmdecoder/zonetracking.py b/alarmdecoder/zonetracking.py index e56ec12..ca3b15f 100644 --- a/alarmdecoder/zonetracking.py +++ b/alarmdecoder/zonetracking.py @@ -198,6 +198,9 @@ def update(self, message): self._update_zone(zone) self._clear_zones(zone) + # Save our spot for the next message. + self._last_zone_fault = zone + else: status = Zone.FAULT if message.check_zone: @@ -207,8 +210,8 @@ def update(self, message): self._zones_faulted.append(zone) self._zones_faulted.sort() - # Save our spot for the next message. - self._last_zone_fault = zone + # A new zone fault, so it is out of sequence. + self._last_zone_fault = 0 self._clear_expired_zones() @@ -245,6 +248,11 @@ def _clear_zones(self, zone): :param zone: current zone being processed :type zone: int """ + + if self._last_zone_fault == 0: + # We don't know what the last faulted zone was, nothing to do + return + cleared_zones = [] found_last_faulted = found_current = at_end = False From 69b0377fb6854206f56eda54a2428fb56c4a43fd Mon Sep 17 00:00:00 2001 From: KRKeegan Date: Tue, 4 Aug 2020 10:49:55 -0700 Subject: [PATCH 03/10] Don't Update Expander Zone Based on Regular Messages Expanders messages handle this directly. Fixes #51 --- alarmdecoder/zonetracking.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alarmdecoder/zonetracking.py b/alarmdecoder/zonetracking.py index e56ec12..739cee5 100644 --- a/alarmdecoder/zonetracking.py +++ b/alarmdecoder/zonetracking.py @@ -151,9 +151,7 @@ def update(self, message): status = Zone.CHECK # NOTE: Expander zone faults are handled differently than - # regular messages. We don't include them in - # self._zones_faulted because they are not reported - # by the panel in it's rolling list of faults. + # regular messages. try: self._update_zone(zone, status=status) @@ -296,7 +294,9 @@ def _clear_zones(self, zone): # Actually remove the zones and trigger the restores. for z in cleared_zones: - self._update_zone(z, Zone.CLEAR) + # Don't clear expander zones, expander messages will fix this + if self._zones[z].expander is False: + self._update_zone(z, Zone.CLEAR) def _clear_expired_zones(self): """ From a3ca3cdead982e4903477254adf97a30be48b2e8 Mon Sep 17 00:00:00 2001 From: f34rdotcom Date: Thu, 27 Aug 2020 13:11:07 -0700 Subject: [PATCH 04/10] Revert "Merge pull request #41 from mstovenour/fix_issue_#40" This reverts commit e2a85ace45281ea17b38ec2c2a641a259cbcf56d, reversing changes made to e9cc8342dd0665bbd3cb333245a9eabb131f1f96. --- alarmdecoder/devices/serial_device.py | 49 ++++++++++----------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/alarmdecoder/devices/serial_device.py b/alarmdecoder/devices/serial_device.py index 69311fc..96bc636 100644 --- a/alarmdecoder/devices/serial_device.py +++ b/alarmdecoder/devices/serial_device.py @@ -210,14 +210,8 @@ def timeout_event(): timeout_event.reading = False timeout_event.reading = True - buf = b'' - if not purge_buffer: - #Add previous leftovers to front of buf so leftovers can be searched for newlines - #If there were no leftovers then self._buffer will be empty - buf = self._buffer - - self._buffer = b'' - + if purge_buffer: + self._buffer = b'' got_line, data = False, '' @@ -228,27 +222,17 @@ def timeout_event(): leftovers = b'' try: while timeout_event.reading and not got_line: - #If there is leftover data then only perform a poll to set the read_ready flag - #otherwise block here until short timeout waiting for more data - select_timeout = 0.5 - if len(self._buffer) > 0: - select_timeout = 0.1 - read_ready, _, _ = select.select([self._device.fileno()], [], [], select_timeout) - if len(read_ready) == 0 and len(self._buffer) == 0: + read_ready, _, _ = select.select([self._device.fileno()], [], [], 0.5) + if len(read_ready) == 0: continue - #Only read data if there is some waiting - if len(read_ready) > 0: - bytes_avail = 0 - if hasattr(self._device, "in_waiting"): - bytes_avail = self._device.in_waiting - else: - bytes_avail = self._device.inWaiting() - - device_data = self._device.read(bytes_avail) + bytes_avail = 0 + if hasattr(self._device, "in_waiting"): + bytes_avail = self._device.in_waiting + else: + bytes_avail = self._device.inWaiting() - #Append new data to previous leftovers, if any - buf = buf + device_data + buf = self._device.read(bytes_avail) for idx in range(len(buf)): c = buf[idx] @@ -258,14 +242,15 @@ def timeout_event(): ub = bytes([ub]) # NOTE: AD2SERIAL and AD2PI apparently sends down \xFF on boot. - if ub != b'' and ub != b"\xff" and ub != b"\r": - if ub != b"\n": - self._buffer += ub - else: - #found end of line + if ub != b'' and ub != b"\xff": + self._buffer += ub + + if ub == b"\n": + self._buffer = self._buffer.strip(b"\r\n") + if len(self._buffer) > 0: got_line = True - leftovers = buf[idx+1:] + leftovers = buf[idx:] break except (OSError, serial.SerialException) as err: From d809cb34cd6123088aa29e94f7b40897f7e7b40a Mon Sep 17 00:00:00 2001 From: f34rdotcom Date: Thu, 27 Aug 2020 16:19:18 -0700 Subject: [PATCH 05/10] Testing new serial reading code. --- alarmdecoder/devices/serial_device.py | 54 ++++++++++----------------- alarmdecoder/util.py | 13 +++++++ 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/alarmdecoder/devices/serial_device.py b/alarmdecoder/devices/serial_device.py index 96bc636..c93b86b 100644 --- a/alarmdecoder/devices/serial_device.py +++ b/alarmdecoder/devices/serial_device.py @@ -14,7 +14,7 @@ import select import sys from .base_device import Device -from ..util import CommError, TimeoutError, NoDeviceError, bytes_hack +from ..util import CommError, TimeoutError, NoDeviceError, bytes_hack, filter_ad2prot_byte class SerialDevice(Device): @@ -141,7 +141,7 @@ def close(self): def fileno(self): """ Returns the file number associated with the device - + :returns: int """ return self._device.fileno() @@ -178,18 +178,18 @@ def read(self): :returns: character read from the device :raises: :py:class:`~alarmdecoder.util.CommError` """ - data = '' + data = b'' try: read_ready, _, _ = select.select([self._device.fileno()], [], [], 0.5) if len(read_ready) != 0: - data = self._device.read(1) + data = filter_ad2prot_byte(self._device.read(1)) except serial.SerialException as err: raise CommError('Error reading from device: {0}'.format(str(err)), err) - return data.decode('utf-8') + return data def read_line(self, timeout=0.0, purge_buffer=False): """ @@ -213,54 +213,38 @@ def timeout_event(): if purge_buffer: self._buffer = b'' - got_line, data = False, '' + got_line, ret = False, None timer = threading.Timer(timeout, timeout_event) if timeout > 0: timer.start() - leftovers = b'' try: - while timeout_event.reading and not got_line: + while timeout_event.reading: read_ready, _, _ = select.select([self._device.fileno()], [], [], 0.5) + if len(read_ready) == 0: continue - bytes_avail = 0 - if hasattr(self._device, "in_waiting"): - bytes_avail = self._device.in_waiting - else: - bytes_avail = self._device.inWaiting() - - buf = self._device.read(bytes_avail) - - for idx in range(len(buf)): - c = buf[idx] - - ub = bytes_hack(c) - if sys.version_info > (3,): - ub = bytes([ub]) - - # NOTE: AD2SERIAL and AD2PI apparently sends down \xFF on boot. - if ub != b'' and ub != b"\xff": - self._buffer += ub + buf = filter_ad2prot_byte(self._device.read(1)) - if ub == b"\n": - self._buffer = self._buffer.strip(b"\r\n") + if buf != b'': + self._buffer += buf[0] - if len(self._buffer) > 0: - got_line = True - leftovers = buf[idx:] - break + if buf == b"\n": + self._buffer = self._buffer.rstrip(b"\r\n") + if len(self._buffer) > 0: + got_line = True + break except (OSError, serial.SerialException) as err: raise CommError('Error reading from device: {0}'.format(str(err)), err) else: if got_line: - data, self._buffer = self._buffer, leftovers + ret, self._buffer = self._buffer, b'' - self.on_read(data=data) + self.on_read(data=ret) else: raise TimeoutError('Timeout while waiting for line terminator.') @@ -268,7 +252,7 @@ def timeout_event(): finally: timer.cancel() - return data.decode('utf-8') + return ret def purge(self): """ diff --git a/alarmdecoder/util.py b/alarmdecoder/util.py index ad6291b..566f2ae 100644 --- a/alarmdecoder/util.py +++ b/alarmdecoder/util.py @@ -93,6 +93,19 @@ def bytes_hack(buf): return ub +def filter_ad2prot_byte(buf): + """ + Return the byte sent in back if valid visible terminal characters or line terminators. + """ + c = buf[0]; + + if (c == '\n' or c == '\r'): + return c + if (c > 31 and c < 127): + return c + else: + return '' + def read_firmware_file(file_path): """ Reads a firmware file into a dequeue for processing. From a48cecee59bfa7e987a7a1aa7741c7b125bf14fb Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 21 Dec 2020 19:44:48 -0500 Subject: [PATCH 06/10] update serial read_line --- alarmdecoder/devices/serial_device.py | 2 +- alarmdecoder/util.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alarmdecoder/devices/serial_device.py b/alarmdecoder/devices/serial_device.py index c93b86b..7be6ddb 100644 --- a/alarmdecoder/devices/serial_device.py +++ b/alarmdecoder/devices/serial_device.py @@ -229,7 +229,7 @@ def timeout_event(): buf = filter_ad2prot_byte(self._device.read(1)) if buf != b'': - self._buffer += buf[0] + self._buffer += buf if buf == b"\n": self._buffer = self._buffer.rstrip(b"\r\n") diff --git a/alarmdecoder/util.py b/alarmdecoder/util.py index 566f2ae..902b21d 100644 --- a/alarmdecoder/util.py +++ b/alarmdecoder/util.py @@ -97,14 +97,14 @@ def filter_ad2prot_byte(buf): """ Return the byte sent in back if valid visible terminal characters or line terminators. """ - c = buf[0]; + c = buf[0] - if (c == '\n' or c == '\r'): - return c + if (c == 10 or c == 13): + return buf if (c > 31 and c < 127): - return c + return buf else: - return '' + return b'' def read_firmware_file(file_path): """ From 33b20aca8c6f26e61d46950030eb9e13aae10ec8 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 21 Dec 2020 21:03:13 -0500 Subject: [PATCH 07/10] fix 1 failing unit test --- alarmdecoder/devices/serial_device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alarmdecoder/devices/serial_device.py b/alarmdecoder/devices/serial_device.py index 7be6ddb..7d38240 100644 --- a/alarmdecoder/devices/serial_device.py +++ b/alarmdecoder/devices/serial_device.py @@ -189,7 +189,7 @@ def read(self): except serial.SerialException as err: raise CommError('Error reading from device: {0}'.format(str(err)), err) - return data + return data.decode('utf-8') def read_line(self, timeout=0.0, purge_buffer=False): """ @@ -252,7 +252,7 @@ def timeout_event(): finally: timer.cancel() - return ret + return ret.decode('utf-8') def purge(self): """ From d143953f8d4c4ff42aea73ecd26a7ca6e7eccebc Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Sun, 31 Jan 2021 18:18:07 -0500 Subject: [PATCH 08/10] fix unit tests --- test/test_ad2.py | 2 ++ test/test_devices.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_ad2.py b/test/test_ad2.py index 716fc49..34feb84 100644 --- a/test/test_ad2.py +++ b/test/test_ad2.py @@ -362,5 +362,7 @@ def test_zone_fault_and_restore(self): self._decoder._on_read(self, data=b'[00010001000000000A--],005,[f70000051003000008020000000000],"FAULT 05 "') self.assertEquals(self._zone_faulted, 5) + self._decoder._on_read(self, data=b'[00010001000000000A--],004,[f70000051003000008020000000000],"FAULT 04 "') + self._decoder._on_read(self, data=b'[00010001000000000A--],004,[f70000051003000008020000000000],"FAULT 05 "') self._decoder._on_read(self, data=b'[00010001000000000A--],004,[f70000051003000008020000000000],"FAULT 04 "') self.assertEquals(self._zone_restored, 3) diff --git a/test/test_devices.py b/test/test_devices.py index 326374a..17cba61 100644 --- a/test/test_devices.py +++ b/test/test_devices.py @@ -81,7 +81,7 @@ def test_read(self): self._device.interface = '/dev/ttyS0' self._device.open(no_reader_thread=True) - with patch.object(self._device._device, 'read') as mock: + with patch.object(self._device._device, 'read', return_value=[1]) as mock: with patch('serial.Serial.fileno', return_value=1): with patch.object(select, 'select', return_value=[[1], [], []]): ret = self._device.read() From 355ff9cc6d5108555e1e601bd203a7e893ad39e6 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 25 Feb 2021 21:52:27 -0500 Subject: [PATCH 09/10] fix py2 unit tests --- alarmdecoder/util.py | 5 ++++- test/test_devices.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/alarmdecoder/util.py b/alarmdecoder/util.py index 902b21d..2702a13 100644 --- a/alarmdecoder/util.py +++ b/alarmdecoder/util.py @@ -97,7 +97,10 @@ def filter_ad2prot_byte(buf): """ Return the byte sent in back if valid visible terminal characters or line terminators. """ - c = buf[0] + if sys.version_info > (3,): + c = buf[0] + else: + c = ord(buf) if (c == 10 or c == 13): return buf diff --git a/test/test_devices.py b/test/test_devices.py index 17cba61..d418780 100644 --- a/test/test_devices.py +++ b/test/test_devices.py @@ -80,8 +80,11 @@ def test_write_exception(self): def test_read(self): self._device.interface = '/dev/ttyS0' self._device.open(no_reader_thread=True) + side_effect = ["t"] + if sys.version_info > (3,): + side_effect = ["t".encode('utf-8')] - with patch.object(self._device._device, 'read', return_value=[1]) as mock: + with patch.object(self._device._device, 'read', side_effect=side_effect) as mock: with patch('serial.Serial.fileno', return_value=1): with patch.object(select, 'select', return_value=[[1], [], []]): ret = self._device.read() From 83cfd4db26ea37d4286e767bbadd7707b7aeca37 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Sun, 28 Feb 2021 15:26:37 -0500 Subject: [PATCH 10/10] update pkg version 1.13.10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec58ed4..c803fc9 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def readme(): extra_requirements.append('future>=0.14.3') setup(name='alarmdecoder', - version='1.13.9', + version='1.13.10', description='Python interface for the AlarmDecoder (AD2) family ' 'of alarm devices which includes the AD2USB, AD2SERIAL and AD2PI.', long_description=readme(),