Sender error improvements (#158)

* feat(PAR-6556): not working yet

* feat(PAR-6556): Improve sender feedback

* fix: Improve some messaging

Author: JuanFranDevo

Diff for: CHANGELOG.md

@@ -4,10 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [5.0.0] - 2022-11-29
+## [5.0.0] - 2022-12-02
 ### Added
- * `DevoClientException` refactored for better details extraction
- * In query error detection and feedback through detailed `DevoClientException` 
+ * `DevoClientException` and `DevoSenderException` refactored for better details extraction
+ * In query error detection and feedback through detailed `DevoClientException`
+ * New test selection in `run_tests.py` tool (included and excluded parameter)
 ### Removed
  * Parameter `key` removed from `devo.sender.lookup.Lookup.send_data_line`. The `key` parameter pointed to the value in the `fields` list that was the key of the lookup item. When the value appeared several times in `fields`, the one expected to be the key cannot be identified. This parameter was set as deprecated since version 3.4.0 (2020-08-06) 
 ### Changed

Diff for: devo/api/client.py

@@ -37,8 +37,15 @@
     "no_endpoint": "Endpoint 'address' not found",
     "to_but_no_from": "If you use end dates for the query 'to' it is "
                       "necessary to use start date 'from'",
-    "binary_format_requires_output": "Binary format like `msgpack` and `xls` "
-                                     "requires output parameter"
+    "binary_format_requires_output": "Binary format like `msgpack` and `xls` requires output parameter",
+    "wrong_processor": "processor must be lambda/function or one of the defaults API processors.",
+    "default_keepalive_only": "Mode '%s' always uses default KeepAlive Token",
+    "keepalive_not_supported": "Mode '%s' does not support KeepAlive Token",
+    "stream_mode_not_supported": "Mode '%s' does not support stream mode",
+    "future_queries_not_supported": "Modes 'xls' and 'msgpack' does not support future queries because KeepAlive"
+                                    " tokens are not available for those resonses type",
+    "missing_api_key": "You need a API Key and API secret to make this",
+    "data_query_error": "Error while receiving query data: %s "
 }
 
 DEFAULT_KEEPALIVE_TOKEN = '\n'
@@ -164,8 +171,7 @@ def set_processor(self, processor=None):
             self.proc = "CUSTOM"
             self.processor = processor
         else:
-            raise_exception("processor must be lambda/function or one of"
-                            "the defaults API processors.")
+            raise_exception(ERROR_MSGS["wrong_processor"])
         return True
 
     def set_user(self, user=CLIENT_DEFAULT_USER):
@@ -203,8 +209,7 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
             if keepAliveToken not in [NO_KEEPALIVE_TOKEN,
                                       DEFAULT_KEEPALIVE_TOKEN]:
                 logging.warning(
-                    f"Mode '{self.response}' always uses default KeepAlive"
-                    f" Token")
+                    ERROR_MSGS["default_keepalive_only"] % self.response)
         # In the cases 'csv', 'tsv' you can use any value passed in
         # 'keepAliveToken'.
         elif self.response in ['csv', 'tsv']:
@@ -213,7 +218,7 @@ def set_keepalive_token(self, keepAliveToken=DEFAULT_KEEPALIVE_TOKEN):
             if keepAliveToken not in [NO_KEEPALIVE_TOKEN,
                                       DEFAULT_KEEPALIVE_TOKEN]:
                 logging.warning(
-                    f"Mode '{self.response}' does not support KeepAlive Token")
+                    ERROR_MSGS["keepalive_not_supported"] % self.response)
             self.keepAliveToken = NO_KEEPALIVE_TOKEN
         return True
 
@@ -399,8 +404,7 @@ def query(self, query=None, query_id=None, dates=None,
             if not dates['to']:
                 dates['to'] = "now()"
             if self.config.stream:
-                logging.warning(f"Mode '{self.config.response}' does not"
-                                f" support stream mode")
+                logging.warning(ERROR_MSGS["stream_mode_not_supported"] % self.config.response)
             # If is a future query and response type is 'xls' or 'msgpack'
             # return warning because is not available.
             if self._future_queries_available(self.config.response):
@@ -411,10 +415,7 @@ def query(self, query=None, query_id=None, dates=None,
                 toDate = self._toDate_parser(fromDate, default_to(dates['to']))
 
                 if toDate > default_to("now()"):
-                    raise raise_exception(
-                        "Modes 'xls' and 'msgpack' does not support future "
-                        "queries because KeepAlive tokens are not available "
-                        "for those resonses type")
+                    raise raise_exception(ERROR_MSGS["future_queries_not_supported"])
 
             self.config.stream = False
 
@@ -676,8 +677,7 @@ def _get_sign(self, data, tstamp):
         """
         if not self.auth.get("key", False) \
                 or not self.auth.get("secret", False):
-            raise DevoClientException(("You need a API Key and "
-                                       "API secret to make this"))
+            raise DevoClientException(ERROR_MSGS["missing_api_key"])
         sign = hmac.new(self.auth.get("secret").encode("utf-8"),
                         (self.auth.get("key") + data + tstamp).encode("utf-8"),
                         hashlib.sha256)
@@ -891,7 +891,7 @@ def _error_handler(self, content):
                 code = int(match.group(1))
                 message = match.group(2).strip()
                 raise DevoClientException(
-                    "Error while receiving query data: %s "
+                    ERROR_MSGS["data_query_error"]
                     % message, code=code, cause=error)
             else:
                 return content

Diff for: devo/sender/data.py

@@ -8,6 +8,7 @@
 import sys
 import time
 import zlib
+from enum import Enum
 from pathlib import Path
 from _socket import SHUT_RDWR
 
@@ -21,16 +22,25 @@
 PYPY = hasattr(sys, 'pypy_version_info')
 
 
-ERROR_MSGS = {
-    "no_query": "Error: Not query provided.",
-    "no_auth": "Client dont have key&secret or auth token/jwt",
-    "no_endpoint": "Endpoint 'url' not found"
-}
+class ERROR_MSGS(str, Enum):
+    WRONG_FILE_TYPE = "'%s' is not a valid type to be opened as a file"
+    ADDRESS_TUPLE = "Devo-SenderConfigSSL| address must be a tuple (\"hostname\", int(port))'",
+    WRONG_SSL_CONFIG = "Devo-SenderConfigSSL|Can't create SSL config: %s",
+    CONFIG_FILE_NOT_FOUND = "Error in the configuration, %s is not a file or the path does not exist",
+    CANT_READ_CONFIG_FILE = "Error in the configuration %s can't be read\noriginal error: %s",
+    CONFIG_FILE_PROBLEM = "Error in the configuration, %s problem related to: %s"
 
 
 class DevoSenderException(Exception):
     """ Default Devo Sender Exception """
 
+    def __init__(self, message):
+        self.message = message
+        super().__init__(message)
+
+    def __str__(self):
+        return self.message
+
 
 class SenderConfigSSL:
     """
@@ -55,13 +65,12 @@ class SenderConfigSSL:
         Sender
 
     """
+
     def __init__(self, address=None, key=None, cert=None, chain=None,
                  pkcs=None, sec_level=None, check_hostname=True,
                  verify_mode=None, verify_config=False):
         if not isinstance(address, tuple):
-            raise DevoSenderException(
-                "Devo-SenderConfigSSL| address must be a tuple "
-                "'(\"hostname\", int(port))'")
+            raise DevoSenderException(ERROR_MSGS.ADDRESS_TUPLE)
         try:
             self.address = address
             self.key = key
@@ -75,8 +84,7 @@ def __init__(self, address=None, key=None, cert=None, chain=None,
             self.verify_mode = verify_mode
         except Exception as error:
             raise DevoSenderException(
-                "Devo-SenderConfigSSL|Can't create SSL config: "
-                "%s" % str(error))
+                ERROR_MSGS.WRONG_SSL_CONFIG % str(error)) from error
 
         if self.verify_config:
             self.check_config_files_path()
@@ -94,23 +102,20 @@ def check_config_files_path(self):
         certificates = [self.key, self.chain, self.cert]
         for file in certificates:
             try:
-                if not Path(file).is_file():
+                if not (file.is_file() if isinstance(file, Path) else Path(
+                        file).is_file()):
                     raise DevoSenderException(
-                        "Error in the configuration, "
-                        + file +
-                        " is not a file or the path does not exist")
+                        ERROR_MSGS.CONFIG_FILE_NOT_FOUND % file)
             except IOError as message:
                 if message.errno == errno.EACCES:
                     raise DevoSenderException(
-                        "Error in the configuration "
-                        + file + " can't be read" +
-                        "\noriginal error: " +
-                        str(message))
+                        ERROR_MSGS.CANT_READ_CONFIG_FILE % (
+                        file, str(message))) \
+                        from message
                 else:
                     raise DevoSenderException(
-                        "Error in the configuration, "
-                        + file + " problem related to: " + str(message))
-
+                        ERROR_MSGS.CONFIG_FILE_PROBLEM % (file, str(message))) \
+                        from message
         return True
 
     def check_config_certificate_key(self):
@@ -121,8 +126,8 @@ def check_config_certificate_key(self):
         :return: Boolean true or raises an exception
         """
 
-        with open(self.cert, "rb") as certificate_file, \
-                open(self.key, "rb") as key_file:
+        with open_file(self.cert, mode="rb") as certificate_file, \
+                open_file(self.key, mode="rb") as key_file:
 
             certificate_raw = certificate_file.read()
             key_raw = key_file.read()
@@ -139,7 +144,7 @@ def check_config_certificate_key(self):
             raise DevoSenderException(
                 "Error in the configuration, the key: " + self.key +
                 " is not compatible with the cert: " + self.cert +
-                "\noriginal error: " + str(message))
+                "\noriginal error: " + str(message)) from message
         return True
 
     def check_config_certificate_chain(self):
@@ -149,8 +154,8 @@ def check_config_certificate_chain(self):
 
         :return: Boolean true or raises an exception
         """
-        with open(self.cert, "rb") as certificate_file, \
-                open(self.chain, "rb") as chain_file:
+        with open_file(self.cert, mode="rb") as certificate_file, \
+                open_file(self.chain, mode="rb") as chain_file:
 
             certificate_raw = certificate_file.read()
             chain_raw = chain_file.read()
@@ -169,7 +174,7 @@ def check_config_certificate_chain(self):
             raise DevoSenderException(
                 "Error in config, the chain: " + self.chain +
                 " is not compatible with the certificate: " + self.cert +
-                "\noriginal error: " + str(message))
+                "\noriginal error: " + str(message)) from message
         return True
 
     def check_config_certificate_address(self):
@@ -190,18 +195,18 @@ def check_config_certificate_address(self):
             raise DevoSenderException(
                 "Possible error in config, a timeout could be related " +
                 "to an incorrect address/port: " + str(self.address) +
-                "\noriginal error: " + str(message))
+                "\noriginal error: " + str(message)) from message
         except ConnectionRefusedError as message:
             raise DevoSenderException(
                 "Error in config, incorrect address/port: "
                 + str(self.address) +
-                "\noriginal error: " + str(message))
+                "\noriginal error: " + str(message)) from message
         sock.setblocking(True)
         connection.do_handshake()
         server_chain = connection.get_peer_cert_chain()
         connection.close()
 
-        with open(self.chain, "rb") as chain_file:
+        with open_file(self.chain, mode="rb") as chain_file:
             chain = chain_file.read()
             chain_certs = []
             for _ca in pem.parse(chain):
@@ -226,15 +231,15 @@ def check_config_certificate_address(self):
     def get_common_names(cert_chain, components_type):
         result = set()
         for temp_cert in cert_chain:
-            for key, value in getattr(temp_cert, components_type)()\
+            for key, value in getattr(temp_cert, components_type)() \
                     .get_components():
                 if key.decode("utf-8") == "CN":
                     result.add(value)
         return result
 
     @staticmethod
     def fake_get_peer_cert_chain(chain):
-        with open(chain, "rb") as chain_file:
+        with open_file(chain, mode="rb") as chain_file:
             chain_certs = []
             for _ca in pem.parse(chain_file.read()):
                 chain_certs.append(
@@ -266,11 +271,12 @@ def __init__(self, address=None):
         except Exception as error:
             raise DevoSenderException(
                 "DevoSenderConfigTCP|Can't create TCP config: "
-                "%s" % str(error))
+                "%s" % str(error)) from error
 
 
 class SenderBuffer:
     """Micro class for buffer values"""
+
     def __init__(self):
         self.length = 19500
         self.compression_level = -1
@@ -289,6 +295,7 @@ class Sender(logging.Handler):
     :param debug: For more info in console/logger output
     :param logger: logger. Default sys.console
     """
+
     def __init__(self, config=None, con_type=None,
                  timeout=30, debug=False, logger=None):
         if config is None:
@@ -312,7 +319,7 @@ def __init__(self, config=None, con_type=None,
         self.logger = logger if logger else \
             get_log(handler=get_stream_handler(
                 msg_format='%(asctime)s|%(levelname)s|Devo-Sender|%(message)s')
-                )
+            )
 
         self._sender_config = config
 
@@ -347,7 +354,8 @@ def __connect_tcp_socket(self):
         except socket.error as error:
             self.close()
             raise DevoSenderException(
-                "TCP conn establishment socket error: %s" % str(error))
+                "TCP conn establishment socket error: %s" % str(
+                    error)) from error
 
         self.timestart = int(round(time.time() * 1000))
 
@@ -373,7 +381,7 @@ def __connect_ssl(self):
             self.close()
             raise DevoSenderException(
                 "PFX Certificate read failed: %s" %
-                str(error))
+                str(error)) from error
         try:
             try:
                 if self._sender_config.key is not None \
@@ -418,7 +426,7 @@ def __connect_ssl(self):
             self.close()
             raise DevoSenderException(
                 "SSL conn establishment socket error: %s" %
-                str(error))
+                str(error)) from error
 
     def info(self, msg):
         """
@@ -531,7 +539,7 @@ def __encode_multiline(record):
             record = Sender.__encode_record(record)
             return b'%d %s' % (len(record), record)
         except Exception as error:
-            raise DevoSenderException(error)
+            raise DevoSenderException(error) from error
 
     @staticmethod
     def __encode_record(record):
@@ -590,14 +598,14 @@ def send_raw(self, record, multiline=False, zip=False):
                 except socket.error:
                     self.close()
                     raise DevoSenderException(
-                        "Socket error: %s" % str(socket.error))
+                        "Socket error: %s" % str(socket.error)) from error
                 finally:
                     if self.debug:
                         self.logger.debug('sent|%d|size|%d|msg|%s' %
                                           (sent, len(record), record))
             raise Exception("Socket cant connect: unknown error")
         except Exception as error:
-            raise DevoSenderException(error)
+            raise DevoSenderException(error) from error
 
     @staticmethod
     def compose_mem(tag, **kwargs):
@@ -810,3 +818,20 @@ def emit(self, record):
                       severity=severity)
         except Exception:
             self.handleError(record)
+
+
+def open_file(file, mode='r', encoding='utf-8'):
+    """
+    Helper class to open file whenever is provided as `Path` or `str` type
+    :param file File to open
+    :param mode Opening mode
+    :param encoding Encoding of content
+    """
+    if isinstance(file, Path):
+        return file.open(mode=mode,
+                         encoding=encoding if not mode.endswith('b') else None)
+    elif isinstance(file, str):
+        return open(file, mode=mode,
+                    encoding=encoding if not mode.endswith('b') else None)
+    else:
+        raise DevoSenderException(ERROR_MSGS.WRONG_FILE_TYPE % str(type(file)))