From 003fc0308410dd3a6a12841503473ae7fd96792c Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 18 Nov 2021 14:15:00 +0000 Subject: [PATCH 1/2] devlib: Add Target.execute_raw() method Add a Target.execute_raw() method that provides raw stderr and stdout output as bytestring. This allows dealing with binary data in the output of commands and also splitting stdout stderr. --- devlib/target.py | 30 +++++++++++++++++++++++++++++- doc/target.rst | 7 ++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/devlib/target.py b/devlib/target.py index 780008aa9..5a37219bf 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -54,7 +54,9 @@ from devlib.exception import (DevlibTransientError, TargetStableError, TargetNotRespondingError, TimeoutError, TargetTransientError, KernelConfigKeyError, - TargetError, HostError, TargetCalledProcessError) # pylint: disable=redefined-builtin + TargetError, HostError, TargetCalledProcessError, + TargetStableCalledProcessError, TargetTransientCalledProcessError, + ) # pylint: disable=redefined-builtin from devlib.utils.ssh import SshConnection from devlib.utils.android import AdbConnection, AndroidProperties, LogcatMonitor, adb_command, adb_disconnect, INTENT_FLAGS from devlib.utils.misc import memoized, isiterable, convert_new_lines, groupby_value @@ -888,6 +890,32 @@ def _execute(self, command, timeout=None, check_exit_code=True, check_exit_code=check_exit_code, as_root=as_root, strip_colors=strip_colors, will_succeed=will_succeed) + @asyn.asyncf + @call_conn + async def execute_raw(self, command, *, timeout=None, check_exit_code=True, + as_root=False, will_succeed=False, force_locale='C'): + bg = self.background( + command=command, + as_root=as_root, + force_locale=force_locale, + ) + + # TODO: make BackgroundCommand API async-friendly and use that + with bg as bg: + try: + # Timeout on communicate() usually saves a thread + stdout, stderr = bg.communicate(timeout=timeout) + except subprocess.CalledProcessError as e: + if check_exit_code: + if will_succeed: + raise TargetTransientCalledProcessError(*e.args) + else: + raise + else: + return (e.stdout, e.stderr) + else: + return (stdout, stderr) + execute = asyn._AsyncPolymorphicFunction( asyn=_execute_async.asyn, blocking=_execute, diff --git a/doc/target.rst b/doc/target.rst index 17b2bbd3a..84ca36b2a 100644 --- a/doc/target.rst +++ b/doc/target.rst @@ -275,7 +275,7 @@ Target notably paramiko + OpenSSH combination having performance issues when pulling big files from sysfs. -.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed [, force_locale]]]]]]) +.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed [, force_locale, ]]]]]]) Execute the specified command on the target device and return its output. @@ -299,6 +299,11 @@ Target command to get predictable output that can be more safely parsed. If ``None``, no locale is prepended. +.. method:: Target.execute_raw(command [, timeout [, check_exit_code [, as_root [, will_succeed [, force_locale, ]]]]]) + + Same as :meth:`Target.execute` except that it will return a ``tuple(stdout, stderr)`` of + bytestrings. + .. method:: Target.background(command [, stdout [, stderr [, as_root, [, force_locale [, timeout]]]) Execute the command on the target, invoking it via subprocess on the host. From d73e95ffd99812a00b1d82a239f07718a160c77e Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 18 Nov 2021 17:58:48 +0000 Subject: [PATCH 2/2] target: Add Target.read_tree_values_flat(decode=None) parameter Allow not decoding the values and keep them as bytestring, for when e.g. sysfs files have binary content (e.g. exposed from device tree) True decodes the bytestring, None attempts to decode and False does not decode. --- devlib/target.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/devlib/target.py b/devlib/target.py index 5a37219bf..15765a3de 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -1348,7 +1348,7 @@ async def read_tree_tar_flat(self, path, depth=1, check_exit_code=True, # if it is a file and not a folder if content_f: content = content_f.read() - if decode_unicode: + if decode_unicode in (True, None): try: content = content.decode('utf-8').strip() if strip_null_chars: @@ -1362,27 +1362,37 @@ async def read_tree_tar_flat(self, path, depth=1, check_exit_code=True, return result @asyn.asyncf - async def read_tree_values_flat(self, path, depth=1, check_exit_code=True): + async def read_tree_values_flat(self, path, depth=1, check_exit_code=True, decode=None): self.async_manager.track_access( asyn.PathAccess(namespace='target', path=path, mode='r') ) command = 'read_tree_values {} {}'.format(quote(path), depth) output = await self._execute_util.asyn(command, as_root=self.is_rooted, - check_exit_code=check_exit_code) - + check_exit_code=check_exit_code, decode=False) accumulator = defaultdict(list) - for entry in output.strip().split('\n'): - if ':' not in entry: + for entry in output.strip().splitlines(): + if b':' not in entry: continue - path, value = entry.strip().split(':', 1) + path, value = entry.strip().split(b':', 1) accumulator[path].append(value) - result = {k: '\n'.join(v).strip() for k, v in accumulator.items()} + if decode is None: + def do_decode(b): + try: + return b.decode() + except UnicodeDecodeError: + return b + elif decode: + do_decode = lambda b: b.decode() + else: + do_decode = lambda b: b + + result = {k.decode(): do_decode(b'\n'.join(v).strip()) for k, v in accumulator.items()} return result @asyn.asyncf async def read_tree_values(self, path, depth=1, dictcls=dict, - check_exit_code=True, tar=False, decode_unicode=True, + check_exit_code=True, tar=False, decode_unicode=None, strip_null_chars=True): """ Reads the content of all files under a given tree @@ -1400,7 +1410,7 @@ async def read_tree_values(self, path, depth=1, dictcls=dict, :returns: a tree-like dict with the content of files as leafs """ if not tar: - value_map = await self.read_tree_values_flat.asyn(path, depth, check_exit_code) + value_map = await self.read_tree_values_flat.asyn(path, depth, check_exit_code, decode=decode_unicode) else: value_map = await self.read_tree_tar_flat.asyn(path, depth, check_exit_code, decode_unicode, @@ -1436,14 +1446,15 @@ async def _setup_shutils(self): @asyn.asyncf @call_conn - async def _execute_util(self, command, timeout=None, check_exit_code=True, as_root=False): + async def _execute_util(self, command, timeout=None, check_exit_code=True, as_root=False, decode=True): command = '{} sh {} {}'.format(quote(self.busybox), quote(self.shutils), command) - return await self.execute.asyn( - command, + stdout, stderr = await self.execute_raw.asyn( + command=command, timeout=timeout, check_exit_code=check_exit_code, - as_root=as_root + as_root=as_root, ) + return stdout.decode() if decode else stdout async def _extract_archive(self, path, cmd, dest=None): cmd = '{} ' + cmd # busybox