From 7d1f854a09017bdaa79464a9ecf6f78e754406d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Tokodi?= Date: Tue, 19 Sep 2023 20:05:26 +0200 Subject: [PATCH] Update tests and CI to python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update debugger and test262 tests to python3 Disable CI jobs that need ubuntu-18.04 (EOL) until they are updated JerryScript-DCO-1.0-Signed-off-by: Máté Tokodi matet@inf.u-szeged.hu --- .github/workflows/gh-actions.yml | 83 +++++++++++------------- jerry-debugger/jerry_client_main.py | 78 +++++++++++----------- jerry-debugger/jerry_client_rawpacket.py | 2 +- jerry-debugger/jerry_client_websocket.py | 6 +- tools/runners/run-test-suite-test262.py | 2 +- tools/runners/test262-harness.py | 10 ++- 6 files changed, 87 insertions(+), 94 deletions(-) diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 158beb79ab..aaa4c43fd4 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -7,38 +7,37 @@ env: jobs: Checks: - runs-on: ubuntu-18.04 # needed for checker version stability + runs-on: ubuntu-22.04 # needed for checker version stability steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.10' - run: sudo apt update - - run: sudo apt install doxygen clang-format-10 cppcheck pylint python-serial +# TODO: update checkers to current versions available in ubuntu 22.04 +# - run: sudo apt install doxygen clang-format-10 cppcheck pylint python-serial - run: $RUNNER --check-signed-off=gh-actions if: ${{ always() }} - - run: $RUNNER --check-doxygen - if: ${{ always() }} - - run: $RUNNER --check-format - if: ${{ always() }} +# - run: $RUNNER --check-doxygen +# if: ${{ always() }} +# - run: $RUNNER --check-format +# if: ${{ always() }} - run: $RUNNER --check-license if: ${{ always() }} - - run: $RUNNER --check-strings - if: ${{ always() }} - - run: $RUNNER --check-pylint - if: ${{ always() }} - - run: $RUNNER --check-cppcheck - if: ${{ always() }} +# - run: $RUNNER --check-strings +# if: ${{ always() }} +# - run: $RUNNER --check-pylint +# if: ${{ always() }} +# - run: $RUNNER --check-cppcheck +# if: ${{ always() }} Linux_x86-64_Build_Correctness_Debugger_Tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - with: - python-version: '2.7' # needed by jerry-debugger - run: $RUNNER -q --jerry-tests - run: $RUNNER -q --jerry-tests --build-debug - run: $RUNNER -q --jerry-debugger @@ -79,8 +78,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: sudo apt update - - run: sudo apt install python2 - run: $RUNNER --test262 update - uses: actions/upload-artifact@v2 if: success() || failure() @@ -93,8 +90,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: sudo apt update - - run: sudo apt install python2 - run: $RUNNER --test262 update --build-debug --test262-test-list=built-ins,annexB,harness,intl402 - uses: actions/upload-artifact@v2 if: success() || failure() @@ -107,8 +102,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - run: sudo apt update - - run: sudo apt install python2 - run: $RUNNER --test262 update --build-debug --test262-test-list=language - uses: actions/upload-artifact@v2 if: success() || failure() @@ -231,18 +224,19 @@ jobs: $RUNNER -q --jerry-tests --build-debug --buildoptions=--toolchain=cmake/toolchain_linux_aarch64.cmake,--linker-flag=-static - MbedOS_K64F_Build_Test: - runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.8' # needed due to 'intelhex' module - - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa - - run: sudo apt update - - run: sudo apt install gcc-arm-embedded python3-setuptools mercurial - - run: make -f ./targets/os/mbedos/Makefile.travis install - - run: make -f ./targets/os/mbedos/Makefile.travis script +# TODO: update to ubuntu-22.04 +# MbedOS_K64F_Build_Test: +# runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa +# steps: +# - uses: actions/checkout@v2 +# - uses: actions/setup-python@v2 +# with: +# python-version: '3.8' # needed due to 'intelhex' module +# - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa +# - run: sudo apt update +# - run: sudo apt install gcc-arm-embedded python3-setuptools mercurial +# - run: make -f ./targets/os/mbedos/Makefile.travis install +# - run: make -f ./targets/os/mbedos/Makefile.travis script Zephyr_STM32F4_Build_Test: runs-on: ubuntu-latest @@ -265,17 +259,18 @@ jobs: - run: make -f ./targets/os/nuttx/Makefile.travis install-noapt - run: make -f ./targets/os/nuttx/Makefile.travis script - RIOT_STM32F4_Build_Test: - runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa - env: - CC: clang - steps: - - uses: actions/checkout@v2 - - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa - - run: sudo apt update - - run: sudo apt install clang gcc-arm-embedded gcc-multilib - - run: make -f ./targets/os/riot/Makefile.travis install-noapt - - run: make -f ./targets/os/riot/Makefile.travis script +# TODO: update to ubuntu-22.04 +# RIOT_STM32F4_Build_Test: +# runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa +# env: +# CC: clang +# steps: +# - uses: actions/checkout@v2 +# - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa +# - run: sudo apt update +# - run: sudo apt install clang gcc-arm-embedded gcc-multilib +# - run: make -f ./targets/os/riot/Makefile.travis install-noapt +# - run: make -f ./targets/os/riot/Makefile.travis script ESP8266_RTOS_SDK_Build_Test: runs-on: ubuntu-latest diff --git a/jerry-debugger/jerry_client_main.py b/jerry-debugger/jerry_client_main.py index 88c1be61e6..1ae7e49bd3 100644 --- a/jerry-debugger/jerry_client_main.py +++ b/jerry-debugger/jerry_client_main.py @@ -113,6 +113,7 @@ JERRY_DEBUGGER_VALUE_ARRAY = 8 JERRY_DEBUGGER_VALUE_OBJECT = 9 + def arguments_parse(): parser = argparse.ArgumentParser(description="JerryScript debugger client") @@ -290,8 +291,8 @@ def __init__(self, channel): self.src_offset = 0 self.src_offset_diff = 0 self.non_interactive = False - self.current_out = b"" - self.current_log = b"" + self.current_out = "" + self.current_log = "" self.channel = channel config_size = 8 @@ -303,12 +304,12 @@ def __init__(self, channel): # cpointer_size [1] result = self.channel.connect(config_size) - if len(result) != config_size or ord(result[0]) != JERRY_DEBUGGER_CONFIGURATION: + if len(result) != config_size or result[0] != JERRY_DEBUGGER_CONFIGURATION: raise Exception("Unexpected configuration") - self.little_endian = ord(result[1]) & JERRY_DEBUGGER_LITTLE_ENDIAN - self.max_message_size = ord(result[6]) - self.cp_size = ord(result[7]) + self.little_endian = result[1] & JERRY_DEBUGGER_LITTLE_ENDIAN + self.max_message_size = result[6] + self.cp_size = result[7] if self.little_endian: self.byte_order = "<" @@ -396,7 +397,7 @@ def delete(self, args): "to clear all the given breakpoints\n " elif args in ['all', 'pending', 'active']: if args != "pending": - for i in self.active_breakpoint_list.values(): + for i in list(self.active_breakpoint_list.values()): breakpoint = self.active_breakpoint_list[i.active_index] del self.active_breakpoint_list[i.active_index] breakpoint.active_index = -1 @@ -557,6 +558,7 @@ def memstats(self): self._exec_command(JERRY_DEBUGGER_MEMSTATS) def _send_string(self, args, message_type, index=0): + args = args.encode('utf8') # 1: length of type byte # 4: length of an uint32 value @@ -680,7 +682,7 @@ def process_messages(self): if not data: # Break the while loop if there is no more data. return DebuggerAction(DebuggerAction.END, "") - buffer_type = ord(data[0]) + buffer_type = data[0] buffer_size = len(data) -1 logging.debug("Main buffer type: %d, message size: %d", buffer_type, buffer_size) @@ -733,11 +735,8 @@ def process_messages(self): self.prompt = True return DebuggerAction(DebuggerAction.TEXT, result) - elif buffer_type == JERRY_DEBUGGER_EXCEPTION_STR: - self.exception_string += data[1:] - - elif buffer_type == JERRY_DEBUGGER_EXCEPTION_STR_END: - self.exception_string += data[1:] + elif buffer_type in [JERRY_DEBUGGER_EXCEPTION_STR, JERRY_DEBUGGER_EXCEPTION_STR_END]: + self.exception_string += data[1:].decode('utf8') elif buffer_type == JERRY_DEBUGGER_BACKTRACE_TOTAL: total = struct.unpack(self.byte_order + self.idx_format, data[1:])[0] @@ -804,7 +803,7 @@ def process_messages(self): return DebuggerAction(DebuggerAction.TEXT, result) elif buffer_type in [JERRY_DEBUGGER_SCOPE_VARIABLES, JERRY_DEBUGGER_SCOPE_VARIABLES_END]: - self.scope_vars += "".join(data[1:]) + self.scope_vars += data[1:].decode('utf8') if buffer_type == JERRY_DEBUGGER_SCOPE_VARIABLES_END: result = self._process_scope_variables() @@ -860,9 +859,9 @@ def print_source(self, line_num, offset): # pylint: disable=too-many-branches,too-many-locals,too-many-statements def _parse_source(self, data): - source_code = "" - source_code_name = "" - function_name = "" + source_code = b"" + source_code_name = b"" + function_name = b"" stack = [{"line": 1, "column": 1, "name": "", @@ -875,7 +874,7 @@ def _parse_source(self, data): if data is None: return "Error: connection lost during source code receiving" - buffer_type = ord(data[0]) + buffer_type = data[0] buffer_size = len(data) - 1 logging.debug("Parser buffer type: %d, message size: %d", buffer_type, buffer_size) @@ -894,19 +893,20 @@ def _parse_source(self, data): function_name += data[1:] elif buffer_type == JERRY_DEBUGGER_PARSE_FUNCTION: - logging.debug("Source name: %s, function name: %s", source_code_name, function_name) + logging.debug("Source name: %s, function name: %s", source_code_name.decode('utf8'), + function_name.decode('utf8')) position = struct.unpack(self.byte_order + self.idx_format + self.idx_format, data[1: 1 + 4 + 4]) - stack.append({"source": source_code, - "source_name": source_code_name, + stack.append({"source": source_code.decode('utf8'), + "source_name": source_code_name.decode('utf8'), "line": position[0], "column": position[1], - "name": function_name, + "name": function_name.decode('utf8'), "lines": [], "offsets": []}) - function_name = "" + function_name = b"" elif buffer_type in [JERRY_DEBUGGER_BREAKPOINT_LIST, JERRY_DEBUGGER_BREAKPOINT_OFFSET_LIST]: name = "lines" @@ -933,8 +933,8 @@ def _parse_source(self, data): # We know the last item in the list is the general byte code. if not stack: - func_desc["source"] = source_code - func_desc["source_name"] = source_code_name + func_desc["source"] = source_code.decode('utf8') + func_desc["source_name"] = source_code_name.decode('utf8') function = JerryFunction(stack, byte_code_cp, @@ -985,7 +985,7 @@ def _parse_source(self, data): logging.debug("Pending breakpoints available") bp_list = self.pending_breakpoint_list - for breakpoint_index, breakpoint in bp_list.items(): + for breakpoint_index, breakpoint in list(bp_list.items()): source_lines = 0 for src in new_function_list.values(): if (src.source_name == breakpoint.source_name or @@ -1123,19 +1123,19 @@ def _get_breakpoint(self, breakpoint_data): return (function.offsets[nearest_offset], False) def _process_incoming_text(self, buffer_type, data): - message = b"" + message = "" msg_type = buffer_type while True: if buffer_type in [JERRY_DEBUGGER_EVAL_RESULT_END, JERRY_DEBUGGER_OUTPUT_RESULT_END]: - subtype = ord(data[-1]) - message += data[1:-1] + subtype = data[-1] + message += data[1:-1].decode('utf8') break else: - message += data[1:] + message += data[1:].decode('utf8') data = self.channel.get_message(True) - buffer_type = ord(data[0]) + buffer_type = data[0] # Checks if the next frame would be an invalid data frame. # If it is not the message type, or the end type of it, an exception is thrown. if buffer_type not in [msg_type, msg_type + 1]: @@ -1176,17 +1176,17 @@ def _process_scope_variables(self): while buff_pos != buff_size: # Process name - name_length = ord(self.scope_vars[buff_pos:buff_pos + 1]) + name_length = ord(self.scope_vars[buff_pos]) buff_pos += 1 name = self.scope_vars[buff_pos:buff_pos + name_length] buff_pos += name_length # Process type - value_type = ord(self.scope_vars[buff_pos:buff_pos + 1]) + value_type = ord(self.scope_vars[buff_pos]) buff_pos += 1 - value_length = ord(self.scope_vars[buff_pos:buff_pos + 1]) + value_length = ord(self.scope_vars[buff_pos]) buff_pos += 1 value = self.scope_vars[buff_pos: buff_pos + value_length] buff_pos += value_length @@ -1217,16 +1217,16 @@ def _process_scope(self): table = [['level', 'type']] for i, level in enumerate(self.scope_data): - if ord(level) == JERRY_DEBUGGER_SCOPE_WITH: + if level == JERRY_DEBUGGER_SCOPE_WITH: table.append([str(i), 'with']) - elif ord(level) == JERRY_DEBUGGER_SCOPE_GLOBAL: + elif level == JERRY_DEBUGGER_SCOPE_GLOBAL: table.append([str(i), 'global']) - elif ord(level) == JERRY_DEBUGGER_SCOPE_NON_CLOSURE: + elif level == JERRY_DEBUGGER_SCOPE_NON_CLOSURE: # Currently it is only marks the catch closure. table.append([str(i), 'catch']) - elif ord(level) == JERRY_DEBUGGER_SCOPE_LOCAL: + elif level == JERRY_DEBUGGER_SCOPE_LOCAL: table.append([str(i), 'local']) - elif ord(level) == JERRY_DEBUGGER_SCOPE_CLOSURE: + elif level == JERRY_DEBUGGER_SCOPE_CLOSURE: table.append([str(i), 'closure']) else: raise Exception("Unexpected scope chain element") diff --git a/jerry-debugger/jerry_client_rawpacket.py b/jerry-debugger/jerry_client_rawpacket.py index 5c3304edd5..aabad6c64f 100644 --- a/jerry-debugger/jerry_client_rawpacket.py +++ b/jerry-debugger/jerry_client_rawpacket.py @@ -70,7 +70,7 @@ def get_message(self, blocking): while True: if len(self.data_buffer) >= 1: - size = ord(self.data_buffer[0]) + size = self.data_buffer[0] if size == 0: raise Exception("Unexpected data frame") diff --git a/jerry-debugger/jerry_client_websocket.py b/jerry-debugger/jerry_client_websocket.py index fe2c761a93..80f3c3e9c8 100644 --- a/jerry-debugger/jerry_client_websocket.py +++ b/jerry-debugger/jerry_client_websocket.py @@ -92,7 +92,7 @@ def send_message(self, byte_order, packed_data): """ Send message. """ message = struct.pack(byte_order + "BBI", WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT, - WEBSOCKET_FIN_BIT + struct.unpack(byte_order + "B", packed_data[0])[0], + WEBSOCKET_FIN_BIT + struct.unpack(byte_order + "B", packed_data[0].to_bytes())[0], 0) + packed_data[1:] self.__send_data(message) @@ -110,10 +110,10 @@ def get_message(self, blocking): while True: if len(self.data_buffer) >= 2: - if ord(self.data_buffer[0]) != WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT: + if self.data_buffer[0] != WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT: raise Exception("Unexpected data frame") - size = ord(self.data_buffer[1]) + size = self.data_buffer[1] if size == 0 or size >= 126: raise Exception("Unexpected data frame") diff --git a/tools/runners/run-test-suite-test262.py b/tools/runners/run-test-suite-test262.py index 240c863fa7..763e07f93e 100755 --- a/tools/runners/run-test-suite-test262.py +++ b/tools/runners/run-test-suite-test262.py @@ -26,7 +26,7 @@ def get_platform_cmd_prefix(): if sys.platform == 'win32': return ['cmd', '/S', '/C'] - return ['python2'] # The official test262.py isn't python3 compatible, but has python shebang. + return ['python3'] def get_arguments(): diff --git a/tools/runners/test262-harness.py b/tools/runners/test262-harness.py index 33350ed194..f29c7266f1 100755 --- a/tools/runners/test262-harness.py +++ b/tools/runners/test262-harness.py @@ -402,13 +402,11 @@ def open_file(self): text=self.text) def write(self, string): - os.write(self.file_desc, string) + os.write(self.file_desc, string.encode('utf8')) def read(self): - file_desc = file(self.name) - result = file_desc.read() - file_desc.close() - return result + with open(self.name, "r", newline='') as file_desc: + return file_desc.read() def close(self): if not self.is_closed: @@ -495,7 +493,7 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module self.name = name self.full_path = full_path self.strict_mode = strict_mode - with open(self.full_path, "rb") as file_desc: + with open(self.full_path, "r", newline='') as file_desc: self.contents = file_desc.read() test_record = parse_test_record(self.contents, name) self.test = test_record["test"]