Update tests and CI to python3

Update debugger and test262 tests to python3
Disable CI jobs that need ubuntu-18.04 (EOL) until they are updated

JerryScript-DCO-1.0-Signed-off-by: Máté Tokodi [email protected]

.github/workflows/gh-actions.yml

@@ -7,38 +7,37 @@ env:
 
 jobs:
   Checks:
-    runs-on: ubuntu-18.04 # needed for checker version stability
+    runs-on: ubuntu-22.04 # needed for checker version stability
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - run: sudo apt update
-      - run: sudo apt install doxygen clang-format-10 cppcheck pylint python-serial
+# TODO: update checkers to current versions available in ubuntu 22.04
+#      - run: sudo apt install doxygen clang-format-10 cppcheck pylint python-serial
       - run: $RUNNER --check-signed-off=gh-actions
         if: ${{ always() }}
-      - run: $RUNNER --check-doxygen
-        if: ${{ always() }}
-      - run: $RUNNER --check-format
-        if: ${{ always() }}
+#      - run: $RUNNER --check-doxygen
+#        if: ${{ always() }}
+#      - run: $RUNNER --check-format
+#        if: ${{ always() }}
       - run: $RUNNER --check-license
         if: ${{ always() }}
-      - run: $RUNNER --check-strings
-        if: ${{ always() }}
-      - run: $RUNNER --check-pylint
-        if: ${{ always() }}
-      - run: $RUNNER --check-cppcheck
-        if: ${{ always() }}
+#      - run: $RUNNER --check-strings
+#        if: ${{ always() }}
+#      - run: $RUNNER --check-pylint
+#        if: ${{ always() }}
+#      - run: $RUNNER --check-cppcheck
+#        if: ${{ always() }}
 
   Linux_x86-64_Build_Correctness_Debugger_Tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
-          python-version: '2.7' # needed by jerry-debugger
       - run: $RUNNER -q --jerry-tests
       - run: $RUNNER -q --jerry-tests --build-debug
       - run: $RUNNER -q --jerry-debugger
@@ -79,8 +78,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: sudo apt update
-      - run: sudo apt install python2
       - run: $RUNNER --test262 update
       - uses: actions/upload-artifact@v2
         if: success() || failure()
@@ -93,8 +90,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: sudo apt update
-      - run: sudo apt install python2
       - run: $RUNNER --test262 update --build-debug --test262-test-list=built-ins,annexB,harness,intl402
       - uses: actions/upload-artifact@v2
         if: success() || failure()
@@ -107,8 +102,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - run: sudo apt update
-      - run: sudo apt install python2
       - run: $RUNNER --test262 update --build-debug --test262-test-list=language
       - uses: actions/upload-artifact@v2
         if: success() || failure()
@@ -231,18 +224,19 @@ jobs:
           $RUNNER -q --jerry-tests --build-debug
           --buildoptions=--toolchain=cmake/toolchain_linux_aarch64.cmake,--linker-flag=-static
 
-  MbedOS_K64F_Build_Test:
-    runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.8' # needed due to 'intelhex' module
-      - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: sudo apt update
-      - run: sudo apt install gcc-arm-embedded python3-setuptools mercurial
-      - run: make -f ./targets/os/mbedos/Makefile.travis install
-      - run: make -f ./targets/os/mbedos/Makefile.travis script
+# TODO: update to ubuntu-22.04
+#  MbedOS_K64F_Build_Test:
+#    runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa
+#    steps:
+#      - uses: actions/checkout@v2
+#      - uses: actions/setup-python@v2
+#        with:
+#          python-version: '3.8' # needed due to 'intelhex' module
+#      - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
+#      - run: sudo apt update
+#      - run: sudo apt install gcc-arm-embedded python3-setuptools mercurial
+#      - run: make -f ./targets/os/mbedos/Makefile.travis install
+#      - run: make -f ./targets/os/mbedos/Makefile.travis script
 
   Zephyr_STM32F4_Build_Test:
     runs-on: ubuntu-latest
@@ -265,17 +259,18 @@ jobs:
       - run: make -f ./targets/os/nuttx/Makefile.travis install-noapt
       - run: make -f ./targets/os/nuttx/Makefile.travis script
 
-  RIOT_STM32F4_Build_Test:
-    runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa
-    env:
-      CC: clang
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
-      - run: sudo apt update
-      - run: sudo apt install clang gcc-arm-embedded gcc-multilib
-      - run: make -f ./targets/os/riot/Makefile.travis install-noapt
-      - run: make -f ./targets/os/riot/Makefile.travis script
+# TODO: update to ubuntu-22.04
+#  RIOT_STM32F4_Build_Test:
+#    runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa
+#    env:
+#      CC: clang
+#    steps:
+#      - uses: actions/checkout@v2
+#      - run: sudo add-apt-repository ppa:team-gcc-arm-embedded/ppa
+#      - run: sudo apt update
+#      - run: sudo apt install clang gcc-arm-embedded gcc-multilib
+#      - run: make -f ./targets/os/riot/Makefile.travis install-noapt
+#      - run: make -f ./targets/os/riot/Makefile.travis script
 
   ESP8266_RTOS_SDK_Build_Test:
     runs-on: ubuntu-latest

jerry-debugger/jerry_client_main.py

@@ -113,6 +113,7 @@
 JERRY_DEBUGGER_VALUE_ARRAY = 8
 JERRY_DEBUGGER_VALUE_OBJECT = 9
 
+
 def arguments_parse():
     parser = argparse.ArgumentParser(description="JerryScript debugger client")
 
@@ -290,8 +291,8 @@ def __init__(self, channel):
         self.src_offset = 0
         self.src_offset_diff = 0
         self.non_interactive = False
-        self.current_out = b""
-        self.current_log = b""
+        self.current_out = ""
+        self.current_log = ""
         self.channel = channel
 
         config_size = 8
@@ -303,12 +304,12 @@ def __init__(self, channel):
         # cpointer_size [1]
         result = self.channel.connect(config_size)
 
-        if len(result) != config_size or ord(result[0]) != JERRY_DEBUGGER_CONFIGURATION:
+        if len(result) != config_size or result[0] != JERRY_DEBUGGER_CONFIGURATION:
             raise Exception("Unexpected configuration")
 
-        self.little_endian = ord(result[1]) & JERRY_DEBUGGER_LITTLE_ENDIAN
-        self.max_message_size = ord(result[6])
-        self.cp_size = ord(result[7])
+        self.little_endian = result[1] & JERRY_DEBUGGER_LITTLE_ENDIAN
+        self.max_message_size = result[6]
+        self.cp_size = result[7]
 
         if self.little_endian:
             self.byte_order = "<"
@@ -396,7 +397,7 @@ def delete(self, args):
                    "to clear all the given breakpoints\n "
         elif args in ['all', 'pending', 'active']:
             if args != "pending":
-                for i in self.active_breakpoint_list.values():
+                for i in list(self.active_breakpoint_list.values()):
                     breakpoint = self.active_breakpoint_list[i.active_index]
                     del self.active_breakpoint_list[i.active_index]
                     breakpoint.active_index = -1
@@ -557,6 +558,7 @@ def memstats(self):
         self._exec_command(JERRY_DEBUGGER_MEMSTATS)
 
     def _send_string(self, args, message_type, index=0):
+        args = args.encode('latin1')
 
         # 1: length of type byte
         # 4: length of an uint32 value
@@ -680,7 +682,7 @@ def process_messages(self):
             if not data:  # Break the while loop if there is no more data.
                 return DebuggerAction(DebuggerAction.END, "")
 
-            buffer_type = ord(data[0])
+            buffer_type = data[0]
             buffer_size = len(data) -1
 
             logging.debug("Main buffer type: %d, message size: %d", buffer_type, buffer_size)
@@ -734,10 +736,10 @@ def process_messages(self):
                 return DebuggerAction(DebuggerAction.TEXT, result)
 
             elif buffer_type == JERRY_DEBUGGER_EXCEPTION_STR:
-                self.exception_string += data[1:]
+                self.exception_string += data[1:].decode('latin1')
 
             elif buffer_type == JERRY_DEBUGGER_EXCEPTION_STR_END:
-                self.exception_string += data[1:]
+                self.exception_string += data[1:].decode('latin1')
 
             elif buffer_type == JERRY_DEBUGGER_BACKTRACE_TOTAL:
                 total = struct.unpack(self.byte_order + self.idx_format, data[1:])[0]
@@ -804,7 +806,7 @@ def process_messages(self):
                 return DebuggerAction(DebuggerAction.TEXT, result)
 
             elif buffer_type in [JERRY_DEBUGGER_SCOPE_VARIABLES, JERRY_DEBUGGER_SCOPE_VARIABLES_END]:
-                self.scope_vars += "".join(data[1:])
+                self.scope_vars += data[1:].decode()
 
                 if buffer_type == JERRY_DEBUGGER_SCOPE_VARIABLES_END:
                     result = self._process_scope_variables()
@@ -875,7 +877,7 @@ def _parse_source(self, data):
             if data is None:
                 return "Error: connection lost during source code receiving"
 
-            buffer_type = ord(data[0])
+            buffer_type = data[0]
             buffer_size = len(data) - 1
 
             logging.debug("Parser buffer type: %d, message size: %d", buffer_type, buffer_size)
@@ -885,13 +887,13 @@ def _parse_source(self, data):
                 return ""
 
             elif buffer_type in [JERRY_DEBUGGER_SOURCE_CODE, JERRY_DEBUGGER_SOURCE_CODE_END]:
-                source_code += data[1:]
+                source_code += data[1:].decode('latin1')
 
             elif buffer_type in [JERRY_DEBUGGER_SOURCE_CODE_NAME, JERRY_DEBUGGER_SOURCE_CODE_NAME_END]:
-                source_code_name += data[1:]
+                source_code_name += data[1:].decode('latin1')
 
             elif buffer_type in [JERRY_DEBUGGER_FUNCTION_NAME, JERRY_DEBUGGER_FUNCTION_NAME_END]:
-                function_name += data[1:]
+                function_name += data[1:].decode('latin1')
 
             elif buffer_type == JERRY_DEBUGGER_PARSE_FUNCTION:
                 logging.debug("Source name: %s, function name: %s", source_code_name, function_name)
@@ -985,7 +987,7 @@ def _parse_source(self, data):
             logging.debug("Pending breakpoints available")
             bp_list = self.pending_breakpoint_list
 
-            for breakpoint_index, breakpoint in bp_list.items():
+            for breakpoint_index, breakpoint in list(bp_list.items()):
                 source_lines = 0
                 for src in new_function_list.values():
                     if (src.source_name == breakpoint.source_name or
@@ -1123,19 +1125,19 @@ def _get_breakpoint(self, breakpoint_data):
         return (function.offsets[nearest_offset], False)
 
     def _process_incoming_text(self, buffer_type, data):
-        message = b""
+        message = ""
         msg_type = buffer_type
         while True:
             if buffer_type in [JERRY_DEBUGGER_EVAL_RESULT_END,
                                JERRY_DEBUGGER_OUTPUT_RESULT_END]:
-                subtype = ord(data[-1])
-                message += data[1:-1]
+                subtype = data[-1]
+                message += data[1:-1].decode('utf8')
                 break
             else:
-                message += data[1:]
+                message += data[1:].decode('utf8')
 
             data = self.channel.get_message(True)
-            buffer_type = ord(data[0])
+            buffer_type = data[0]
             # Checks if the next frame would be an invalid data frame.
             # If it is not the message type, or the end type of it, an exception is thrown.
             if buffer_type not in [msg_type, msg_type + 1]:
@@ -1176,17 +1178,17 @@ def _process_scope_variables(self):
 
         while buff_pos != buff_size:
             # Process name
-            name_length = ord(self.scope_vars[buff_pos:buff_pos + 1])
+            name_length = int.from_bytes(self.scope_vars[buff_pos:buff_pos + 1].encode('latin1'))
             buff_pos += 1
             name = self.scope_vars[buff_pos:buff_pos + name_length]
             buff_pos += name_length
 
             # Process type
-            value_type = ord(self.scope_vars[buff_pos:buff_pos + 1])
+            value_type = int.from_bytes(self.scope_vars[buff_pos:buff_pos + 1].encode('latin1'))
 
             buff_pos += 1
 
-            value_length = ord(self.scope_vars[buff_pos:buff_pos + 1])
+            value_length = int.from_bytes(self.scope_vars[buff_pos:buff_pos + 1].encode('latin1'))
             buff_pos += 1
             value = self.scope_vars[buff_pos: buff_pos + value_length]
             buff_pos += value_length
@@ -1217,16 +1219,16 @@ def _process_scope(self):
         table = [['level', 'type']]
 
         for i, level in enumerate(self.scope_data):
-            if ord(level) == JERRY_DEBUGGER_SCOPE_WITH:
+            if level == JERRY_DEBUGGER_SCOPE_WITH:
                 table.append([str(i), 'with'])
-            elif ord(level) == JERRY_DEBUGGER_SCOPE_GLOBAL:
+            elif level == JERRY_DEBUGGER_SCOPE_GLOBAL:
                 table.append([str(i), 'global'])
-            elif ord(level) == JERRY_DEBUGGER_SCOPE_NON_CLOSURE:
+            elif level == JERRY_DEBUGGER_SCOPE_NON_CLOSURE:
                 # Currently it is only marks the catch closure.
                 table.append([str(i), 'catch'])
-            elif ord(level) == JERRY_DEBUGGER_SCOPE_LOCAL:
+            elif level == JERRY_DEBUGGER_SCOPE_LOCAL:
                 table.append([str(i), 'local'])
-            elif ord(level) == JERRY_DEBUGGER_SCOPE_CLOSURE:
+            elif level == JERRY_DEBUGGER_SCOPE_CLOSURE:
                 table.append([str(i), 'closure'])
             else:
                 raise Exception("Unexpected scope chain element")

jerry-debugger/jerry_client_rawpacket.py

@@ -70,7 +70,7 @@ def get_message(self, blocking):
 
         while True:
             if len(self.data_buffer) >= 1:
-                size = ord(self.data_buffer[0])
+                size = self.data_buffer[0]
                 if size == 0:
                     raise Exception("Unexpected data frame")
 

jerry-debugger/jerry_client_websocket.py

@@ -92,7 +92,7 @@ def send_message(self, byte_order, packed_data):
         """ Send message. """
         message = struct.pack(byte_order + "BBI",
                               WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT,
-                              WEBSOCKET_FIN_BIT + struct.unpack(byte_order + "B", packed_data[0])[0],
+                              WEBSOCKET_FIN_BIT + struct.unpack(byte_order + "B", packed_data[0].to_bytes())[0],
                               0) + packed_data[1:]
 
         self.__send_data(message)
@@ -110,10 +110,10 @@ def get_message(self, blocking):
 
         while True:
             if len(self.data_buffer) >= 2:
-                if ord(self.data_buffer[0]) != WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT:
+                if self.data_buffer[0] != WEBSOCKET_BINARY_FRAME | WEBSOCKET_FIN_BIT:
                     raise Exception("Unexpected data frame")
 
-                size = ord(self.data_buffer[1])
+                size = self.data_buffer[1]
                 if size == 0 or size >= 126:
                     raise Exception("Unexpected data frame")
 

tools/runners/run-test-suite-test262.py

@@ -26,7 +26,7 @@
 def get_platform_cmd_prefix():
     if sys.platform == 'win32':
         return ['cmd', '/S', '/C']
-    return ['python2']  # The official test262.py isn't python3 compatible, but has python shebang.
+    return ['python3']
 
 
 def get_arguments():

tools/runners/test262-harness.py

@@ -402,13 +402,11 @@ def open_file(self):
             text=self.text)
 
     def write(self, string):
-        os.write(self.file_desc, string)
+        os.write(self.file_desc, string.encode('utf8'))
 
     def read(self):
-        file_desc = file(self.name)
-        result = file_desc.read()
-        file_desc.close()
-        return result
+        with open(self.name, "r", newline='') as file_desc:
+            return file_desc.read()
 
     def close(self):
         if not self.is_closed:
@@ -495,7 +493,7 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module
         self.name = name
         self.full_path = full_path
         self.strict_mode = strict_mode
-        with open(self.full_path, "rb") as file_desc:
+        with open(self.full_path, "r", newline='') as file_desc:
             self.contents = file_desc.read()
         test_record = parse_test_record(self.contents, name)
         self.test = test_record["test"]