From c22f4c3437b09d2404c9b3d96c3a58f6d8d68cf6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 15 Jun 2018 13:08:32 +0200 Subject: [PATCH 01/30] Add single sensor mode (sensorcontainer.single_sensor_mode config option) to the sensor container. When running in this mode, it's assumed that sensor process life cycle and failover is handled by an external service so the sensor process container exits immediately when a server crashes. This is useful in scenarios where sensor process life cycle and failover is handled by external service such as kubernetes. --- st2reactor/st2reactor/cmd/sensormanager.py | 12 +++++++++- st2reactor/st2reactor/container/manager.py | 24 ++++++++++++++----- .../st2reactor/container/process_container.py | 24 ++++++++++++++++--- st2reactor/st2reactor/sensor/config.py | 24 +++++++++++++++---- 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/st2reactor/st2reactor/cmd/sensormanager.py b/st2reactor/st2reactor/cmd/sensormanager.py index 0fe192196d..1537c7b8e5 100644 --- a/st2reactor/st2reactor/cmd/sensormanager.py +++ b/st2reactor/st2reactor/cmd/sensormanager.py @@ -14,9 +14,12 @@ # limitations under the License. from __future__ import absolute_import + import os import sys +from oslo_config import cfg + from st2common import log as logging from st2common.logging.misc import get_logger_name_for_module from st2common.service_setup import setup as common_setup @@ -50,8 +53,15 @@ def _teardown(): def main(): try: _setup() + + if cfg.CONF.sensorcontainer.single_sensor_mode and not cfg.CONF.sensor_ref: + raise ValueError('--sensor-ref argument must be provided when running in single ' + 'sensor mode') + sensors_partitioner = get_sensors_partitioner() - container_manager = SensorContainerManager(sensors_partitioner=sensors_partitioner) + single_sensor_mode = cfg.CONF.sensorcontainer.single_sensor_mode + container_manager = SensorContainerManager(sensors_partitioner=sensors_partitioner, + single_sensor_mode=single_sensor_mode) return container_manager.run_sensors() except SystemExit as exit_code: return exit_code diff --git a/st2reactor/st2reactor/container/manager.py b/st2reactor/st2reactor/container/manager.py index d9f4e3071e..66306821a9 100644 --- a/st2reactor/st2reactor/container/manager.py +++ b/st2reactor/st2reactor/container/manager.py @@ -27,19 +27,27 @@ LOG = logging.getLogger(__name__) +__all__ = [ + 'SensorContainerManager' +] + class SensorContainerManager(object): - def __init__(self, sensors_partitioner): + def __init__(self, sensors_partitioner, single_sensor_mode=False): + if not sensors_partitioner: + raise ValueError('sensors_partitioner should be non-None.') + + self._sensors_partitioner = sensors_partitioner + self._single_sensor_mode = single_sensor_mode + self._sensor_container = None + self._container_thread = None + self._sensors_watcher = SensorWatcher(create_handler=self._handle_create_sensor, update_handler=self._handle_update_sensor, delete_handler=self._handle_delete_sensor, queue_suffix='sensor_container') - self._container_thread = None - if not sensors_partitioner: - raise ValueError('sensors_partitioner should be non-None.') - self._sensors_partitioner = sensors_partitioner def run_sensors(self): """ @@ -61,10 +69,14 @@ def run_sensors(self): def _spin_container_and_wait(self, sensors): try: - self._sensor_container = ProcessSensorContainer(sensors=sensors) + self._sensor_container = ProcessSensorContainer( + sensors=sensors, + single_sensor_mode=self._single_sensor_mode) self._container_thread = eventlet.spawn(self._sensor_container.run) + LOG.debug('Starting sensor CUD watcher...') self._sensors_watcher.start() + exit_code = self._container_thread.wait() LOG.error('Process container quit with exit_code %d.', exit_code) LOG.error('(PID:%s) SensorContainer stopped.', os.getpid()) diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 39cb4fea24..33aaec95cf 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -77,7 +77,7 @@ class ProcessSensorContainer(object): Sensor container which runs sensors in a separate process. """ - def __init__(self, sensors, poll_interval=5, dispatcher=None): + def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatcher=None): """ :param sensors: A list of sensor dicts. :type sensors: ``list`` of ``dict`` @@ -86,6 +86,12 @@ def __init__(self, sensors, poll_interval=5, dispatcher=None): :type poll_interval: ``float`` """ self._poll_interval = poll_interval + self._single_sensor_mode = single_sensor_mode + + if self._single_sensor_mode: + # For more immediate feedback we use lower poll interval when running in single sensor + # mode + self._poll_interval = 1 self._sensors = {} # maps sensor_id -> sensor object self._processes = {} # maps sensor_id -> sensor process @@ -95,6 +101,7 @@ def __init__(self, sensors, poll_interval=5, dispatcher=None): self._dispatcher = dispatcher self._stopped = False + self._exit_code = None # exit code with which this process should exit sensors = sensors or [] for sensor_obj in sensors: @@ -144,8 +151,10 @@ def run(self): return FAILURE_EXIT_CODE self._stopped = True - LOG.error('Process container quit. It shouldn\'t.') - return SUCCESS_EXIT_CODE + LOG.error('Process container stopped.') + + exit_code = self._exit_code or SUCCESS_EXIT_CODE + return exit_code def _poll_sensors_for_results(self, sensor_ids): """ @@ -390,6 +399,15 @@ def _respawn_sensor(self, sensor_id, sensor, exit_code): """ extra = {'sensor_id': sensor_id, 'sensor': sensor} + if self._single_sensor_mode: + # In single sensor mode we want to exit immediately on failure + LOG.info('Not respawning a sensor since running in single sensor mode', + extra=extra) + + self._stopped = True + self._exit_code = exit_code + return + if self._stopped: LOG.debug('Stopped, not respawning a dead sensor', extra=extra) return diff --git a/st2reactor/st2reactor/sensor/config.py b/st2reactor/st2reactor/sensor/config.py index de8376c25f..6e3fa37c42 100644 --- a/st2reactor/st2reactor/sensor/config.py +++ b/st2reactor/st2reactor/sensor/config.py @@ -52,6 +52,7 @@ def _register_sensor_container_opts(ignore_errors=False): st2cfg.do_register_opts(logging_opts, group='sensorcontainer', ignore_errors=ignore_errors) + # Partitioning options partition_opts = [ cfg.StrOpt( 'sensor_node_name', default='sensornode1', @@ -63,13 +64,28 @@ def _register_sensor_container_opts(ignore_errors=False): help='Provider of sensor node partition config.') ] + # Other options st2cfg.do_register_opts(partition_opts, group='sensorcontainer', ignore_errors=ignore_errors) - sensor_test_opt = cfg.StrOpt( - 'sensor-ref', - help='Only run sensor with the provided reference. Value is of the form pack.sensor-name.') + other_opts = [ + cfg.BoolOpt( + 'single_sensor_mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] + + st2cfg.do_register_opts(other_opts, group='sensorcontainer', ignore_errors=ignore_errors) + + # CLI options + cli_opts = [ + cfg.StrOpt( + 'sensor-ref', + help='Only run sensor with the provided reference. Value is of the form ' + '. (e.g. linux.FileWatchSensor).') + ] - st2cfg.do_register_cli_opts(sensor_test_opt, ignore_errors=ignore_errors) + st2cfg.do_register_cli_opts(cli_opts, ignore_errors=ignore_errors) register_opts(ignore_errors=True) From 8384eecc01294e87f63b54ebd7d3416371d801bf Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 15 Jun 2018 13:12:43 +0200 Subject: [PATCH 02/30] Correctly propagate sensor exit code all the way to the parent process when running in single sensor mode. --- st2reactor/st2reactor/container/manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/st2reactor/st2reactor/container/manager.py b/st2reactor/st2reactor/container/manager.py index 66306821a9..7ba5e8628d 100644 --- a/st2reactor/st2reactor/container/manager.py +++ b/st2reactor/st2reactor/container/manager.py @@ -65,9 +65,13 @@ def run_sensors(self): LOG.info('(PID:%s) SensorContainer started.', os.getpid()) self._setup_sigterm_handler() - self._spin_container_and_wait(sensors_to_run) + + exit_code = self._spin_container_and_wait(sensors_to_run) + return exit_code def _spin_container_and_wait(self, sensors): + exit_code = 0 + try: self._sensor_container = ProcessSensorContainer( sensors=sensors, @@ -90,7 +94,9 @@ def _spin_container_and_wait(self, sensors): eventlet.kill(self._container_thread) self._container_thread = None - return 0 + return exit_code + + return exit_code def _setup_sigterm_handler(self): From 231f7abc1b6c137f5d7191f79a1d1047079b34c2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 15 Jun 2018 14:25:47 +0200 Subject: [PATCH 03/30] Log a message when running in a single sensor mode and a single sensor partitioner is used. --- st2reactor/st2reactor/container/partitioner_lookup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/st2reactor/st2reactor/container/partitioner_lookup.py b/st2reactor/st2reactor/container/partitioner_lookup.py index ca2c1c7ba3..e291254f85 100644 --- a/st2reactor/st2reactor/container/partitioner_lookup.py +++ b/st2reactor/st2reactor/container/partitioner_lookup.py @@ -41,16 +41,19 @@ def get_sensors_partitioner(): if cfg.CONF.sensor_ref: + LOG.info('Running in single sensor mode, using a single sensor partitioner...') return SingleSensorPartitioner(sensor_ref=cfg.CONF.sensor_ref) + partition_provider_config = copy.copy(cfg.CONF.sensorcontainer.partition_provider) partition_provider = partition_provider_config.pop('name') sensor_node_name = cfg.CONF.sensorcontainer.sensor_node_name provider = PROVIDERS.get(partition_provider.lower(), None) - LOG.info('Using partitioner %s with sensornode %s.', partition_provider, sensor_node_name) if not provider: - raise SensorPartitionerNotSupportedException( - 'Partition provider %s not found.' % partition_provider) + raise SensorPartitionerNotSupportedException('Partition provider %s not found.' % + (partition_provider)) + + LOG.info('Using partitioner %s with sensornode %s.', partition_provider, sensor_node_name) # pass in extra config with no analysis return provider(sensor_node_name=sensor_node_name, **partition_provider_config) From 4ec2334ac11dd7791ced1062c339a5f134d12b53 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 15 Jun 2018 14:47:03 +0200 Subject: [PATCH 04/30] Update tests config. --- st2reactor/st2reactor/sensor/config.py | 2 +- st2tests/st2tests/config.py | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/st2reactor/st2reactor/sensor/config.py b/st2reactor/st2reactor/sensor/config.py index 6e3fa37c42..a1842ec83e 100644 --- a/st2reactor/st2reactor/sensor/config.py +++ b/st2reactor/st2reactor/sensor/config.py @@ -64,9 +64,9 @@ def _register_sensor_container_opts(ignore_errors=False): help='Provider of sensor node partition config.') ] - # Other options st2cfg.do_register_opts(partition_opts, group='sensorcontainer', ignore_errors=ignore_errors) + # Other options other_opts = [ cfg.BoolOpt( 'single_sensor_mode', default=False, diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 6222e95567..94ce970701 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -279,11 +279,26 @@ def _register_sensor_container_opts(): _register_opts(partition_opts, group='sensorcontainer') - sensor_test_opt = cfg.StrOpt( - 'sensor-ref', - help='Only run sensor with the provided reference. Value is of the form pack.sensor-name.') + # Other options + other_opts = [ + cfg.BoolOpt( + 'single_sensor_mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] + + _register_opts(other_opts, group='sensorcontainer') + + # CLI options + cli_opts = [ + cfg.StrOpt( + 'sensor-ref', + help='Only run sensor with the provided reference. Value is of the form ' + '. (e.g. linux.FileWatchSensor).') + ] - _register_cli_opts([sensor_test_opt]) + _register_cli_opts(cli_opts) def _register_opts(opts, group=None): From dfc57f852436109609adcd7d513762a75ef3e641 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 15 Jun 2018 14:51:01 +0200 Subject: [PATCH 05/30] Also allow user to enable sensor container single user mode by passing --single-user-mode flag to the process. --- st2reactor/st2reactor/cmd/sensormanager.py | 6 ++++-- st2reactor/st2reactor/sensor/config.py | 7 ++++++- st2tests/st2tests/config.py | 7 ++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/st2reactor/st2reactor/cmd/sensormanager.py b/st2reactor/st2reactor/cmd/sensormanager.py index 1537c7b8e5..f990450b33 100644 --- a/st2reactor/st2reactor/cmd/sensormanager.py +++ b/st2reactor/st2reactor/cmd/sensormanager.py @@ -54,12 +54,14 @@ def main(): try: _setup() - if cfg.CONF.sensorcontainer.single_sensor_mode and not cfg.CONF.sensor_ref: + single_sensor_mode = (cfg.CONF.single_sensor_mode or + cfg.CONF.sensorcontainer.single_sensor_mode) + + if single_sensor_mode and not cfg.CONF.sensor_ref: raise ValueError('--sensor-ref argument must be provided when running in single ' 'sensor mode') sensors_partitioner = get_sensors_partitioner() - single_sensor_mode = cfg.CONF.sensorcontainer.single_sensor_mode container_manager = SensorContainerManager(sensors_partitioner=sensors_partitioner, single_sensor_mode=single_sensor_mode) return container_manager.run_sensors() diff --git a/st2reactor/st2reactor/sensor/config.py b/st2reactor/st2reactor/sensor/config.py index a1842ec83e..90d9d5e974 100644 --- a/st2reactor/st2reactor/sensor/config.py +++ b/st2reactor/st2reactor/sensor/config.py @@ -82,7 +82,12 @@ def _register_sensor_container_opts(ignore_errors=False): cfg.StrOpt( 'sensor-ref', help='Only run sensor with the provided reference. Value is of the form ' - '. (e.g. linux.FileWatchSensor).') + '. (e.g. linux.FileWatchSensor).'), + cfg.BoolOpt( + 'single-sensor-mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') ] st2cfg.do_register_cli_opts(cli_opts, ignore_errors=ignore_errors) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 94ce970701..99f4acbda4 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -295,7 +295,12 @@ def _register_sensor_container_opts(): cfg.StrOpt( 'sensor-ref', help='Only run sensor with the provided reference. Value is of the form ' - '. (e.g. linux.FileWatchSensor).') + '. (e.g. linux.FileWatchSensor).'), + cfg.BoolOpt( + 'single-sensor-mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') ] _register_cli_opts(cli_opts) From cf5e7d2ee2c517319eb66842118b0d7dbffcc0e9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 10:54:06 +0200 Subject: [PATCH 06/30] Add tests for sensor container single sensor mode. --- contrib/hello_st2/config.yaml | 0 .../integration/test_sensor_container.py | 63 ++++++++++++++++--- 2 files changed, 53 insertions(+), 10 deletions(-) delete mode 100644 contrib/hello_st2/config.yaml diff --git a/contrib/hello_st2/config.yaml b/contrib/hello_st2/config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 98212d3afe..52ad83597a 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -35,14 +35,20 @@ ] BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + ST2_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests.conf') ST2_CONFIG_PATH = os.path.abspath(ST2_CONFIG_PATH) + BINARY = os.path.join(BASE_DIR, '../../../st2reactor/bin/st2sensorcontainer') BINARY = os.path.abspath(BINARY) -CMD = [BINARY, '--config-file', ST2_CONFIG_PATH, '--sensor-ref=examples.SamplePollingSensor'] +PACKS_BASE_PATH = os.path.join(BASE_DIR, '../../../contrib') + +DEFAULT_CMD = [BINARY, '--config-file', ST2_CONFIG_PATH, + '--sensor-ref=examples.SamplePollingSensor'] -@unittest2.skipIf(True, 'Skipped until we improve integration tests setup') + +#@unittest2.skipIf(True, 'Skipped until we improve integration tests setup') class SensorContainerTestCase(IntegrationTestCase): """ Note: For those tests MongoDB must be running, virtualenv must exist for @@ -54,7 +60,6 @@ class SensorContainerTestCase(IntegrationTestCase): @classmethod def setUpClass(cls): super(SensorContainerTestCase, cls).setUpClass() - return st2tests.config.parse_args() @@ -65,10 +70,10 @@ def setUpClass(cls): username=username, password=password, ensure_indexes=False) # Register sensors - register_sensors(packs_base_paths=['/opt/stackstorm/packs'], use_pack_cache=False) + register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False) # Create virtualenv for examples pack - virtualenv_path = '/opt/stackstorm/virtualenvs/examples' + virtualenv_path = '/tmp/virtualenvs/examples' cmd = ['virtualenv', '--system-site-packages', virtualenv_path] run_command(cmd=cmd) @@ -84,7 +89,7 @@ def test_child_processes_are_killed_on_sigint(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) self.assertEqual(len(children_pp), 1) # Send SIGINT @@ -109,7 +114,7 @@ def test_child_processes_are_killed_on_sigterm(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) self.assertEqual(len(children_pp), 1) # Send SIGTERM @@ -134,7 +139,7 @@ def test_child_processes_are_killed_on_sigkill(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) self.assertEqual(len(children_pp), 1) # Send SIGKILL @@ -149,8 +154,46 @@ def test_child_processes_are_killed_on_sigkill(self): self.remove_process(process=process) - def _start_sensor_container(self): - process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + def test_single_sensor_mode(self): + # 1. --sensor-ref not provided + cmd = [BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode'] + + process = self._start_sensor_container(cmd=cmd) + pp = psutil.Process(process.pid) + + # Give it some time to start up + eventlet.sleep(3) + + stdout = process.stdout.read() + self.assertTrue(('--sensor-ref argument must be provided when running in single sensor ' + 'mode') in stdout) + self.assertProcessExited(proc=pp) + self.remove_process(process=process) + + # 2. sensor ref provided + cmd = [BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode', + '--sensor-ref=examples.SampleSensorExit'] + + process = self._start_sensor_container(cmd=cmd) + pp = psutil.Process(process.pid) + + # Give it some time to start up + eventlet.sleep(8) + + # Container should exit and not respawn a sensor in single sensor mode + stdout = process.stdout.read() + + self.assertTrue('Process for sensor examples.SampleSensorExit has exited with code 110') + self.assertTrue('Not respawning a sensor since running in single sensor mode') + self.assertTrue('Process container quit with exit_code 110.') + + eventlet.sleep(2) + self.assertProcessExited(proc=pp) + + self.remove_process(process=process) + + def _start_sensor_container(self, cmd=DEFAULT_CMD): + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, preexec_fn=os.setsid) self.add_process(process=process) return process From f732cfc9764b95af30a92601edf49ed3fa7db7d6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 10:59:55 +0200 Subject: [PATCH 07/30] Fix lint. --- st2reactor/tests/integration/test_sensor_container.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 52ad83597a..af8dd6a819 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -16,7 +16,6 @@ from __future__ import absolute_import import os import signal -import unittest2 import psutil import eventlet @@ -48,7 +47,7 @@ '--sensor-ref=examples.SamplePollingSensor'] -#@unittest2.skipIf(True, 'Skipped until we improve integration tests setup') +# @unittest2.skipIf(True, 'Skipped until we improve integration tests setup') class SensorContainerTestCase(IntegrationTestCase): """ Note: For those tests MongoDB must be running, virtualenv must exist for From c7bc2f53d2cab789310f8068200c0ab0c06477f5 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 11:19:48 +0200 Subject: [PATCH 08/30] Re-generate sample configs. --- conf/st2.conf.sample | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index d71ce0a595..73fdf0d8bd 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -257,6 +257,8 @@ draft = http://json-schema.org/draft-04/schema# [sensorcontainer] # Provider of sensor node partition config. partition_provider = {'name': 'default'} +# Run in a single sensor mode where parent process exits when a sensor crashes / dies. This is useful in environments where partitioning, sensor process life cycle and failover is handled by a 3rd party service such as kubernetes. +single_sensor_mode = False # location of the logging.conf file logging = conf/logging.sensorcontainer.conf # name of the sensor node. From 2f6b8c19eaaf9f58b35330b7b22a4a759699d596 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 11:23:58 +0200 Subject: [PATCH 09/30] Use longer sleep for more predictable tests. --- st2reactor/tests/integration/test_sensor_container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index af8dd6a819..db4c39a616 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -133,7 +133,7 @@ def test_child_processes_are_killed_on_sigkill(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) @@ -161,7 +161,7 @@ def test_single_sensor_mode(self): pp = psutil.Process(process.pid) # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) stdout = process.stdout.read() self.assertTrue(('--sensor-ref argument must be provided when running in single sensor ' From 1deb6bacfe43917b4d6dc342fc3036e7f0e311be Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 11:27:04 +0200 Subject: [PATCH 10/30] Fix tests so they also work under Python 3. --- st2reactor/tests/integration/test_sensor_container.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index db4c39a616..f83c67f406 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -164,8 +164,8 @@ def test_single_sensor_mode(self): eventlet.sleep(4) stdout = process.stdout.read() - self.assertTrue(('--sensor-ref argument must be provided when running in single sensor ' - 'mode') in stdout) + self.assertTrue((b'--sensor-ref argument must be provided when running in single sensor ' + b'mode') in stdout) self.assertProcessExited(proc=pp) self.remove_process(process=process) @@ -182,9 +182,9 @@ def test_single_sensor_mode(self): # Container should exit and not respawn a sensor in single sensor mode stdout = process.stdout.read() - self.assertTrue('Process for sensor examples.SampleSensorExit has exited with code 110') - self.assertTrue('Not respawning a sensor since running in single sensor mode') - self.assertTrue('Process container quit with exit_code 110.') + self.assertTrue(b'Process for sensor examples.SampleSensorExit has exited with code 110') + self.assertTrue(b'Not respawning a sensor since running in single sensor mode') + self.assertTrue(b'Process container quit with exit_code 110.') eventlet.sleep(2) self.assertProcessExited(proc=pp) From d74c9d4c06a25082df4285957d8c2659960f0fe9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 11:57:36 +0200 Subject: [PATCH 11/30] Run GC tests under Python 3. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e18256d096..207199ee46 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ deps = virtualenv commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ - nosetests --with-timer --rednose -sv st2reactor/tests/integration/ --ignore-files=test_garbage_collector.* + nosetests --with-timer --rednose -sv st2reactor/tests/integration/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ nosetests --with-timer --rednose -sv st2common/tests/unit/ From a16938a88c6f4a8dc110465d27fceef1c7d90669 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 14:13:25 +0200 Subject: [PATCH 12/30] Split tox Python 3 task into two tasks - one for unit and one for integration tests and correctly set up the environment on Travis. --- .travis.yml | 4 +++- Makefile | 11 +++++++++-- tox.ini | 15 +++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6dad816791..e3083f210c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,8 @@ matrix: python: 2.7 - env: TASK="compilepy3 ci-py3-unit" CACHE_NAME=py3 python: 3.6 + - env: TASK="ci-py3-integration" CACHE_NAME=py3 + python: 3.6 addons: apt: sources: @@ -55,7 +57,7 @@ before_install: - sudo pip install --upgrade "virtualenv==15.1.0" install: - - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then pip install "tox==3.0.0"; else make requirements; fi + - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then sudo .circle/add-itest-user.sh; fi diff --git a/Makefile b/Makefile index 0ef1df486d..0c963f6afe 100644 --- a/Makefile +++ b/Makefile @@ -704,11 +704,18 @@ ci: ci-checks ci-unit ci-integration ci-mistral ci-packs-tests ci-checks: compile .generated-files-check .pylint .flake8 .bandit .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .PHONY: ci-py3-unit -ci-py3-unit: +ci-py3-unit: .ci-prepare-integration @echo @echo "==================== ci-py3-unit ====================" @echo - tox -e py36 -vv + tox -e py36-unit -vv + +.PHONY: ci-py3-integration +ci-py3-integration: .ci-prepare-integration + @echo + @echo "==================== ci-py3-integration ====================" + @echo + tox -e py36-integration -vv .PHONY: .rst-check .rst-check: diff --git a/tox.ini b/tox.ini index 207199ee46..eb6f33c6e3 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = nosetests -sv st2reactor/tests/unit nosetests -sv st2tests/tests/unit -[testenv:py36] +[testenv:py36-unit] basepython = python3.6 setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} @@ -38,7 +38,6 @@ deps = virtualenv commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ - nosetests --with-timer --rednose -sv st2reactor/tests/integration/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ nosetests --with-timer --rednose -sv st2common/tests/unit/ @@ -53,6 +52,18 @@ commands = nosetests --with-timer --rednose -sv contrib/runners/python_runner/tests/unit/ contrib/runners/python_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/windows_runner/tests/unit/ +[testenv:py36-integration] +basepython = python3.6 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner + VIRTUALENV_DIR = {envdir} +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + [testenv:venv] commands = {posargs} From 93b5e115d1dbd672adddfc499082d38e6d63c52e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 14:30:03 +0200 Subject: [PATCH 13/30] Update make targets, increase sleep. --- Makefile | 4 ++-- st2reactor/tests/integration/test_sensor_container.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0c963f6afe..40a79e9b22 100644 --- a/Makefile +++ b/Makefile @@ -704,14 +704,14 @@ ci: ci-checks ci-unit ci-integration ci-mistral ci-packs-tests ci-checks: compile .generated-files-check .pylint .flake8 .bandit .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .PHONY: ci-py3-unit -ci-py3-unit: .ci-prepare-integration +ci-py3-unit: @echo @echo "==================== ci-py3-unit ====================" @echo tox -e py36-unit -vv .PHONY: ci-py3-integration -ci-py3-integration: .ci-prepare-integration +ci-py3-integration: @echo @echo "==================== ci-py3-integration ====================" @echo diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index f83c67f406..7de14f337a 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -80,7 +80,7 @@ def test_child_processes_are_killed_on_sigint(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(5) # Assert process has started and is running self.assertProcessIsRunning(process=process) @@ -96,7 +96,7 @@ def test_child_processes_are_killed_on_sigint(self): # SIGINT causes graceful shutdown so give it some time to gracefuly shut down the sensor # child processes - eventlet.sleep(PROCESS_EXIT_TIMEOUT + 1) + eventlet.sleep(PROCESS_EXIT_TIMEOUT + 2) # Verify parent and children processes have exited self.assertProcessExited(proc=pp) From c92ac57576256555783ddbba7e0d985fc638bf80 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 14:50:34 +0200 Subject: [PATCH 14/30] Use the same python binary which is used for running the tests. --- .../tests/integration/test_sensor_container.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 7de14f337a..60b3439f82 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -14,7 +14,9 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import signal import psutil @@ -38,13 +40,20 @@ ST2_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests.conf') ST2_CONFIG_PATH = os.path.abspath(ST2_CONFIG_PATH) +PYTHON_BINARY = sys.executable + BINARY = os.path.join(BASE_DIR, '../../../st2reactor/bin/st2sensorcontainer') BINARY = os.path.abspath(BINARY) PACKS_BASE_PATH = os.path.join(BASE_DIR, '../../../contrib') -DEFAULT_CMD = [BINARY, '--config-file', ST2_CONFIG_PATH, - '--sensor-ref=examples.SamplePollingSensor'] +DEFAULT_CMD = [ + PYTHON_BINARY, + BINARY, + '--config-file', + ST2_CONFIG_PATH, + '--sensor-ref=examples.SamplePollingSensor' +] # @unittest2.skipIf(True, 'Skipped until we improve integration tests setup') From 74e35439aae7340118205a2f6a110bd7742ca42b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 15:02:39 +0200 Subject: [PATCH 15/30] Use the same binary which is used for the tests process. --- .../tests/integration/test_garbage_collector.py | 12 ++++++++++-- .../tests/integration/test_sensor_container.py | 13 ++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 18091e851e..698ad423be 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -14,7 +14,9 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import signal import datetime @@ -43,14 +45,20 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + ST2_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests.conf') ST2_CONFIG_PATH = os.path.abspath(ST2_CONFIG_PATH) + INQUIRY_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests2.conf') INQUIRY_CONFIG_PATH = os.path.abspath(INQUIRY_CONFIG_PATH) + +PYTHON_BINARY = sys.executable + BINARY = os.path.join(BASE_DIR, '../../../st2reactor/bin/st2garbagecollector') BINARY = os.path.abspath(BINARY) -CMD = [BINARY, '--config-file', ST2_CONFIG_PATH] -CMD_INQUIRY = [BINARY, '--config-file', INQUIRY_CONFIG_PATH] + +CMD = [PYTHON_BINARY, BINARY, '--config-file', ST2_CONFIG_PATH] +CMD_INQUIRY = [PYTHON_BINARY, BINARY, '--config-file', INQUIRY_CONFIG_PATH] TEST_FIXTURES = { 'runners': ['inquirer.yaml'], diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 60b3439f82..21fc4bf790 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -82,7 +82,10 @@ def setUpClass(cls): # Create virtualenv for examples pack virtualenv_path = '/tmp/virtualenvs/examples' - cmd = ['virtualenv', '--system-site-packages', virtualenv_path] + + run_command(cmd=['rm', '-rf', virtualenv_path]) + + cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path] run_command(cmd=cmd) def test_child_processes_are_killed_on_sigint(self): @@ -97,7 +100,7 @@ def test_child_processes_are_killed_on_sigint(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGINT @@ -122,7 +125,7 @@ def test_child_processes_are_killed_on_sigterm(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGTERM @@ -147,7 +150,7 @@ def test_child_processes_are_killed_on_sigkill(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGKILL @@ -164,7 +167,7 @@ def test_child_processes_are_killed_on_sigkill(self): def test_single_sensor_mode(self): # 1. --sensor-ref not provided - cmd = [BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode'] + cmd = [PYTHON_BINARY, BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode'] process = self._start_sensor_container(cmd=cmd) pp = psutil.Process(process.pid) From 27d40d07920c1512e0d8ac4ad32b6c510f551b87 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 15:53:36 +0200 Subject: [PATCH 16/30] Update exporter code so it works under Python 3. --- st2exporter/st2exporter/exporter/dumper.py | 4 ++-- st2exporter/st2exporter/worker.py | 5 ++--- st2exporter/tests/integration/test_dumper_integration.py | 5 +++-- st2exporter/tests/integration/test_export_worker.py | 2 +- st2exporter/tests/unit/test_dumper.py | 6 +++--- st2exporter/tests/unit/test_json_converter.py | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/st2exporter/st2exporter/exporter/dumper.py b/st2exporter/st2exporter/exporter/dumper.py index 49810e0f82..4fc671f06f 100644 --- a/st2exporter/st2exporter/exporter/dumper.py +++ b/st2exporter/st2exporter/exporter/dumper.py @@ -14,9 +14,9 @@ # limitations under the License. import os -import Queue import eventlet +from six.moves import queue from st2common import log as logging from st2exporter.exporter.file_writer import TextFileWriter @@ -94,7 +94,7 @@ def _get_batch(self): for _ in range(self._batch_size): try: item = self._queue.get(block=False) - except Queue.Empty: + except queue.Empty: break else: executions_to_write.append(item) diff --git a/st2exporter/st2exporter/worker.py b/st2exporter/st2exporter/worker.py index d4a478089c..49ba9a8ad0 100644 --- a/st2exporter/st2exporter/worker.py +++ b/st2exporter/st2exporter/worker.py @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import Queue - import eventlet +from six.moves import queue from kombu import Connection from oslo_config import cfg @@ -47,7 +46,7 @@ class ExecutionsExporter(consumers.MessageHandler): def __init__(self, connection, queues): super(ExecutionsExporter, self).__init__(connection, queues) - self.pending_executions = Queue.Queue() + self.pending_executions = queue.Queue() self._dumper = Dumper(queue=self.pending_executions, export_dir=cfg.CONF.exporter.dump_dir) self._consumer_thread = None diff --git a/st2exporter/tests/integration/test_dumper_integration.py b/st2exporter/tests/integration/test_dumper_integration.py index 3c5b1d26ac..a4d655091b 100644 --- a/st2exporter/tests/integration/test_dumper_integration.py +++ b/st2exporter/tests/integration/test_dumper_integration.py @@ -15,11 +15,12 @@ import datetime import os -import Queue import mock import six +from six.moves import queue + from st2common.models.api.execution import ActionExecutionAPI from st2common.persistence.marker import DumperMarker from st2common.util import isotime @@ -47,7 +48,7 @@ class TestDumper(DbTestCase): execution_apis.append(ActionExecutionAPI(**execution)) def get_queue(self): - executions_queue = Queue.Queue() + executions_queue = queue.Queue() for execution in self.execution_apis: executions_queue.put(execution) diff --git a/st2exporter/tests/integration/test_export_worker.py b/st2exporter/tests/integration/test_export_worker.py index 1d21f81c4d..3e539b97b7 100644 --- a/st2exporter/tests/integration/test_export_worker.py +++ b/st2exporter/tests/integration/test_export_worker.py @@ -90,7 +90,7 @@ def test_bootstrap(self): @mock.patch.object(os.path, 'exists', mock.MagicMock(return_value=True)) def test_process(self): - some_execution = self.saved_executions.values()[5] + some_execution = list(self.saved_executions.values())[5] exec_exporter = ExecutionsExporter(None, None) self.assertEqual(exec_exporter.pending_executions.qsize(), 0) exec_exporter.process(some_execution) diff --git a/st2exporter/tests/unit/test_dumper.py b/st2exporter/tests/unit/test_dumper.py index c2d97be181..99dc917f20 100644 --- a/st2exporter/tests/unit/test_dumper.py +++ b/st2exporter/tests/unit/test_dumper.py @@ -15,10 +15,10 @@ import datetime import os -import Queue import eventlet import mock +from six.moves import queue from st2common.models.api.execution import ActionExecutionAPI from st2common.util import isotime @@ -48,7 +48,7 @@ class TestDumper(EventletTestCase): execution_apis.append(ActionExecutionAPI(**execution)) def get_queue(self): - executions_queue = Queue.Queue() + executions_queue = queue.Queue() for execution in self.execution_apis: executions_queue.put(execution) @@ -88,7 +88,7 @@ def test_get_file_name(self): @mock.patch.object(os.path, 'exists', mock.MagicMock(return_value=True)) def test_write_to_disk_empty_queue(self): - dumper = Dumper(queue=Queue.Queue(), + dumper = Dumper(queue=queue.Queue(), export_dir='/tmp', file_prefix='st2-stuff-', file_format='json') # We just make sure this doesn't blow up. diff --git a/st2exporter/tests/unit/test_json_converter.py b/st2exporter/tests/unit/test_json_converter.py index 27e5905a7f..61ac2ff73a 100644 --- a/st2exporter/tests/unit/test_json_converter.py +++ b/st2exporter/tests/unit/test_json_converter.py @@ -36,7 +36,7 @@ class TestJsonConverter(unittest2.TestCase): fixtures_dict=DESCENDANTS_FIXTURES) def test_convert(self): - executions_list = self.loaded_fixtures['executions'].values() + executions_list = list(self.loaded_fixtures['executions'].values()) converter = JsonConverter() converted_doc = converter.convert(executions_list) self.assertTrue(type(converted_doc), 'string') From 5d138db76925bf9755247234a63ca79c2340e5f2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 15:57:23 +0200 Subject: [PATCH 17/30] Also run exporter unit and integration tests under Python 3. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index eb6f33c6e3..afbe055725 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ + nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ @@ -63,6 +64,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + nosetests --with-timer --rednose -sv st2exporter/tests/integration/ [testenv:venv] commands = {posargs} From ece9d6e0a7e60b1b0279f3ffe9d8111a35d28b79 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 18 Jun 2018 15:57:23 +0200 Subject: [PATCH 18/30] Also run exporter unit and integration tests under Python 3. --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index eb6f33c6e3..1b43bc6666 100644 --- a/tox.ini +++ b/tox.ini @@ -25,10 +25,9 @@ commands = nosetests -sv st2common/tests/unit nosetests -sv st2reactor/tests/unit nosetests -sv st2tests/tests/unit - [testenv:py36-unit] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv @@ -37,6 +36,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ + nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ @@ -54,7 +54,7 @@ commands = [testenv:py36-integration] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv @@ -63,6 +63,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + nosetests --with-timer --rednose -sv st2exporter/tests/integration/ [testenv:venv] commands = {posargs} From 6b3f1393bcc7844e9e02cd1f6a27240fb9c37b8d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 08:57:22 +0200 Subject: [PATCH 19/30] Fix st2debug tests so they work under Python 3. --- st2debug/st2debug/cmd/submit_debug_info.py | 9 +++++++-- st2debug/tests/integration/fixtures/configs/st2.conf | 3 --- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/st2debug/st2debug/cmd/submit_debug_info.py b/st2debug/st2debug/cmd/submit_debug_info.py index 136e11f7a2..4fe6c60a5d 100644 --- a/st2debug/st2debug/cmd/submit_debug_info.py +++ b/st2debug/st2debug/cmd/submit_debug_info.py @@ -244,7 +244,7 @@ def create_archive(self): # Prepend temp_dir_path to OUTPUT_PATHS output_paths = {} - for key, path in OUTPUT_PATHS.iteritems(): + for key, path in six.iteritems(OUTPUT_PATHS): output_paths[key] = os.path.join(self._temp_dir_path, path) # 2. Moves all the files to the temporary directory @@ -489,7 +489,12 @@ def format_output_filename(cmd): :return: Formatted filename. :rtype: ``str`` """ - return cmd.translate(None, """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""") + if six.PY3: + cmd = cmd.translate(cmd.maketrans('', '', """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""")) + else: + cmd = cmd.translate(None, """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""") + + return cmd @staticmethod def get_system_information(): diff --git a/st2debug/tests/integration/fixtures/configs/st2.conf b/st2debug/tests/integration/fixtures/configs/st2.conf index 2dbadf4b89..e113aa822d 100644 --- a/st2debug/tests/integration/fixtures/configs/st2.conf +++ b/st2debug/tests/integration/fixtures/configs/st2.conf @@ -13,9 +13,6 @@ logging = st2api/conf/logging.conf username = ponies password = ponies -[messaging] -url = ponies - [sensorcontainer] logging = st2reactor/conf/logging.sensorcontainer.conf From e6e03052ae26cffeca14934c267548b7be302cd4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 08:58:00 +0200 Subject: [PATCH 20/30] Run st2debug tests under tox. --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1b43bc6666..143c87ff62 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ + nosetests --with-timer --rednose -sv st2debug/tests/unit/ nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ @@ -62,8 +63,9 @@ deps = virtualenv -e{toxinidir}/st2client -e{toxinidir}/st2common commands = - nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + nosetests --with-timer --rednose -sv st2debug/tests/integration/ nosetests --with-timer --rednose -sv st2exporter/tests/integration/ + nosetests --with-timer --rednose -sv st2reactor/tests/integration/ [testenv:venv] commands = {posargs} From f2d48e38115770cf16a5970c8070283a1c18cc68 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 17:12:41 +0200 Subject: [PATCH 21/30] Also run st2common and st2api integration tests under Python 3 in tox. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 143c87ff62..d3a6b5a8c3 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,8 @@ deps = virtualenv -e{toxinidir}/st2client -e{toxinidir}/st2common commands = + nosetests --with-timer --rednose -sv st2api/tests/integration/ + nosetests --with-timer --rednose -sv st2common/tests/integration/ nosetests --with-timer --rednose -sv st2debug/tests/integration/ nosetests --with-timer --rednose -sv st2exporter/tests/integration/ nosetests --with-timer --rednose -sv st2reactor/tests/integration/ From 56a08fae0947f1c3057aebc39a898a4dd390da6e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 17:14:35 +0200 Subject: [PATCH 22/30] Fix st2actions tests so they pass under Python 3. --- st2common/st2common/constants/scheduler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/constants/scheduler.py b/st2common/st2common/constants/scheduler.py index bad30150ce..a64b9cf35d 100644 --- a/st2common/st2common/constants/scheduler.py +++ b/st2common/st2common/constants/scheduler.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['SCHEDULER_ENABLED_LOG_LINE', 'SCHEDULER_DISABLED_LOG_LINE'] +__all__ = [ + 'SCHEDULER_ENABLED_LOG_LINE', + 'SCHEDULER_DISABLED_LOG_LINE' +] # Integration tests look for these loglines to validate scheduler enable/disable -SCHEDULER_ENABLED_LOG_LINE = 'Scheduler is enabled.' -SCHEDULER_DISABLED_LOG_LINE = 'Scheduler is disabled.' +SCHEDULER_ENABLED_LOG_LINE = b'Scheduler is enabled.' +SCHEDULER_DISABLED_LOG_LINE = b'Scheduler is disabled.' From f7abf5d87e54f261882e8ec1f0fba21a59e32dfe Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 17:14:46 +0200 Subject: [PATCH 23/30] Run st2actions integration tests under Python 3 in tox. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index d3a6b5a8c3..855538cf55 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = virtualenv -e{toxinidir}/st2client -e{toxinidir}/st2common commands = + nosetests --with-timer --rednose -sv st2actions/tests/integration/ nosetests --with-timer --rednose -sv st2api/tests/integration/ nosetests --with-timer --rednose -sv st2common/tests/integration/ nosetests --with-timer --rednose -sv st2debug/tests/integration/ From c5138b1ef28fc69ec7bcb9625932fa67959dced0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 17:28:54 +0200 Subject: [PATCH 24/30] Make sure st2auth is in PYTHONPATH for tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 855538cf55..ba4db63078 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ commands = [testenv:py36-integration] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv From 6ea097a32a6c36b50d4bc5361b72f12bed854bf9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 18:19:23 +0200 Subject: [PATCH 25/30] Use the same binary which is used to run the tests. --- st2common/tests/integration/test_register_content_script.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index e0e428022f..c0ccaa1a11 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -14,7 +14,9 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import glob from st2tests.base import IntegrationTestCase @@ -27,7 +29,7 @@ SCRIPT_PATH = os.path.join(BASE_DIR, '../../bin/st2-register-content') SCRIPT_PATH = os.path.abspath(SCRIPT_PATH) -BASE_CMD_ARGS = [SCRIPT_PATH, '--config-file=conf/st2.tests.conf', '-v'] +BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests.conf', '-v'] BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ['--register-actions'] PACKS_PATH = get_fixtures_packs_base_path() From 7098561687c9b630da218ba8c4e518887fb9405a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 18:45:39 +0200 Subject: [PATCH 26/30] Include more info to aid with debugging. --- st2common/tests/integration/test_register_content_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index c0ccaa1a11..e1a8291f5e 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -115,7 +115,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self): # dir as packs_base_paths cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-sensors'] exit_code, _, stderr = run_command(cmd=cmd) - self.assertTrue('Registered 0 sensors.' in stderr) + self.assertTrue('Registered 0 sensors.' in stderr, 'Actual stderr: %s' % (stderr)) self.assertEqual(exit_code, 0) cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-all', @@ -131,7 +131,7 @@ def test_register_all_and_register_setup_virtualenvs(self): cmd = BASE_CMD_ARGS + ['--register-all', '--register-setup-virtualenvs', '--register-no-fail-on-failure'] exit_code, stdout, stderr = run_command(cmd=cmd) - self.assertTrue('Registering actions' in stderr) + self.assertTrue('Registering actions' in stderr, 'Actual stderr: %s' % (stderr)) self.assertTrue('Registering rules' in stderr) self.assertTrue('Setup virtualenv for %s pack(s)' % (PACKS_COUNT) in stderr) self.assertEqual(exit_code, 0) From 6fdda8c91c57af3b29c6de294fba7915e4542a3a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 19:12:29 +0200 Subject: [PATCH 27/30] Use executable which is used to run the tests. --- .../tests/integration/test_register_content_script.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index e1a8291f5e..53740d5cf7 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -113,13 +113,14 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self): # Note: We want to use a different config which sets fixtures/packs_1/ # dir as packs_base_paths - cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-sensors'] + cmd = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', + '--register-sensors'] exit_code, _, stderr = run_command(cmd=cmd) self.assertTrue('Registered 0 sensors.' in stderr, 'Actual stderr: %s' % (stderr)) self.assertEqual(exit_code, 0) - cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-all', - '--register-no-fail-on-failure'] + cmd = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', + '--register-all', '--register-no-fail-on-failure'] exit_code, _, stderr = run_command(cmd=cmd) self.assertTrue('Registered 0 actions.' in stderr) self.assertTrue('Registered 0 sensors.' in stderr) From f4afc848271ee20bec6a6fae395a23b70a785ea3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 19 Jun 2018 19:24:53 +0200 Subject: [PATCH 28/30] Update tests config. --- conf/st2.tests.conf | 1 + conf/st2.tests1.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/st2.tests.conf b/conf/st2.tests.conf index ea50cbc4a2..d6ec8bc6a9 100644 --- a/conf/st2.tests.conf +++ b/conf/st2.tests.conf @@ -56,6 +56,7 @@ sleep_delay = 0.1 [content] system_packs_base_path = packs_base_paths = st2tests/st2tests/fixtures/packs/ +system_runners_base_path = contrib/runners [syslog] host = 127.0.0.1 diff --git a/conf/st2.tests1.conf b/conf/st2.tests1.conf index 2ec4592807..7cad5b74de 100644 --- a/conf/st2.tests1.conf +++ b/conf/st2.tests1.conf @@ -42,6 +42,7 @@ base_path = /tmp [content] system_packs_base_path = packs_base_paths = st2tests/st2tests/fixtures/packs_1/ +system_runners_base_path = contrib/runners [syslog] host = 127.0.0.1 From 4d3530fa8d51311041f86d41c1c24204012104d2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 12:45:00 +0200 Subject: [PATCH 29/30] Add a fix for router so it correctly handles data of type text/plain under Python 3. Input body is actually bytes under Python 3, but we except and work with text type (str / unicode) so we need to convert from bytes to unicode. --- st2common/st2common/router.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index 7691a44d85..18f42c3ff0 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -370,6 +370,11 @@ def __call__(self, req): detail = 'Failed to parse request body: %s' % str(e) raise exc.HTTPBadRequest(detail=detail) + # Special case for Python 3 + if six.PY3 and content_type == 'text/plain' and isinstance(data, six.binary_type): + # Convert bytes to text type (string / unicode) + data = data.decode('utf-8') + try: CustomValidator(schema, resolver=self.spec_resolver).validate(data) except (jsonschema.ValidationError, ValueError) as e: From 3ee535133eac0450a8f8b8be6ffd75c7a517dbe1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 12:46:12 +0200 Subject: [PATCH 30/30] Mistral validation tests now pass under Python 3. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ba4db63078..73eb41901d 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ commands = nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ - nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ + nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/exp/ nosetests --with-timer --rednose -sv st2common/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/