diff --git a/.travis.yml b/.travis.yml index 6dad816791..e3083f210c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,8 @@ matrix: python: 2.7 - env: TASK="compilepy3 ci-py3-unit" CACHE_NAME=py3 python: 3.6 + - env: TASK="ci-py3-integration" CACHE_NAME=py3 + python: 3.6 addons: apt: sources: @@ -55,7 +57,7 @@ before_install: - sudo pip install --upgrade "virtualenv==15.1.0" install: - - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then pip install "tox==3.0.0"; else make requirements; fi + - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then sudo .circle/add-itest-user.sh; fi diff --git a/Makefile b/Makefile index 5942dffaeb..f1ff191871 100644 --- a/Makefile +++ b/Makefile @@ -728,7 +728,14 @@ ci-py3-unit: @echo @echo "==================== ci-py3-unit ====================" @echo - tox -e py36 -vv + tox -e py36-unit -vv + +.PHONY: ci-py3-integration +ci-py3-integration: + @echo + @echo "==================== ci-py3-integration ====================" + @echo + tox -e py36-integration -vv .PHONY: .rst-check .rst-check: diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index d71ce0a595..73fdf0d8bd 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -257,6 +257,8 @@ draft = http://json-schema.org/draft-04/schema# [sensorcontainer] # Provider of sensor node partition config. partition_provider = {'name': 'default'} +# Run in a single sensor mode where parent process exits when a sensor crashes / dies. This is useful in environments where partitioning, sensor process life cycle and failover is handled by a 3rd party service such as kubernetes. +single_sensor_mode = False # location of the logging.conf file logging = conf/logging.sensorcontainer.conf # name of the sensor node. diff --git a/conf/st2.tests.conf b/conf/st2.tests.conf index ea50cbc4a2..d6ec8bc6a9 100644 --- a/conf/st2.tests.conf +++ b/conf/st2.tests.conf @@ -56,6 +56,7 @@ sleep_delay = 0.1 [content] system_packs_base_path = packs_base_paths = st2tests/st2tests/fixtures/packs/ +system_runners_base_path = contrib/runners [syslog] host = 127.0.0.1 diff --git a/conf/st2.tests1.conf b/conf/st2.tests1.conf index 2ec4592807..7cad5b74de 100644 --- a/conf/st2.tests1.conf +++ b/conf/st2.tests1.conf @@ -42,6 +42,7 @@ base_path = /tmp [content] system_packs_base_path = packs_base_paths = st2tests/st2tests/fixtures/packs_1/ +system_runners_base_path = contrib/runners [syslog] host = 127.0.0.1 diff --git a/contrib/hello_st2/config.yaml b/contrib/hello_st2/config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/st2common/st2common/constants/scheduler.py b/st2common/st2common/constants/scheduler.py index bad30150ce..a64b9cf35d 100644 --- a/st2common/st2common/constants/scheduler.py +++ b/st2common/st2common/constants/scheduler.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['SCHEDULER_ENABLED_LOG_LINE', 'SCHEDULER_DISABLED_LOG_LINE'] +__all__ = [ + 'SCHEDULER_ENABLED_LOG_LINE', + 'SCHEDULER_DISABLED_LOG_LINE' +] # Integration tests look for these loglines to validate scheduler enable/disable -SCHEDULER_ENABLED_LOG_LINE = 'Scheduler is enabled.' -SCHEDULER_DISABLED_LOG_LINE = 'Scheduler is disabled.' +SCHEDULER_ENABLED_LOG_LINE = b'Scheduler is enabled.' +SCHEDULER_DISABLED_LOG_LINE = b'Scheduler is disabled.' diff --git a/st2common/st2common/router.py b/st2common/st2common/router.py index 7691a44d85..18f42c3ff0 100644 --- a/st2common/st2common/router.py +++ b/st2common/st2common/router.py @@ -370,6 +370,11 @@ def __call__(self, req): detail = 'Failed to parse request body: %s' % str(e) raise exc.HTTPBadRequest(detail=detail) + # Special case for Python 3 + if six.PY3 and content_type == 'text/plain' and isinstance(data, six.binary_type): + # Convert bytes to text type (string / unicode) + data = data.decode('utf-8') + try: CustomValidator(schema, resolver=self.spec_resolver).validate(data) except (jsonschema.ValidationError, ValueError) as e: diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index e0e428022f..53740d5cf7 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -14,7 +14,9 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import glob from st2tests.base import IntegrationTestCase @@ -27,7 +29,7 @@ SCRIPT_PATH = os.path.join(BASE_DIR, '../../bin/st2-register-content') SCRIPT_PATH = os.path.abspath(SCRIPT_PATH) -BASE_CMD_ARGS = [SCRIPT_PATH, '--config-file=conf/st2.tests.conf', '-v'] +BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests.conf', '-v'] BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ['--register-actions'] PACKS_PATH = get_fixtures_packs_base_path() @@ -111,13 +113,14 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self): # Note: We want to use a different config which sets fixtures/packs_1/ # dir as packs_base_paths - cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-sensors'] + cmd = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', + '--register-sensors'] exit_code, _, stderr = run_command(cmd=cmd) - self.assertTrue('Registered 0 sensors.' in stderr) + self.assertTrue('Registered 0 sensors.' in stderr, 'Actual stderr: %s' % (stderr)) self.assertEqual(exit_code, 0) - cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-all', - '--register-no-fail-on-failure'] + cmd = [sys.executable, SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', + '--register-all', '--register-no-fail-on-failure'] exit_code, _, stderr = run_command(cmd=cmd) self.assertTrue('Registered 0 actions.' in stderr) self.assertTrue('Registered 0 sensors.' in stderr) @@ -129,7 +132,7 @@ def test_register_all_and_register_setup_virtualenvs(self): cmd = BASE_CMD_ARGS + ['--register-all', '--register-setup-virtualenvs', '--register-no-fail-on-failure'] exit_code, stdout, stderr = run_command(cmd=cmd) - self.assertTrue('Registering actions' in stderr) + self.assertTrue('Registering actions' in stderr, 'Actual stderr: %s' % (stderr)) self.assertTrue('Registering rules' in stderr) self.assertTrue('Setup virtualenv for %s pack(s)' % (PACKS_COUNT) in stderr) self.assertEqual(exit_code, 0) diff --git a/st2debug/st2debug/cmd/submit_debug_info.py b/st2debug/st2debug/cmd/submit_debug_info.py index 136e11f7a2..4fe6c60a5d 100644 --- a/st2debug/st2debug/cmd/submit_debug_info.py +++ b/st2debug/st2debug/cmd/submit_debug_info.py @@ -244,7 +244,7 @@ def create_archive(self): # Prepend temp_dir_path to OUTPUT_PATHS output_paths = {} - for key, path in OUTPUT_PATHS.iteritems(): + for key, path in six.iteritems(OUTPUT_PATHS): output_paths[key] = os.path.join(self._temp_dir_path, path) # 2. Moves all the files to the temporary directory @@ -489,7 +489,12 @@ def format_output_filename(cmd): :return: Formatted filename. :rtype: ``str`` """ - return cmd.translate(None, """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""") + if six.PY3: + cmd = cmd.translate(cmd.maketrans('', '', """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""")) + else: + cmd = cmd.translate(None, """ !@#$%^&*()[]{};:,./<>?\|`~=+"'""") + + return cmd @staticmethod def get_system_information(): diff --git a/st2debug/tests/integration/fixtures/configs/st2.conf b/st2debug/tests/integration/fixtures/configs/st2.conf index 2dbadf4b89..e113aa822d 100644 --- a/st2debug/tests/integration/fixtures/configs/st2.conf +++ b/st2debug/tests/integration/fixtures/configs/st2.conf @@ -13,9 +13,6 @@ logging = st2api/conf/logging.conf username = ponies password = ponies -[messaging] -url = ponies - [sensorcontainer] logging = st2reactor/conf/logging.sensorcontainer.conf diff --git a/st2exporter/st2exporter/exporter/dumper.py b/st2exporter/st2exporter/exporter/dumper.py index 49810e0f82..4fc671f06f 100644 --- a/st2exporter/st2exporter/exporter/dumper.py +++ b/st2exporter/st2exporter/exporter/dumper.py @@ -14,9 +14,9 @@ # limitations under the License. import os -import Queue import eventlet +from six.moves import queue from st2common import log as logging from st2exporter.exporter.file_writer import TextFileWriter @@ -94,7 +94,7 @@ def _get_batch(self): for _ in range(self._batch_size): try: item = self._queue.get(block=False) - except Queue.Empty: + except queue.Empty: break else: executions_to_write.append(item) diff --git a/st2exporter/st2exporter/worker.py b/st2exporter/st2exporter/worker.py index d4a478089c..49ba9a8ad0 100644 --- a/st2exporter/st2exporter/worker.py +++ b/st2exporter/st2exporter/worker.py @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import Queue - import eventlet +from six.moves import queue from kombu import Connection from oslo_config import cfg @@ -47,7 +46,7 @@ class ExecutionsExporter(consumers.MessageHandler): def __init__(self, connection, queues): super(ExecutionsExporter, self).__init__(connection, queues) - self.pending_executions = Queue.Queue() + self.pending_executions = queue.Queue() self._dumper = Dumper(queue=self.pending_executions, export_dir=cfg.CONF.exporter.dump_dir) self._consumer_thread = None diff --git a/st2exporter/tests/integration/test_dumper_integration.py b/st2exporter/tests/integration/test_dumper_integration.py index 3c5b1d26ac..a4d655091b 100644 --- a/st2exporter/tests/integration/test_dumper_integration.py +++ b/st2exporter/tests/integration/test_dumper_integration.py @@ -15,11 +15,12 @@ import datetime import os -import Queue import mock import six +from six.moves import queue + from st2common.models.api.execution import ActionExecutionAPI from st2common.persistence.marker import DumperMarker from st2common.util import isotime @@ -47,7 +48,7 @@ class TestDumper(DbTestCase): execution_apis.append(ActionExecutionAPI(**execution)) def get_queue(self): - executions_queue = Queue.Queue() + executions_queue = queue.Queue() for execution in self.execution_apis: executions_queue.put(execution) diff --git a/st2exporter/tests/integration/test_export_worker.py b/st2exporter/tests/integration/test_export_worker.py index 1d21f81c4d..3e539b97b7 100644 --- a/st2exporter/tests/integration/test_export_worker.py +++ b/st2exporter/tests/integration/test_export_worker.py @@ -90,7 +90,7 @@ def test_bootstrap(self): @mock.patch.object(os.path, 'exists', mock.MagicMock(return_value=True)) def test_process(self): - some_execution = self.saved_executions.values()[5] + some_execution = list(self.saved_executions.values())[5] exec_exporter = ExecutionsExporter(None, None) self.assertEqual(exec_exporter.pending_executions.qsize(), 0) exec_exporter.process(some_execution) diff --git a/st2exporter/tests/unit/test_dumper.py b/st2exporter/tests/unit/test_dumper.py index c2d97be181..99dc917f20 100644 --- a/st2exporter/tests/unit/test_dumper.py +++ b/st2exporter/tests/unit/test_dumper.py @@ -15,10 +15,10 @@ import datetime import os -import Queue import eventlet import mock +from six.moves import queue from st2common.models.api.execution import ActionExecutionAPI from st2common.util import isotime @@ -48,7 +48,7 @@ class TestDumper(EventletTestCase): execution_apis.append(ActionExecutionAPI(**execution)) def get_queue(self): - executions_queue = Queue.Queue() + executions_queue = queue.Queue() for execution in self.execution_apis: executions_queue.put(execution) @@ -88,7 +88,7 @@ def test_get_file_name(self): @mock.patch.object(os.path, 'exists', mock.MagicMock(return_value=True)) def test_write_to_disk_empty_queue(self): - dumper = Dumper(queue=Queue.Queue(), + dumper = Dumper(queue=queue.Queue(), export_dir='/tmp', file_prefix='st2-stuff-', file_format='json') # We just make sure this doesn't blow up. diff --git a/st2exporter/tests/unit/test_json_converter.py b/st2exporter/tests/unit/test_json_converter.py index 27e5905a7f..61ac2ff73a 100644 --- a/st2exporter/tests/unit/test_json_converter.py +++ b/st2exporter/tests/unit/test_json_converter.py @@ -36,7 +36,7 @@ class TestJsonConverter(unittest2.TestCase): fixtures_dict=DESCENDANTS_FIXTURES) def test_convert(self): - executions_list = self.loaded_fixtures['executions'].values() + executions_list = list(self.loaded_fixtures['executions'].values()) converter = JsonConverter() converted_doc = converter.convert(executions_list) self.assertTrue(type(converted_doc), 'string') diff --git a/st2reactor/st2reactor/cmd/sensormanager.py b/st2reactor/st2reactor/cmd/sensormanager.py index 0fe192196d..f990450b33 100644 --- a/st2reactor/st2reactor/cmd/sensormanager.py +++ b/st2reactor/st2reactor/cmd/sensormanager.py @@ -14,9 +14,12 @@ # limitations under the License. from __future__ import absolute_import + import os import sys +from oslo_config import cfg + from st2common import log as logging from st2common.logging.misc import get_logger_name_for_module from st2common.service_setup import setup as common_setup @@ -50,8 +53,17 @@ def _teardown(): def main(): try: _setup() + + single_sensor_mode = (cfg.CONF.single_sensor_mode or + cfg.CONF.sensorcontainer.single_sensor_mode) + + if single_sensor_mode and not cfg.CONF.sensor_ref: + raise ValueError('--sensor-ref argument must be provided when running in single ' + 'sensor mode') + sensors_partitioner = get_sensors_partitioner() - container_manager = SensorContainerManager(sensors_partitioner=sensors_partitioner) + container_manager = SensorContainerManager(sensors_partitioner=sensors_partitioner, + single_sensor_mode=single_sensor_mode) return container_manager.run_sensors() except SystemExit as exit_code: return exit_code diff --git a/st2reactor/st2reactor/container/manager.py b/st2reactor/st2reactor/container/manager.py index d9f4e3071e..7ba5e8628d 100644 --- a/st2reactor/st2reactor/container/manager.py +++ b/st2reactor/st2reactor/container/manager.py @@ -27,19 +27,27 @@ LOG = logging.getLogger(__name__) +__all__ = [ + 'SensorContainerManager' +] + class SensorContainerManager(object): - def __init__(self, sensors_partitioner): + def __init__(self, sensors_partitioner, single_sensor_mode=False): + if not sensors_partitioner: + raise ValueError('sensors_partitioner should be non-None.') + + self._sensors_partitioner = sensors_partitioner + self._single_sensor_mode = single_sensor_mode + self._sensor_container = None + self._container_thread = None + self._sensors_watcher = SensorWatcher(create_handler=self._handle_create_sensor, update_handler=self._handle_update_sensor, delete_handler=self._handle_delete_sensor, queue_suffix='sensor_container') - self._container_thread = None - if not sensors_partitioner: - raise ValueError('sensors_partitioner should be non-None.') - self._sensors_partitioner = sensors_partitioner def run_sensors(self): """ @@ -57,14 +65,22 @@ def run_sensors(self): LOG.info('(PID:%s) SensorContainer started.', os.getpid()) self._setup_sigterm_handler() - self._spin_container_and_wait(sensors_to_run) + + exit_code = self._spin_container_and_wait(sensors_to_run) + return exit_code def _spin_container_and_wait(self, sensors): + exit_code = 0 + try: - self._sensor_container = ProcessSensorContainer(sensors=sensors) + self._sensor_container = ProcessSensorContainer( + sensors=sensors, + single_sensor_mode=self._single_sensor_mode) self._container_thread = eventlet.spawn(self._sensor_container.run) + LOG.debug('Starting sensor CUD watcher...') self._sensors_watcher.start() + exit_code = self._container_thread.wait() LOG.error('Process container quit with exit_code %d.', exit_code) LOG.error('(PID:%s) SensorContainer stopped.', os.getpid()) @@ -78,7 +94,9 @@ def _spin_container_and_wait(self, sensors): eventlet.kill(self._container_thread) self._container_thread = None - return 0 + return exit_code + + return exit_code def _setup_sigterm_handler(self): diff --git a/st2reactor/st2reactor/container/partitioner_lookup.py b/st2reactor/st2reactor/container/partitioner_lookup.py index ca2c1c7ba3..e291254f85 100644 --- a/st2reactor/st2reactor/container/partitioner_lookup.py +++ b/st2reactor/st2reactor/container/partitioner_lookup.py @@ -41,16 +41,19 @@ def get_sensors_partitioner(): if cfg.CONF.sensor_ref: + LOG.info('Running in single sensor mode, using a single sensor partitioner...') return SingleSensorPartitioner(sensor_ref=cfg.CONF.sensor_ref) + partition_provider_config = copy.copy(cfg.CONF.sensorcontainer.partition_provider) partition_provider = partition_provider_config.pop('name') sensor_node_name = cfg.CONF.sensorcontainer.sensor_node_name provider = PROVIDERS.get(partition_provider.lower(), None) - LOG.info('Using partitioner %s with sensornode %s.', partition_provider, sensor_node_name) if not provider: - raise SensorPartitionerNotSupportedException( - 'Partition provider %s not found.' % partition_provider) + raise SensorPartitionerNotSupportedException('Partition provider %s not found.' % + (partition_provider)) + + LOG.info('Using partitioner %s with sensornode %s.', partition_provider, sensor_node_name) # pass in extra config with no analysis return provider(sensor_node_name=sensor_node_name, **partition_provider_config) diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 39cb4fea24..33aaec95cf 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -77,7 +77,7 @@ class ProcessSensorContainer(object): Sensor container which runs sensors in a separate process. """ - def __init__(self, sensors, poll_interval=5, dispatcher=None): + def __init__(self, sensors, poll_interval=5, single_sensor_mode=False, dispatcher=None): """ :param sensors: A list of sensor dicts. :type sensors: ``list`` of ``dict`` @@ -86,6 +86,12 @@ def __init__(self, sensors, poll_interval=5, dispatcher=None): :type poll_interval: ``float`` """ self._poll_interval = poll_interval + self._single_sensor_mode = single_sensor_mode + + if self._single_sensor_mode: + # For more immediate feedback we use lower poll interval when running in single sensor + # mode + self._poll_interval = 1 self._sensors = {} # maps sensor_id -> sensor object self._processes = {} # maps sensor_id -> sensor process @@ -95,6 +101,7 @@ def __init__(self, sensors, poll_interval=5, dispatcher=None): self._dispatcher = dispatcher self._stopped = False + self._exit_code = None # exit code with which this process should exit sensors = sensors or [] for sensor_obj in sensors: @@ -144,8 +151,10 @@ def run(self): return FAILURE_EXIT_CODE self._stopped = True - LOG.error('Process container quit. It shouldn\'t.') - return SUCCESS_EXIT_CODE + LOG.error('Process container stopped.') + + exit_code = self._exit_code or SUCCESS_EXIT_CODE + return exit_code def _poll_sensors_for_results(self, sensor_ids): """ @@ -390,6 +399,15 @@ def _respawn_sensor(self, sensor_id, sensor, exit_code): """ extra = {'sensor_id': sensor_id, 'sensor': sensor} + if self._single_sensor_mode: + # In single sensor mode we want to exit immediately on failure + LOG.info('Not respawning a sensor since running in single sensor mode', + extra=extra) + + self._stopped = True + self._exit_code = exit_code + return + if self._stopped: LOG.debug('Stopped, not respawning a dead sensor', extra=extra) return diff --git a/st2reactor/st2reactor/sensor/config.py b/st2reactor/st2reactor/sensor/config.py index de8376c25f..90d9d5e974 100644 --- a/st2reactor/st2reactor/sensor/config.py +++ b/st2reactor/st2reactor/sensor/config.py @@ -52,6 +52,7 @@ def _register_sensor_container_opts(ignore_errors=False): st2cfg.do_register_opts(logging_opts, group='sensorcontainer', ignore_errors=ignore_errors) + # Partitioning options partition_opts = [ cfg.StrOpt( 'sensor_node_name', default='sensornode1', @@ -65,11 +66,31 @@ def _register_sensor_container_opts(ignore_errors=False): st2cfg.do_register_opts(partition_opts, group='sensorcontainer', ignore_errors=ignore_errors) - sensor_test_opt = cfg.StrOpt( - 'sensor-ref', - help='Only run sensor with the provided reference. Value is of the form pack.sensor-name.') + # Other options + other_opts = [ + cfg.BoolOpt( + 'single_sensor_mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] + + st2cfg.do_register_opts(other_opts, group='sensorcontainer', ignore_errors=ignore_errors) + + # CLI options + cli_opts = [ + cfg.StrOpt( + 'sensor-ref', + help='Only run sensor with the provided reference. Value is of the form ' + '. (e.g. linux.FileWatchSensor).'), + cfg.BoolOpt( + 'single-sensor-mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] - st2cfg.do_register_cli_opts(sensor_test_opt, ignore_errors=ignore_errors) + st2cfg.do_register_cli_opts(cli_opts, ignore_errors=ignore_errors) register_opts(ignore_errors=True) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 18091e851e..698ad423be 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -14,7 +14,9 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import signal import datetime @@ -43,14 +45,20 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + ST2_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests.conf') ST2_CONFIG_PATH = os.path.abspath(ST2_CONFIG_PATH) + INQUIRY_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests2.conf') INQUIRY_CONFIG_PATH = os.path.abspath(INQUIRY_CONFIG_PATH) + +PYTHON_BINARY = sys.executable + BINARY = os.path.join(BASE_DIR, '../../../st2reactor/bin/st2garbagecollector') BINARY = os.path.abspath(BINARY) -CMD = [BINARY, '--config-file', ST2_CONFIG_PATH] -CMD_INQUIRY = [BINARY, '--config-file', INQUIRY_CONFIG_PATH] + +CMD = [PYTHON_BINARY, BINARY, '--config-file', ST2_CONFIG_PATH] +CMD_INQUIRY = [PYTHON_BINARY, BINARY, '--config-file', INQUIRY_CONFIG_PATH] TEST_FIXTURES = { 'runners': ['inquirer.yaml'], diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 98212d3afe..21fc4bf790 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -14,9 +14,10 @@ # limitations under the License. from __future__ import absolute_import + import os +import sys import signal -import unittest2 import psutil import eventlet @@ -35,14 +36,27 @@ ] BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + ST2_CONFIG_PATH = os.path.join(BASE_DIR, '../../../conf/st2.tests.conf') ST2_CONFIG_PATH = os.path.abspath(ST2_CONFIG_PATH) + +PYTHON_BINARY = sys.executable + BINARY = os.path.join(BASE_DIR, '../../../st2reactor/bin/st2sensorcontainer') BINARY = os.path.abspath(BINARY) -CMD = [BINARY, '--config-file', ST2_CONFIG_PATH, '--sensor-ref=examples.SamplePollingSensor'] +PACKS_BASE_PATH = os.path.join(BASE_DIR, '../../../contrib') -@unittest2.skipIf(True, 'Skipped until we improve integration tests setup') +DEFAULT_CMD = [ + PYTHON_BINARY, + BINARY, + '--config-file', + ST2_CONFIG_PATH, + '--sensor-ref=examples.SamplePollingSensor' +] + + +# @unittest2.skipIf(True, 'Skipped until we improve integration tests setup') class SensorContainerTestCase(IntegrationTestCase): """ Note: For those tests MongoDB must be running, virtualenv must exist for @@ -54,7 +68,6 @@ class SensorContainerTestCase(IntegrationTestCase): @classmethod def setUpClass(cls): super(SensorContainerTestCase, cls).setUpClass() - return st2tests.config.parse_args() @@ -65,18 +78,21 @@ def setUpClass(cls): username=username, password=password, ensure_indexes=False) # Register sensors - register_sensors(packs_base_paths=['/opt/stackstorm/packs'], use_pack_cache=False) + register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False) # Create virtualenv for examples pack - virtualenv_path = '/opt/stackstorm/virtualenvs/examples' - cmd = ['virtualenv', '--system-site-packages', virtualenv_path] + virtualenv_path = '/tmp/virtualenvs/examples' + + run_command(cmd=['rm', '-rf', virtualenv_path]) + + cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path] run_command(cmd=cmd) def test_child_processes_are_killed_on_sigint(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(5) # Assert process has started and is running self.assertProcessIsRunning(process=process) @@ -84,7 +100,7 @@ def test_child_processes_are_killed_on_sigint(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGINT @@ -92,7 +108,7 @@ def test_child_processes_are_killed_on_sigint(self): # SIGINT causes graceful shutdown so give it some time to gracefuly shut down the sensor # child processes - eventlet.sleep(PROCESS_EXIT_TIMEOUT + 1) + eventlet.sleep(PROCESS_EXIT_TIMEOUT + 2) # Verify parent and children processes have exited self.assertProcessExited(proc=pp) @@ -109,7 +125,7 @@ def test_child_processes_are_killed_on_sigterm(self): # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGTERM @@ -129,12 +145,12 @@ def test_child_processes_are_killed_on_sigkill(self): process = self._start_sensor_container() # Give it some time to start up - eventlet.sleep(3) + eventlet.sleep(4) # Verify container process and children sensor / wrapper processes are running pp = psutil.Process(process.pid) children_pp = pp.children() - self.assertEqual(pp.cmdline()[1:], CMD) + self.assertEqual(pp.cmdline()[1:], DEFAULT_CMD[1:]) self.assertEqual(len(children_pp), 1) # Send SIGKILL @@ -149,8 +165,46 @@ def test_child_processes_are_killed_on_sigkill(self): self.remove_process(process=process) - def _start_sensor_container(self): - process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + def test_single_sensor_mode(self): + # 1. --sensor-ref not provided + cmd = [PYTHON_BINARY, BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode'] + + process = self._start_sensor_container(cmd=cmd) + pp = psutil.Process(process.pid) + + # Give it some time to start up + eventlet.sleep(4) + + stdout = process.stdout.read() + self.assertTrue((b'--sensor-ref argument must be provided when running in single sensor ' + b'mode') in stdout) + self.assertProcessExited(proc=pp) + self.remove_process(process=process) + + # 2. sensor ref provided + cmd = [BINARY, '--config-file', ST2_CONFIG_PATH, '--single-sensor-mode', + '--sensor-ref=examples.SampleSensorExit'] + + process = self._start_sensor_container(cmd=cmd) + pp = psutil.Process(process.pid) + + # Give it some time to start up + eventlet.sleep(8) + + # Container should exit and not respawn a sensor in single sensor mode + stdout = process.stdout.read() + + self.assertTrue(b'Process for sensor examples.SampleSensorExit has exited with code 110') + self.assertTrue(b'Not respawning a sensor since running in single sensor mode') + self.assertTrue(b'Process container quit with exit_code 110.') + + eventlet.sleep(2) + self.assertProcessExited(proc=pp) + + self.remove_process(process=process) + + def _start_sensor_container(self, cmd=DEFAULT_CMD): + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, preexec_fn=os.setsid) self.add_process(process=process) return process diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 6222e95567..99f4acbda4 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -279,11 +279,31 @@ def _register_sensor_container_opts(): _register_opts(partition_opts, group='sensorcontainer') - sensor_test_opt = cfg.StrOpt( - 'sensor-ref', - help='Only run sensor with the provided reference. Value is of the form pack.sensor-name.') + # Other options + other_opts = [ + cfg.BoolOpt( + 'single_sensor_mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] + + _register_opts(other_opts, group='sensorcontainer') + + # CLI options + cli_opts = [ + cfg.StrOpt( + 'sensor-ref', + help='Only run sensor with the provided reference. Value is of the form ' + '. (e.g. linux.FileWatchSensor).'), + cfg.BoolOpt( + 'single-sensor-mode', default=False, + help='Run in a single sensor mode where parent process exits when a sensor crashes / ' + 'dies. This is useful in environments where partitioning, sensor process life ' + 'cycle and failover is handled by a 3rd party service such as kubernetes.') + ] - _register_cli_opts([sensor_test_opt]) + _register_cli_opts(cli_opts) def _register_opts(opts, group=None): diff --git a/tox.ini b/tox.ini index d58db60938..374d1dee67 100644 --- a/tox.ini +++ b/tox.ini @@ -25,10 +25,9 @@ commands = nosetests -sv st2common/tests/unit nosetests -sv st2reactor/tests/unit nosetests -sv st2tests/tests/unit - -[testenv:py36] +[testenv:py36-unit] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orchestra:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orchestra:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv @@ -37,10 +36,11 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ + nosetests --with-timer --rednose -sv st2debug/tests/unit/ + nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ - nosetests --with-timer --rednose -sv st2reactor/tests/integration/ --ignore-files=test_garbage_collector.* nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ - nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ + nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/exp/ nosetests --with-timer --rednose -sv st2common/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/ @@ -54,6 +54,23 @@ commands = nosetests --with-timer --rednose -sv contrib/runners/python_runner/tests/unit/ contrib/runners/python_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/windows_runner/tests/unit/ +[testenv:py36-integration] +basepython = python3.6 +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner + VIRTUALENV_DIR = {envdir} +install_command = pip install -U --force-reinstall {opts} {packages} +deps = virtualenv + -r{toxinidir}/requirements.txt + -e{toxinidir}/st2client + -e{toxinidir}/st2common +commands = + nosetests --with-timer --rednose -sv st2actions/tests/integration/ + nosetests --with-timer --rednose -sv st2api/tests/integration/ + nosetests --with-timer --rednose -sv st2common/tests/integration/ + nosetests --with-timer --rednose -sv st2debug/tests/integration/ + nosetests --with-timer --rednose -sv st2exporter/tests/integration/ + nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + [testenv:venv] commands = {posargs}