From 9ebd3c132e969755217d083188967c5e70c5bacf Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 17:05:02 +0200 Subject: [PATCH 01/21] Use alphabetic ordering for tests. --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 374d1dee67..0cc20e81c1 100644 --- a/tox.ini +++ b/tox.ini @@ -35,13 +35,14 @@ deps = virtualenv -e{toxinidir}/st2client -e{toxinidir}/st2common commands = + nosetests --with-timer --rednose -sv st2actions/tests/unit/ + nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ + nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/exp/ + nosetests --with-timer --rednose -sv st2common/tests/unit/ nosetests --with-timer --rednose -sv st2client/tests/unit/ nosetests --with-timer --rednose -sv st2debug/tests/unit/ nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ - nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ - nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/exp/ - nosetests --with-timer --rednose -sv st2common/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/inquirer_runner/tests/unit/ From 8ec13949a637c366a14ac22e8320ca6c6ae5ba42 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 17:47:15 +0200 Subject: [PATCH 02/21] Various Python 3 fixes for st2actions tests. --- st2actions/tests/unit/test_paramiko_ssh.py | 9 +++++---- .../tests/unit/test_runner_container.py | 4 ++++ st2common/st2common/runners/paramiko_ssh.py | 20 +++++++++++++++++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/st2actions/tests/unit/test_paramiko_ssh.py b/st2actions/tests/unit/test_paramiko_ssh.py index efba4bfe63..c62ff54016 100644 --- a/st2actions/tests/unit/test_paramiko_ssh.py +++ b/st2actions/tests/unit/test_paramiko_ssh.py @@ -514,13 +514,14 @@ def test_consume_stdout(self): chan = Mock() chan.recv_ready.side_effect = [True, True, True, True, False] - chan.recv.side_effect = ['\xF0', '\x90', '\x8D', '\x88'] + chan.recv.side_effect = [b'\xF0', b'\x90', b'\x8D', b'\x88'] try: - '\xF0'.decode('utf-8') + b'\xF0'.decode('utf-8') self.fail('Test fixture is not right.') except UnicodeDecodeError: pass stdout = mock._consume_stdout(chan) + self.assertEqual(u'\U00010348', stdout.getvalue()) @patch('paramiko.SSHClient', Mock) @@ -537,9 +538,9 @@ def test_consume_stderr(self): chan = Mock() chan.recv_stderr_ready.side_effect = [True, True, True, True, False] - chan.recv_stderr.side_effect = ['\xF0', '\x90', '\x8D', '\x88'] + chan.recv_stderr.side_effect = [b'\xF0', b'\x90', b'\x8D', b'\x88'] try: - '\xF0'.decode('utf-8') + b'\xF0'.decode('utf-8') self.fail('Test fixture is not right.') except UnicodeDecodeError: pass diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index 60d1c8a60f..805badbede 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -14,7 +14,10 @@ # limitations under the License. from __future__ import absolute_import + +import six import mock +import unittest2 from bson.errors import InvalidStringData from oslo_config import cfg @@ -296,6 +299,7 @@ def test_dispatch_unsupported_status(self): liveaction_db ) + @unittest2.skipIf(six.PY3, 'non-utf8 works fine in MongoDB under Python 3') @mock.patch.object(LocalShellCommandRunner, 'run', mock.MagicMock( return_value=(action_constants.LIVEACTION_STATUS_SUCCEEDED, NON_UTF8_RESULT, None))) @mock.patch('st2common.runners.base.register_runner', diff --git a/st2common/st2common/runners/paramiko_ssh.py b/st2common/st2common/runners/paramiko_ssh.py index d0f18692c5..a7cfa45809 100644 --- a/st2common/st2common/runners/paramiko_ssh.py +++ b/st2common/st2common/runners/paramiko_ssh.py @@ -59,8 +59,8 @@ def __init__(self, cmd, timeout, stdout=None, stderr=None): self.timeout = timeout self.stdout = stdout self.stderr = stderr - message = 'Command didn\'t finish in %s seconds' % (timeout) - super(SSHCommandTimeoutError, self).__init__(message) + self.message = 'Command didn\'t finish in %s seconds' % (timeout) + super(SSHCommandTimeoutError, self).__init__(self.message) def __repr__(self): return ('' % @@ -476,6 +476,10 @@ def _consume_stdout(self, chan, call_line_handler_func=False): if chan.recv_ready(): data = chan.recv(self.CHUNK_SIZE) + + if six.PY3 and isinstance(data, six.text_type): + data = data.encode('utf-8') + out += data while data: @@ -485,6 +489,10 @@ def _consume_stdout(self, chan, call_line_handler_func=False): break data = chan.recv(self.CHUNK_SIZE) + + if six.PY3 and isinstance(data, six.text_type): + data = data.encode('utf-8') + out += data stdout.write(self._get_decoded_data(out)) @@ -514,6 +522,10 @@ def _consume_stderr(self, chan, call_line_handler_func=False): if chan.recv_stderr_ready(): data = chan.recv_stderr(self.CHUNK_SIZE) + + if six.PY3 and isinstance(data, six.text_type): + data = data.encode('utf-8') + out += data while data: @@ -523,6 +535,10 @@ def _consume_stderr(self, chan, call_line_handler_func=False): break data = chan.recv_stderr(self.CHUNK_SIZE) + + if six.PY3 and isinstance(data, six.text_type): + data = data.encode('utf-8') + out += data stderr.write(self._get_decoded_data(out)) From e21a990c2ca9037906df92195d0bc04fae06c893 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 17:59:40 +0200 Subject: [PATCH 03/21] Run st2auth tests under Python 3. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0cc20e81c1..e5a5703bf8 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,7 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --with-timer --rednose -sv st2actions/tests/unit/ + nosetests --with-timer --rednose -sv st2auth/tests/unit/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/exp/ nosetests --with-timer --rednose -sv st2common/tests/unit/ From 3a9f53295ff6773f14706f5b05f05cc8bb129960 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 18:01:07 +0200 Subject: [PATCH 04/21] Python 3 compatibility fixes for code in st2auth/. --- st2auth/st2auth/handlers.py | 12 ++++++++++-- st2auth/tests/unit/controllers/v1/test_token.py | 2 +- st2auth/tests/unit/test_handlers.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/st2auth/st2auth/handlers.py b/st2auth/st2auth/handlers.py index 167d9ea162..46127bbca0 100644 --- a/st2auth/st2auth/handlers.py +++ b/st2auth/st2auth/handlers.py @@ -14,6 +14,8 @@ # limitations under the License. import base64 + +import six from six.moves import http_client from oslo_config import cfg @@ -117,7 +119,7 @@ def handle_auth(self, request, headers=None, remote_addr=None, ttl=ttl) except TTLTooLargeException as e: abort_request(status_code=http_client.BAD_REQUEST, - message=e.message) + message=str(e)) return token LOG.audit('Access denied to anonymous user.', extra=extra) @@ -154,7 +156,7 @@ def handle_auth(self, request, headers=None, remote_addr=None, remote_user=None, abort_request() return - split = auth_value.split(':', 1) + split = auth_value.split(b':', 1) if len(split) != 2: LOG.audit('Invalid authorization header', extra=extra) abort_request() @@ -162,6 +164,12 @@ def handle_auth(self, request, headers=None, remote_addr=None, remote_user=None, username, password = split + if six.PY3 and isinstance(username, six.binary_type): + username = username.decode('utf-8') + + if six.PY3 and isinstance(password, six.binary_type): + password = password.decode('utf-8') + result = self._auth_backend.authenticate(username=username, password=password) if result is True: diff --git a/st2auth/tests/unit/controllers/v1/test_token.py b/st2auth/tests/unit/controllers/v1/test_token.py index 0bcf857079..f3b61a9e18 100644 --- a/st2auth/tests/unit/controllers/v1/test_token.py +++ b/st2auth/tests/unit/controllers/v1/test_token.py @@ -29,7 +29,7 @@ from st2common.persistence.auth import User, Token, ApiKey -USERNAME = ''.join(random.choice(string.lowercase) for i in range(10)) +USERNAME = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) TOKEN_DEFAULT_PATH = '/tokens' TOKEN_V1_PATH = '/v1/tokens' TOKEN_VERIFY_PATH = '/v1/tokens/validate' diff --git a/st2auth/tests/unit/test_handlers.py b/st2auth/tests/unit/test_handlers.py index cebeafc99e..36444f9f75 100644 --- a/st2auth/tests/unit/test_handlers.py +++ b/st2auth/tests/unit/test_handlers.py @@ -245,7 +245,7 @@ def test_password_contains_colon(self): h = handlers.StandaloneAuthHandler() request = MockRequest(60) - authorization = ('Basic', base64.b64encode('username:password:password')) + authorization = ('Basic', base64.b64encode(b'username:password:password')) token = h.handle_auth( request, headers={}, remote_addr=None, remote_user=None, authorization=authorization) From 4cc8b495c3cbe140fbcce214d92c835e2be7bdd7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 20 Jun 2018 19:25:02 +0200 Subject: [PATCH 05/21] Fix stream tests so they work under Python 3. --- st2stream/st2stream/controllers/v1/stream.py | 5 +++-- .../controllers/v1/test_rbac_for_supported_endpoints.py | 2 +- st2stream/tests/unit/controllers/v1/test_stream.py | 6 ++++-- tox.ini | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/st2stream/st2stream/controllers/v1/stream.py b/st2stream/st2stream/controllers/v1/stream.py index 95112afea5..477528d938 100644 --- a/st2stream/st2stream/controllers/v1/stream.py +++ b/st2stream/st2stream/controllers/v1/stream.py @@ -45,11 +45,12 @@ def format(gen): for pack in gen: if not pack: # Note: gunicorn wsgi handler expect bytes, not unicode - yield six.binary_type('\n') + yield six.binary_type(b'\n') else: (event, body) = pack # Note: gunicorn wsgi handler expect bytes, not unicode - yield six.binary_type(message % (event, json_encode(body, indent=None))) + yield six.binary_type((message % (event, json_encode(body, + indent=None))).encode('utf-8')) class StreamController(object): diff --git a/st2stream/tests/unit/controllers/v1/test_rbac_for_supported_endpoints.py b/st2stream/tests/unit/controllers/v1/test_rbac_for_supported_endpoints.py index 016fc6d7e5..38331ee0f5 100644 --- a/st2stream/tests/unit/controllers/v1/test_rbac_for_supported_endpoints.py +++ b/st2stream/tests/unit/controllers/v1/test_rbac_for_supported_endpoints.py @@ -19,7 +19,7 @@ from st2common.models.db.rbac import UserRoleAssignmentDB from st2common.rbac.types import PermissionType -from base import APIControllerWithRBACTestCase +from .base import APIControllerWithRBACTestCase http_client = six.moves.http_client diff --git a/st2stream/tests/unit/controllers/v1/test_stream.py b/st2stream/tests/unit/controllers/v1/test_stream.py index 9e5f88fccf..9625a482a4 100644 --- a/st2stream/tests/unit/controllers/v1/test_stream.py +++ b/st2stream/tests/unit/controllers/v1/test_stream.py @@ -29,7 +29,8 @@ import st2common.stream.listener from st2stream.controllers.v1 import stream from st2tests.api import SUPER_SECRET_PARAMETER -from base import FunctionalTest + +from .base import FunctionalTest RUNNER_TYPE_1 = { @@ -138,6 +139,7 @@ def test_get_all(self): message = None for message in resp._app_iter: + message = message.decode('utf-8') if message != '\n': break process(LiveActionDB(**LIVE_ACTION_1), META()) @@ -165,7 +167,7 @@ def dispatch_and_handle_mock_data(resp): received_messages_data = '' for index, message in enumerate(resp._app_iter): if message.strip(): - received_messages_data += message + received_messages_data += message.decode('utf-8') # Dispatch some mock events if index == 0: diff --git a/tox.ini b/tox.ini index e5a5703bf8..ec187b65de 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,7 @@ commands = nosetests --with-timer --rednose -sv st2debug/tests/unit/ nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ + nosetests --with-timer --rednose -sv st2stream/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/inquirer_runner/tests/unit/ From 20b3e64aba2dcb9a1cb897b6da75520ff7b04fc8 Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Thu, 21 Jun 2018 09:42:41 +1000 Subject: [PATCH 06/21] Add jinja filter to manipulate paths --- st2common/st2common/jinja/filters/path.py | 29 +++++++++++ .../unit/test_jinja_render_path_filters.py | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 st2common/st2common/jinja/filters/path.py create mode 100644 st2common/tests/unit/test_jinja_render_path_filters.py diff --git a/st2common/st2common/jinja/filters/path.py b/st2common/st2common/jinja/filters/path.py new file mode 100644 index 0000000000..7c7cc73388 --- /dev/null +++ b/st2common/st2common/jinja/filters/path.py @@ -0,0 +1,29 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import os + +__all__ = [ + 'basename', + 'dirname' +] + + +def basename(path): + return os.path.basename(path) + +def dirname(path): + return os.path.dirname(path) \ No newline at end of file diff --git a/st2common/tests/unit/test_jinja_render_path_filters.py b/st2common/tests/unit/test_jinja_render_path_filters.py new file mode 100644 index 0000000000..a459f924de --- /dev/null +++ b/st2common/tests/unit/test_jinja_render_path_filters.py @@ -0,0 +1,49 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import +import unittest2 + +from st2common.util import jinja as jinja_utils + + +class JinjaUtilsPathFilterTestCase(unittest2.TestCase): + + def test_basename(self): + env = jinja_utils.get_jinja_environment() + + template = '{{k1 | basename}}' + actual = env.from_string(template).render({'k1': '/some/path/to/file.txt'}) + self.assertEqual(actual, 'file.txt') + + actual = env.from_string(template).render({'k1': '/some/path/to/dir'}) + self.assertEqual(actual, 'dir') + + actual = env.from_string(template).render({'k1': '/some/path/to/dir/'}) + self.assertEqual(actual, 'dir') + + def test_dirname(self): + env = jinja_utils.get_jinja_environment() + + template = '{{k1 | dirname}}' + actual = env.from_string(template).render({'k1': '/some/path/to/file.txt'}) + self.assertEqual(actual, '/some/path/to') + + actual = env.from_string(template).render({'k1': '/some/path/to/dir'}) + self.assertEqual(actual, '/some/path/to') + + actual = env.from_string(template).render({'k1': '/some/path/to/dir/'}) + self.assertEqual(actual, '/some/path/to') From b1c992aa4144b94adc6b254660b04a3a3fa6edcd Mon Sep 17 00:00:00 2001 From: Florian Reisinger Date: Thu, 21 Jun 2018 12:16:11 +1000 Subject: [PATCH 07/21] Fix flake8 errors --- st2common/st2common/jinja/filters/path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/jinja/filters/path.py b/st2common/st2common/jinja/filters/path.py index 7c7cc73388..c84500e1af 100644 --- a/st2common/st2common/jinja/filters/path.py +++ b/st2common/st2common/jinja/filters/path.py @@ -25,5 +25,6 @@ def basename(path): return os.path.basename(path) + def dirname(path): - return os.path.dirname(path) \ No newline at end of file + return os.path.dirname(path) From 3c8ba02786e18b766053ba0aba801fcb492664f5 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 15:52:36 +0200 Subject: [PATCH 08/21] Fix PYTHONPATH. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ec187b65de..d0a520cfa5 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ commands = nosetests -sv st2tests/tests/unit [testenv:py36-unit] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orchestra:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/orchestra:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv @@ -59,7 +59,7 @@ commands = [testenv:py36-integration] basepython = python3.6 -setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_script_runner:{toxinidir}/contrib/runners/remote_command_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner +setenv = PYTHONPATH = {toxinidir}/external:{toxinidir}/st2common:{toxinidir}/st2auth:{toxinidir}/st2api:{toxinidir}/st2actions:{toxinidir}/st2exporter:{toxinidir}/st2reactor:{toxinidir}/st2tests:{toxinidir}/contrib/runners/action_chain_runner:{toxinidir}/contrib/runners/local_runner:{toxinidir}/contrib/runners/windows_runner:{toxinidir}/contrib/runners/python_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/noop_runner:{toxinidir}/contrib/runners/announcement_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/remote_runner:{toxinidir}/contrib/runners/mistral_v2:{toxinidir}/contrib/runners/inquirer_runner:{toxinidir}/contrib/runners/http_runner:{toxinidir}/contrib/runners/cloudslang_runner VIRTUALENV_DIR = {envdir} install_command = pip install -U --force-reinstall {opts} {packages} deps = virtualenv From 793b567a8809e98303958f1ae8ff8b1d955240a9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 16:10:42 +0200 Subject: [PATCH 09/21] Move runner integration tests to py36-integration tox task. --- tox.ini | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index d0a520cfa5..474e085066 100644 --- a/tox.ini +++ b/tox.ini @@ -45,16 +45,16 @@ commands = nosetests --with-timer --rednose -sv st2exporter/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2stream/tests/unit/ - nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/inquirer_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/announcement_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/http_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/noop_runner/tests/unit/ - nosetests --with-timer --rednose -sv contrib/runners/local_runner/tests/unit/ contrib/runners/local_runner/tests/integration/ - nosetests --with-timer --rednose -sv contrib/runners/mistral_v2/tests/unit/ contrib/runners/mistral_v2/tests/integration/ - nosetests --with-timer --rednose -sv contrib/runners/orchestra_runner/tests/unit/ contrib/runners/orchestra_runner/tests/integration/ - nosetests --with-timer --rednose -sv contrib/runners/python_runner/tests/unit/ contrib/runners/python_runner/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/local_runner/tests/unit/ + nosetests --with-timer --rednose -sv contrib/runners/mistral_v2/tests/unit/ + nosetests --with-timer --rednose -sv contrib/runners/orchestra_runner/tests/unit/ + nosetests --with-timer --rednose -sv contrib/runners/python_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/windows_runner/tests/unit/ [testenv:py36-integration] @@ -73,6 +73,11 @@ commands = nosetests --with-timer --rednose -sv st2debug/tests/integration/ nosetests --with-timer --rednose -sv st2exporter/tests/integration/ nosetests --with-timer --rednose -sv st2reactor/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/local_runner/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/mistral_v2/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/orchestra_runner/tests/integration/ + nosetests --with-timer --rednose -sv contrib/runners/python_runner/tests/integration/ [testenv:venv] commands = {posargs} From 99d0f3d62941c75fa1fcb9f43c4f420f84fa2205 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 16:29:06 +0200 Subject: [PATCH 10/21] Add two new jinja filters to st2common.util.jinja.get_filters function dictionary. --- st2common/st2common/util/jinja.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/jinja.py b/st2common/st2common/util/jinja.py index e0f0cb0b41..4d7053bfb8 100644 --- a/st2common/st2common/util/jinja.py +++ b/st2common/st2common/util/jinja.py @@ -62,6 +62,7 @@ def get_filters(): from st2common.jinja.filters import version from st2common.jinja.filters import json_escape from st2common.jinja.filters import jsonpath_query + from st2common.jinja.filters import path # IMPORTANT NOTE - these filters were recently duplicated in st2mistral so that # they are also available in Mistral workflows. Please ensure any additions you @@ -94,7 +95,10 @@ def get_filters(): 'use_none': use_none, 'json_escape': json_escape.json_escape, - 'jsonpath_query': jsonpath_query.jsonpath_query + 'jsonpath_query': jsonpath_query.jsonpath_query, + + 'basename': path.basename, + 'dirname': path.dirname } From cc9bc445f73050378824a28485dc5357b7bf73df Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 16:30:58 +0200 Subject: [PATCH 11/21] Add changelog entry. --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f0e1b92bdd..0cc3dad19a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,6 +36,10 @@ Added pack in question needs to support Python 3. Note 2: This feature is experimental and opt-in. (new feature) #4016 #3922 #4149 +* Add two new Jinja filters - ``basename`` (``os.path.basename``) and ``dirname`` + (``os.path.dirname``). #4184 + + Contributed by Florian Reisinger (@reisingerf). Changed ~~~~~~~ From 90cd02fd2e22f15e5d1d69dcb36c9132b1c0ac18 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 17:06:59 +0200 Subject: [PATCH 12/21] Setup user for integration tests. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e3083f210c..9a42811a52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: install: - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. From 069d429d6b0be74768dcfb8476badea0fad68a4b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 17:10:54 +0200 Subject: [PATCH 13/21] Increase threshold since Python 3 tests are sometimes a bit slower. --- .../tests/integration/test_python_action_process_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py index 30721dce30..37e62c91d6 100644 --- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py +++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py @@ -47,7 +47,7 @@ ] # Maximum limit for the process wrapper script execution time (in seconds) -WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.30 +WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.31 ASSERTION_ERROR_MESSAGE = (""" Python wrapper process script took more than %s seconds to execute (%s). This most likely means From 4f6532a6b6253cb9a1922c3257280e8d1bfcc394 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 17:18:53 +0200 Subject: [PATCH 14/21] Fix / update tests. --- st2common/tests/unit/test_jinja_render_path_filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/tests/unit/test_jinja_render_path_filters.py b/st2common/tests/unit/test_jinja_render_path_filters.py index a459f924de..f6bb66689e 100644 --- a/st2common/tests/unit/test_jinja_render_path_filters.py +++ b/st2common/tests/unit/test_jinja_render_path_filters.py @@ -33,7 +33,7 @@ def test_basename(self): self.assertEqual(actual, 'dir') actual = env.from_string(template).render({'k1': '/some/path/to/dir/'}) - self.assertEqual(actual, 'dir') + self.assertEqual(actual, '') def test_dirname(self): env = jinja_utils.get_jinja_environment() @@ -46,4 +46,4 @@ def test_dirname(self): self.assertEqual(actual, '/some/path/to') actual = env.from_string(template).render({'k1': '/some/path/to/dir/'}) - self.assertEqual(actual, '/some/path/to') + self.assertEqual(actual, '/some/path/to/dir') From 024d5453ff3c14af3d4afabbf3e90117930ecc91 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 17:27:55 +0200 Subject: [PATCH 15/21] Update travis config. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a42811a52..19cdd17744 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: install: - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. From ffa0b83a1219b5326692da61cef67a1912d82c35 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 17:41:51 +0200 Subject: [PATCH 16/21] System user also needs to be set up for py3 unit tests. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 19cdd17744..1cb3ba13b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: install: - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. From 7967d0f51d4c121b1e002a7c6a9e371eee55d7e2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 18:54:26 +0200 Subject: [PATCH 17/21] Include more context on failure. --- st2actions/tests/unit/test_worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2actions/tests/unit/test_worker.py b/st2actions/tests/unit/test_worker.py index 43f7a3d762..0567a6036f 100644 --- a/st2actions/tests/unit/test_worker.py +++ b/st2actions/tests/unit/test_worker.py @@ -126,7 +126,8 @@ def test_worker_shutdown(self): # Verify that _running_liveactions is empty and the liveaction is abandoned. self.assertEqual(len(action_worker._running_liveactions), 0) - self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_ABANDONED) + self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_ABANDONED, + str(liveaction_db)) # Make sure the temporary file has been deleted. self.assertFalse(os.path.isfile(temp_file)) From 7af0ba2b1bd17e6ef25e2d9d135e0250f30e0666 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 19:00:36 +0200 Subject: [PATCH 18/21] Fix double escaping of the file name. --- st2actions/tests/unit/test_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2actions/tests/unit/test_worker.py b/st2actions/tests/unit/test_worker.py index 0567a6036f..197c142f7c 100644 --- a/st2actions/tests/unit/test_worker.py +++ b/st2actions/tests/unit/test_worker.py @@ -106,7 +106,7 @@ def test_worker_shutdown(self): self.assertTrue(os.path.isfile(temp_file)) # Launch the action execution in a separate thread. - params = {'cmd': 'while [ -e \'%s\' ]; do sleep 0.1; done' % temp_file} + params = {'cmd': 'while [ -e %s ]; do sleep 0.1; done' % temp_file} liveaction_db = self._get_liveaction_model(WorkerTestCase.local_action_db, params) liveaction_db = LiveAction.add_or_update(liveaction_db) executions.create_execution_object(liveaction_db) From 40184a9f07ef3ff13758e4bf366f5a34dde84ab7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 19:08:11 +0200 Subject: [PATCH 19/21] Revert "Fix double escaping of the file name." This reverts commit 7af0ba2b1bd17e6ef25e2d9d135e0250f30e0666. --- st2actions/tests/unit/test_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2actions/tests/unit/test_worker.py b/st2actions/tests/unit/test_worker.py index 197c142f7c..0567a6036f 100644 --- a/st2actions/tests/unit/test_worker.py +++ b/st2actions/tests/unit/test_worker.py @@ -106,7 +106,7 @@ def test_worker_shutdown(self): self.assertTrue(os.path.isfile(temp_file)) # Launch the action execution in a separate thread. - params = {'cmd': 'while [ -e %s ]; do sleep 0.1; done' % temp_file} + params = {'cmd': 'while [ -e \'%s\' ]; do sleep 0.1; done' % temp_file} liveaction_db = self._get_liveaction_model(WorkerTestCase.local_action_db, params) liveaction_db = LiveAction.add_or_update(liveaction_db) executions.create_execution_object(liveaction_db) From 8c52009fe44039291e4deabcaba297b7c804053a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 21 Jun 2018 19:09:15 +0200 Subject: [PATCH 20/21] Fix syntax. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1cb3ba13b5..0d79960b38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ before_install: install: - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. From b35f90a6f0539550553481fecca055a749dcc7f1 Mon Sep 17 00:00:00 2001 From: Winson Chan Date: Thu, 21 Jun 2018 18:25:37 +0000 Subject: [PATCH 21/21] Add st2workflowengine to st2ctl Add st2workflowengine to the list of service components in st2ctl. --- st2common/bin/st2ctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index 90bed7364e..db2a55cf09 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops mistral" +COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops mistral st2workflowengine" ST2_CONF="/etc/st2/st2.conf" # Ensure global environment is sourced if exists