Merge branch 'master' into warn-and-fix-config-file-permissions

.travis.yml

@@ -59,7 +59,7 @@ before_install:
 install:
   - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi
   - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ]; then pip install codecov; fi
-  - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ]; then sudo .circle/add-itest-user.sh; fi
+  - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi
 
 # Let's enable rabbitmqadmin
 # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis.

CHANGELOG.rst

@@ -36,6 +36,10 @@ Added
   pack in question needs to support Python 3.
 
   Note 2: This feature is experimental and opt-in. (new feature) #4016 #3922 #4149
+* Add two new Jinja filters - ``basename`` (``os.path.basename``) and ``dirname``
+  (``os.path.dirname``). #4184
+
+  Contributed by Florian Reisinger (@reisingerf).
 
 Changed
 ~~~~~~~

contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py

@@ -47,7 +47,7 @@
 ]
 
 # Maximum limit for the process wrapper script execution time (in seconds)
-WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.30
+WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.31
 
 ASSERTION_ERROR_MESSAGE = ("""
 Python wrapper process script took more than %s seconds to execute (%s). This most likely means

st2actions/tests/unit/test_paramiko_ssh.py

@@ -514,13 +514,14 @@ def test_consume_stdout(self):
         chan = Mock()
         chan.recv_ready.side_effect = [True, True, True, True, False]
 
-        chan.recv.side_effect = ['\xF0', '\x90', '\x8D', '\x88']
+        chan.recv.side_effect = [b'\xF0', b'\x90', b'\x8D', b'\x88']
         try:
-            '\xF0'.decode('utf-8')
+            b'\xF0'.decode('utf-8')
             self.fail('Test fixture is not right.')
         except UnicodeDecodeError:
             pass
         stdout = mock._consume_stdout(chan)
+
         self.assertEqual(u'\U00010348', stdout.getvalue())
 
     @patch('paramiko.SSHClient', Mock)
@@ -537,9 +538,9 @@ def test_consume_stderr(self):
         chan = Mock()
         chan.recv_stderr_ready.side_effect = [True, True, True, True, False]
 
-        chan.recv_stderr.side_effect = ['\xF0', '\x90', '\x8D', '\x88']
+        chan.recv_stderr.side_effect = [b'\xF0', b'\x90', b'\x8D', b'\x88']
         try:
-            '\xF0'.decode('utf-8')
+            b'\xF0'.decode('utf-8')
             self.fail('Test fixture is not right.')
         except UnicodeDecodeError:
             pass

st2actions/tests/unit/test_runner_container.py

@@ -14,7 +14,10 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
+import six
 import mock
+import unittest2
 
 from bson.errors import InvalidStringData
 from oslo_config import cfg
@@ -296,6 +299,7 @@ def test_dispatch_unsupported_status(self):
             liveaction_db
         )
 
+    @unittest2.skipIf(six.PY3, 'non-utf8 works fine in MongoDB under Python 3')
     @mock.patch.object(LocalShellCommandRunner, 'run', mock.MagicMock(
         return_value=(action_constants.LIVEACTION_STATUS_SUCCEEDED, NON_UTF8_RESULT, None)))
     @mock.patch('st2common.runners.base.register_runner',

st2actions/tests/unit/test_worker.py

@@ -126,7 +126,8 @@ def test_worker_shutdown(self):
 
             # Verify that _running_liveactions is empty and the liveaction is abandoned.
             self.assertEqual(len(action_worker._running_liveactions), 0)
-            self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_ABANDONED)
+            self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_ABANDONED,
+                             str(liveaction_db))
 
         # Make sure the temporary file has been deleted.
         self.assertFalse(os.path.isfile(temp_file))

st2auth/st2auth/handlers.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 import base64
+
+import six
 from six.moves import http_client
 from oslo_config import cfg
 
@@ -117,7 +119,7 @@ def handle_auth(self, request, headers=None, remote_addr=None,
                                                     ttl=ttl)
             except TTLTooLargeException as e:
                 abort_request(status_code=http_client.BAD_REQUEST,
-                              message=e.message)
+                              message=str(e))
             return token
 
         LOG.audit('Access denied to anonymous user.', extra=extra)
@@ -154,14 +156,20 @@ def handle_auth(self, request, headers=None, remote_addr=None, remote_user=None,
             abort_request()
             return
 
-        split = auth_value.split(':', 1)
+        split = auth_value.split(b':', 1)
         if len(split) != 2:
             LOG.audit('Invalid authorization header', extra=extra)
             abort_request()
             return
 
         username, password = split
 
+        if six.PY3 and isinstance(username, six.binary_type):
+            username = username.decode('utf-8')
+
+        if six.PY3 and isinstance(password, six.binary_type):
+            password = password.decode('utf-8')
+
         result = self._auth_backend.authenticate(username=username, password=password)
 
         if result is True:

st2auth/tests/unit/controllers/v1/test_token.py

@@ -29,7 +29,7 @@
 from st2common.persistence.auth import User, Token, ApiKey
 
 
-USERNAME = ''.join(random.choice(string.lowercase) for i in range(10))
+USERNAME = ''.join(random.choice(string.ascii_lowercase) for i in range(10))
 TOKEN_DEFAULT_PATH = '/tokens'
 TOKEN_V1_PATH = '/v1/tokens'
 TOKEN_VERIFY_PATH = '/v1/tokens/validate'

st2auth/tests/unit/test_handlers.py

@@ -245,7 +245,7 @@ def test_password_contains_colon(self):
         h = handlers.StandaloneAuthHandler()
         request = MockRequest(60)
 
-        authorization = ('Basic', base64.b64encode('username:password:password'))
+        authorization = ('Basic', base64.b64encode(b'username:password:password'))
         token = h.handle_auth(
             request, headers={}, remote_addr=None,
             remote_user=None, authorization=authorization)

st2common/bin/st2ctl

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops mistral"
+COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2resultstracker st2rulesengine st2sensorcontainer st2chatops mistral st2workflowengine"
 ST2_CONF="/etc/st2/st2.conf"
 
 # Ensure global environment is sourced if exists

st2common/st2common/jinja/filters/path.py

@@ -0,0 +1,30 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+
+__all__ = [
+    'basename',
+    'dirname'
+]
+
+
+def basename(path):
+    return os.path.basename(path)
+
+
+def dirname(path):
+    return os.path.dirname(path)

st2common/st2common/runners/paramiko_ssh.py

@@ -59,8 +59,8 @@ def __init__(self, cmd, timeout, stdout=None, stderr=None):
         self.timeout = timeout
         self.stdout = stdout
         self.stderr = stderr
-        message = 'Command didn\'t finish in %s seconds' % (timeout)
-        super(SSHCommandTimeoutError, self).__init__(message)
+        self.message = 'Command didn\'t finish in %s seconds' % (timeout)
+        super(SSHCommandTimeoutError, self).__init__(self.message)
 
     def __repr__(self):
         return ('<SSHCommandTimeoutError: cmd="%s",timeout=%s)>' %
@@ -476,6 +476,10 @@ def _consume_stdout(self, chan, call_line_handler_func=False):
 
         if chan.recv_ready():
             data = chan.recv(self.CHUNK_SIZE)
+
+            if six.PY3 and isinstance(data, six.text_type):
+                data = data.encode('utf-8')
+
             out += data
 
             while data:
@@ -485,6 +489,10 @@ def _consume_stdout(self, chan, call_line_handler_func=False):
                     break
 
                 data = chan.recv(self.CHUNK_SIZE)
+
+                if six.PY3 and isinstance(data, six.text_type):
+                    data = data.encode('utf-8')
+
                 out += data
 
         stdout.write(self._get_decoded_data(out))
@@ -514,6 +522,10 @@ def _consume_stderr(self, chan, call_line_handler_func=False):
 
         if chan.recv_stderr_ready():
             data = chan.recv_stderr(self.CHUNK_SIZE)
+
+            if six.PY3 and isinstance(data, six.text_type):
+                data = data.encode('utf-8')
+
             out += data
 
             while data:
@@ -523,6 +535,10 @@ def _consume_stderr(self, chan, call_line_handler_func=False):
                     break
 
                 data = chan.recv_stderr(self.CHUNK_SIZE)
+
+                if six.PY3 and isinstance(data, six.text_type):
+                    data = data.encode('utf-8')
+
                 out += data
 
         stderr.write(self._get_decoded_data(out))

st2common/st2common/util/jinja.py

@@ -62,6 +62,7 @@ def get_filters():
     from st2common.jinja.filters import version
     from st2common.jinja.filters import json_escape
     from st2common.jinja.filters import jsonpath_query
+    from st2common.jinja.filters import path
 
     # IMPORTANT NOTE - these filters were recently duplicated in st2mistral so that
     # they are also available in Mistral workflows. Please ensure any additions you
@@ -94,7 +95,10 @@ def get_filters():
         'use_none': use_none,
 
         'json_escape': json_escape.json_escape,
-        'jsonpath_query': jsonpath_query.jsonpath_query
+        'jsonpath_query': jsonpath_query.jsonpath_query,
+
+        'basename': path.basename,
+        'dirname': path.dirname
     }
 
 

st2common/tests/unit/test_jinja_render_path_filters.py

@@ -0,0 +1,49 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import absolute_import
+import unittest2
+
+from st2common.util import jinja as jinja_utils
+
+
+class JinjaUtilsPathFilterTestCase(unittest2.TestCase):
+
+    def test_basename(self):
+        env = jinja_utils.get_jinja_environment()
+
+        template = '{{k1 | basename}}'
+        actual = env.from_string(template).render({'k1': '/some/path/to/file.txt'})
+        self.assertEqual(actual, 'file.txt')
+
+        actual = env.from_string(template).render({'k1': '/some/path/to/dir'})
+        self.assertEqual(actual, 'dir')
+
+        actual = env.from_string(template).render({'k1': '/some/path/to/dir/'})
+        self.assertEqual(actual, '')
+
+    def test_dirname(self):
+        env = jinja_utils.get_jinja_environment()
+
+        template = '{{k1 | dirname}}'
+        actual = env.from_string(template).render({'k1': '/some/path/to/file.txt'})
+        self.assertEqual(actual, '/some/path/to')
+
+        actual = env.from_string(template).render({'k1': '/some/path/to/dir'})
+        self.assertEqual(actual, '/some/path/to')
+
+        actual = env.from_string(template).render({'k1': '/some/path/to/dir/'})
+        self.assertEqual(actual, '/some/path/to/dir')

st2stream/st2stream/controllers/v1/stream.py

@@ -45,11 +45,12 @@ def format(gen):
     for pack in gen:
         if not pack:
             # Note: gunicorn wsgi handler expect bytes, not unicode
-            yield six.binary_type('\n')
+            yield six.binary_type(b'\n')
         else:
             (event, body) = pack
             # Note: gunicorn wsgi handler expect bytes, not unicode
-            yield six.binary_type(message % (event, json_encode(body, indent=None)))
+            yield six.binary_type((message % (event, json_encode(body,
+                                                                 indent=None))).encode('utf-8'))
 
 
 class StreamController(object):

st2stream/tests/unit/controllers/v1/test_rbac_for_supported_endpoints.py

@@ -19,7 +19,7 @@
 from st2common.models.db.rbac import UserRoleAssignmentDB
 from st2common.rbac.types import PermissionType
 
-from base import APIControllerWithRBACTestCase
+from .base import APIControllerWithRBACTestCase
 
 http_client = six.moves.http_client
 

st2stream/tests/unit/controllers/v1/test_stream.py

@@ -29,7 +29,8 @@
 import st2common.stream.listener
 from st2stream.controllers.v1 import stream
 from st2tests.api import SUPER_SECRET_PARAMETER
-from base import FunctionalTest
+
+from .base import FunctionalTest
 
 
 RUNNER_TYPE_1 = {
@@ -138,6 +139,7 @@ def test_get_all(self):
         message = None
 
         for message in resp._app_iter:
+            message = message.decode('utf-8')
             if message != '\n':
                 break
             process(LiveActionDB(**LIVE_ACTION_1), META())
@@ -165,7 +167,7 @@ def dispatch_and_handle_mock_data(resp):
             received_messages_data = ''
             for index, message in enumerate(resp._app_iter):
                 if message.strip():
-                    received_messages_data += message
+                    received_messages_data += message.decode('utf-8')
 
                 # Dispatch some mock events
                 if index == 0: