Merge pull request #3766 from nmaludy/feature/json-jinja-filters

Implement new JSON Jinja filters

CHANGELOG.rst

@@ -35,7 +35,9 @@ Added
   StackStorm role mappings. This means that the same role can now be granted via multiple RBAC
   mapping files.
   #3763
-
+* Add new Jinja filters ``from_json_string``, ``from_yaml_string``, and ``jsonpath_query``.
+  #3763
+
 Fixed
 ~~~~~
 

contrib/examples/actions/workflows/tests/mistral-test-func-from-json-string.yaml

@@ -0,0 +1,16 @@
+version: '2.0'
+
+examples.mistral-test-func-from-json-string:
+    description: A workflow for testing from_json_string custom filter in mistral
+    type: direct
+    input:
+        - input_str
+    output:
+        result_jinja: <% $.result_jinja %>
+        result_yaql: <% $.result_yaql %>
+    tasks:
+        task1:
+            action: std.noop
+            publish:
+              result_jinja: "{{ from_json_string(_.input_str) }}"
+              result_yaql: '<% from_json_string($.input_str) %>'          

contrib/examples/actions/workflows/tests/mistral-test-func-from-yaml-string.yaml

@@ -0,0 +1,16 @@
+version: '2.0'
+
+examples.mistral-test-func-from-yaml-string:
+    description: A workflow for testing from_yaml_string custom filter in mistral
+    type: direct
+    input:
+        - input_str
+    output:
+        result_jinja: <% $.result_jinja %>
+        result_yaql: <% $.result_yaql %>
+    tasks:
+        task1:
+            action: std.noop
+            publish:
+              result_jinja: "{{ from_yaml_string(_.input_str) }}"
+              result_yaql: '<% from_yaml_string($.input_str) %>'          

contrib/examples/actions/workflows/tests/mistral-test-func-jsonpath-query.yaml

@@ -0,0 +1,18 @@
+version: '2.0'
+
+examples.mistral-test-func-jsonpath-query:
+    description: A workflow for testing jsonpath_query custom filter in mistral
+    type: direct
+    input:
+        - input_obj
+        - input_query
+    output:
+        result_jinja: <% $.result_jinja %>
+        result_yaql: <% $.result_yaql %>
+    tasks:
+
+        task2:
+            action: std.noop
+            publish:
+                result_jinja: '{{ jsonpath_query(_.input_obj, _.input_query) }}'
+                result_yaql: '<% jsonpath_query($.input_obj, $.input_query) %>'

st2common/st2common/jinja/filters/data.py

@@ -17,11 +17,21 @@
 import yaml
 
 __all__ = [
+    'from_json_string',
+    'from_yaml_string',
     'to_json_string',
     'to_yaml_string',
 ]
 
 
+def from_json_string(value):
+    return json.loads(value)
+
+
+def from_yaml_string(value):
+    return yaml.safe_load(value)
+
+
 def to_json_string(value, indent=4, sort_keys=False, separators=(',', ':')):
     return json.dumps(value, indent=indent, separators=separators,
                       sort_keys=sort_keys)

st2common/st2common/jinja/filters/jsonpath_query.py

@@ -0,0 +1,35 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jsonpath_rw
+
+__all__ = [
+    'jsonpath_query',
+]
+
+
+def jsonpath_query(value, query):
+    """Extracts data from an object `value` using a JSONPath `query`.
+    :link: https://github.com/kennknowles/python-jsonpath-rw
+    :param value: a object (dict, array, etc) to query
+    :param query: a JSONPath query expression (string)
+    :returns: the result of the query executed on the value
+    :rtype: dict, array, int, string, bool
+    """
+    expr = jsonpath_rw.parse(query)
+    matches = [match.value for match in expr.find(value)]
+    if not matches:
+        return None
+    return matches

st2common/st2common/util/jinja.py

@@ -60,12 +60,15 @@ def get_filters():
     from st2common.jinja.filters import time
     from st2common.jinja.filters import version
     from st2common.jinja.filters import json_escape
+    from st2common.jinja.filters import jsonpath_query
 
     # IMPORTANT NOTE - these filters were recently duplicated in st2mistral so that
     # they are also available in Mistral workflows. Please ensure any additions you
     # make here are also made there so that feature parity is maintained.
     return {
         'decrypt_kv': crypto.decrypt_kv,
+        'from_json_string': data.from_json_string,
+        'from_yaml_string': data.from_yaml_string,
         'to_json_string': data.to_json_string,
         'to_yaml_string': data.to_yaml_string,
 
@@ -89,7 +92,8 @@ def get_filters():
         'version_strip_patch': version.version_strip_patch,
         'use_none': use_none,
 
-        'json_escape': json_escape.json_escape
+        'json_escape': json_escape.json_escape,
+        'jsonpath_query': jsonpath_query.jsonpath_query
     }
 
 

st2common/tests/unit/test_jinja_render_data_filters.py

@@ -22,6 +22,32 @@
 
 class JinjaUtilsDataFilterTestCase(unittest2.TestCase):
 
+    def test_filter_from_json_string(self):
+        env = jinja_utils.get_jinja_environment()
+        expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}'
+
+        template = '{{k1 | from_json_string}}'
+
+        obj_str = env.from_string(template).render({'k1': obj_json_str})
+        obj = eval(obj_str)
+        self.assertDictEqual(obj, expected_obj)
+
+    def test_filter_from_yaml_string(self):
+        env = jinja_utils.get_jinja_environment()
+        expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
+        obj_yaml_str = ("---\n"
+                        "a: b\n"
+                        "c:\n"
+                        "  d: e\n"
+                        "  f: 1\n"
+                        "  g: true\n")
+
+        template = '{{k1 | from_yaml_string}}'
+        obj_str = env.from_string(template).render({'k1': obj_yaml_str})
+        obj = eval(obj_str)
+        self.assertDictEqual(obj, expected_obj)
+
     def test_filter_to_json_string(self):
         env = jinja_utils.get_jinja_environment()
         obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}

st2common/tests/unit/test_jinja_render_jsonpath_query_filters.py

@@ -0,0 +1,68 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest2
+
+from st2common.util import jinja as jinja_utils
+
+
+class JinjaUtilsJsonpathQueryTestCase(unittest2.TestCase):
+
+    def test_jsonpath_query_static(self):
+        env = jinja_utils.get_jinja_environment()
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+
+        template = '{{ obj | jsonpath_query("people[*].first") }}'
+        actual_str = env.from_string(template).render({'obj': obj})
+        actual = eval(actual_str)
+        expected = ['James', 'Jacob', 'Jayden']
+        self.assertEqual(actual, expected)
+
+    def test_jsonpath_query_dynamic(self):
+        env = jinja_utils.get_jinja_environment()
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "people[*].last"
+
+        template = '{{ obj | jsonpath_query(query) }}'
+        actual_str = env.from_string(template).render({'obj': obj,
+                                                       'query': query})
+        actual = eval(actual_str)
+        expected = ['d', 'e', 'f']
+        self.assertEqual(actual, expected)
+
+    def test_jsonpath_query_no_results(self):
+        env = jinja_utils.get_jinja_environment()
+        obj = {'people': [{'first': 'James', 'last': 'd'},
+                          {'first': 'Jacob', 'last': 'e'},
+                          {'first': 'Jayden', 'last': 'f'},
+                          {'missing': 'different'}],
+               'foo': {'bar': 'baz'}}
+        query = "query_returns_no_results"
+
+        template = '{{ obj | jsonpath_query(query) }}'
+        actual_str = env.from_string(template).render({'obj': obj,
+                                                       'query': query})
+        actual = eval(actual_str)
+        expected = None
+        self.assertEqual(actual, expected)

st2tests/integration/mistral/test_filters.py

@@ -25,6 +25,46 @@
 ]
 
 
+class FromJsonStringFiltersTest(base.TestWorkflowExecution):
+
+    def test_from_json_string(self):
+
+        execution = self._execute_workflow(
+            'examples.mistral-test-func-from-json-string',
+            parameters={
+                "input_str": '{"a": "b"}'
+            }
+        )
+        execution = self._wait_for_completion(execution)
+        self._assert_success(execution, num_tasks=1)
+        jinja_dict = execution.result['result_jinja']
+        yaql_dict = execution.result['result_yaql']
+        self.assertTrue(isinstance(jinja_dict, dict))
+        self.assertEqual(jinja_dict["a"], "b")
+        self.assertTrue(isinstance(yaql_dict, dict))
+        self.assertEqual(yaql_dict["a"], "b")
+
+
+class FromYamlStringFiltersTest(base.TestWorkflowExecution):
+
+    def test_from_yaml_string(self):
+
+        execution = self._execute_workflow(
+            'examples.mistral-test-func-from-yaml-string',
+            parameters={
+                "input_str": 'a: b'
+            }
+        )
+        execution = self._wait_for_completion(execution)
+        self._assert_success(execution, num_tasks=1)
+        jinja_dict = execution.result['result_jinja']
+        yaql_dict = execution.result['result_yaql']
+        self.assertTrue(isinstance(jinja_dict, dict))
+        self.assertEqual(jinja_dict["a"], "b")
+        self.assertTrue(isinstance(yaql_dict, dict))
+        self.assertEqual(yaql_dict["a"], "b")
+
+
 class JsonEscapeFiltersTest(base.TestWorkflowExecution):
 
     def test_json_escape(self):
@@ -44,6 +84,32 @@ def test_json_escape(self):
         self.assertEqual(yaql_dict["title"], breaking_str)
 
 
+class JsonpathQueryFiltersTest(base.TestWorkflowExecution):
+
+    def test_jsonpath_query(self):
+
+        execution = self._execute_workflow(
+            'examples.mistral-test-func-jsonpath-query',
+            parameters={
+                "input_obj": {'people': [{'first': 'James', 'last': 'Smith'},
+                                         {'first': 'Jacob', 'last': 'Alberts'},
+                                         {'first': 'Jayden', 'last': 'Davis'},
+                                         {'missing': 'different'}]},
+                "input_query": "people[*].last"
+            }
+        )
+        expected_result = ['Smith', 'Alberts', 'Davis']
+
+        execution = self._wait_for_completion(execution)
+        self._assert_success(execution, num_tasks=1)
+        jinja_result = execution.result['result_jinja']
+        yaql_result = execution.result['result_yaql']
+        self.assertTrue(isinstance(jinja_result, list))
+        self.assertEqual(jinja_result, expected_result)
+        self.assertTrue(isinstance(yaql_result, list))
+        self.assertEqual(yaql_result, expected_result)
+
+
 class RegexMatchFiltersTest(base.TestWorkflowExecution):
 
     def test_regex_match(self):