From 798f9221cd1ce5a0664ac505919280b015c2b5a3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 23 Jun 2017 20:39:45 -0500 Subject: [PATCH] Add extra_vars parsing directives due to Jinja Jinja always casts extra_vars to string because it is impossible to specify in a schema what vars might pass through extra_vars. So, to make it possible to pass objects through Jinja and have them show up as objects or ints or whatever, this adds directives with tests to show they work. The directives available are: '!INT', '!JSON', '!AST'. See the readme for how to use them. --- CHANGES.md | 17 ++++++ README.md | 47 +++++++++++++++ actions/lib/ansible_base.py | 27 ++++++++- pack.yaml | 2 +- tests/fixtures/extra_vars_complex.yaml | 66 +++++++++++++++++++++ tests/test_actions_lib_ansiblebaserunner.py | 21 +++++++ 6 files changed, 178 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8c967a9..078441b 100755 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Changelog +## v0.6.0 + +* Add extra_vars parsing directives to get around difficulties with Jinja (which casts all + extra_vars as strings). When passing in an object via Jinja, all values become strings. To get + around this, add "!AST", "!JSON", or "!INT" directives in your action-chain yaml: + +```yaml +chain: + name: 'example' + ref: 'ansible.command_local' + extra_parameters: + - + keyA: "!AST{{ jinja_variable_a }}" + keyB: "!JSON{{ jinja_variable_b | tojson }}" + keyC: "!INT{{ jinja_variable_c | int }}" +``` + ## v0.5.0 * Added ability to use yaml structures to pass arbitrarily complex values through extra_vars. key=value and @file syntax is still supported. Example usage: diff --git a/README.md b/README.md index 75e537e..0022e33 100755 --- a/README.md +++ b/README.md @@ -72,6 +72,53 @@ sample_task: key8: value8 ``` +##### Structured input and Jinja +Note that `value3`, in the previous example, was passed in as a variable in a Jinja expression. +StackStorm normally casts variables to the types like int, array, and object. +However, it can't do that for extra_vars because we cannot know ahead of time, +for all Ansible playbooks, what schema and data types each entry will be. As such, +within extra_vars, all Jinja expressions result in strings. To get around this, we've +added some extra_vars parsing diretives: `!INT`, `!JSON`, and `!AST`. + +Use `!INT` if you are using a playbook that expects a value to be an integer: + +```yaml +extra_vars: + port: '!INT{{ port_number }}' +``` + +Use `!JSON` when you have something that is already a JSON string or when it makes sense +to convert it into a json string using Jinja filters: + +```yaml +extra_vars: + special_config: '!JSON{{ config | tojson }}' +``` + +Use `!AST` (referring to python AST) when you have an object that you want to send straight over +without any additional jinja filters: + +```yaml +extra_vars: + data: '!AST{{ data }}' +``` + +These directives are supported recursively. So, you can embed directives in other directives. +Consider this contrived example (though mostly you'll only hit embedding directives or objects +if you're chaining workflows together to build up a data object): + +```yaml +vars: + answer: "!INT{{ life_universe_everything }}" +chain: + - + name: '...' + ref: 'ansible.playbook' + ... + extra_vars: + earth: '!JSON{"question": "unknown", "answer": {{ answer }} }' +``` + ##### Structured output ```sh # get structured JSON output from a playbook diff --git a/actions/lib/ansible_base.py b/actions/lib/ansible_base.py index 9b33539..a0fb668 100755 --- a/actions/lib/ansible_base.py +++ b/actions/lib/ansible_base.py @@ -1,6 +1,9 @@ import os import sys import subprocess + +import yaml + import shell import ast import json @@ -27,6 +30,24 @@ def __init__(self, args): self._parse_extra_vars() # handle multiple entries in --extra_vars arg self._prepend_venv_path() + def _parse_extra_vars_directives(self, value): + if isinstance(value, six.string_types): + if value.strip().startswith("!AST"): + a = ast.literal_eval(value.strip()[4:]) + return self._parse_extra_vars_directives(a) + elif value.strip().startswith("!JSON"): + j = json.loads(value.strip()[5:]) + return self._parse_extra_vars_directives(j) + elif value.strip().startswith("!INT"): + return int(value.strip()[4:]) + else: + return value + elif isinstance(value, dict): + return {k: self._parse_extra_vars_directives(v) for k, v in six.iteritems(value)} + elif isinstance(value, list): + return [self._parse_extra_vars_directives(item) for item in value] + return value + def _parse_extra_vars(self): """ This method turns the string list ("--extra_vars=[...]") passed in from the args @@ -50,10 +71,14 @@ def _parse_extra_vars(self): if isinstance(n, six.string_types): if n.strip().startswith("@"): var_list.append(('file', n.strip())) + elif (n.strip().startswith("!AST") + or n.strip().startswith("!JSON") + or '=' not in n): + var_list.append(('json', self._parse_extra_vars_directives(n))) else: var_list.append(('kwarg', n.strip())) elif isinstance(n, dict): - var_list.append(('json', n)) + var_list.append(('json', self._parse_extra_vars_directives(n))) last = '' kv_param = '' diff --git a/pack.yaml b/pack.yaml index b50e5a6..fa43501 100644 --- a/pack.yaml +++ b/pack.yaml @@ -6,6 +6,6 @@ keywords: - ansible - cfg management - configuration management -version : 0.5.1 +version : 0.6.0 author : StackStorm, Inc. email : info@stackstorm.com diff --git a/tests/fixtures/extra_vars_complex.yaml b/tests/fixtures/extra_vars_complex.yaml index de4dc26..da5ff5c 100644 --- a/tests/fixtures/extra_vars_complex.yaml +++ b/tests/fixtures/extra_vars_complex.yaml @@ -25,4 +25,70 @@ key6: key7: 'value7' - 'key8=value8' +- + name: ast_directive + test: + - "!AST{u'key1': u'12345', u'key2': u'value2'}" + expected: + - + key1: '12345' + key2: 'value2' +- + name: json_directive + test: + - '!JSON{"key1": "12345", "key2": "value2"}' + expected: + - + key1: '12345' + key2: 'value2' +- + name: sub_ast_directive + test: + - + astkey: "!AST[{u'key1': u'12345', u'key2': u'value2'}]" + expected: + - + astkey: + - + key1: '12345' + key2: 'value2' +- + name: sub_json_directive + test: + - + jsonkey: '!JSON[{"key1": "12345", "key2": "value2"}]' + expected: + - + jsonkey: + - + key1: '12345' + key2: 'value2' +- + name: int_directive + test: + - + key1: "!INT12345" + expected: + - + key1: 12345 +- + name: multi_directives_ast_int + test: + - + astkey: "!AST[{u'intkey': u'!INT12345'}]" + expected: + - + astkey: + - + intkey: 12345 +- + name: multi_directives_json_int + test: + - + jsonkey: '!JSON[{"intkey": "!INT12345"}]' + expected: + - + jsonkey: + - + intkey: 12345 diff --git a/tests/test_actions_lib_ansiblebaserunner.py b/tests/test_actions_lib_ansiblebaserunner.py index 924fc16..e6e3339 100644 --- a/tests/test_actions_lib_ansiblebaserunner.py +++ b/tests/test_actions_lib_ansiblebaserunner.py @@ -121,3 +121,24 @@ def extra_vars_complex_yaml_fixture(self, test_name): def test_parse_extra_vars_complex_yaml_arbitrarily_complex(self): self.extra_vars_complex_yaml_fixture('arbitrarily_complex') + + def test_parse_extra_vars_complex_yaml_ast_directive(self): + self.extra_vars_complex_yaml_fixture('ast_directive') + + def test_parse_extra_vars_complex_yaml_json_directive(self): + self.extra_vars_complex_yaml_fixture('json_directive') + + def test_parse_extra_vars_complex_yaml_sub_ast_directive(self): + self.extra_vars_complex_yaml_fixture('sub_ast_directive') + + def test_parse_extra_vars_complex_yaml_sub_json_directive(self): + self.extra_vars_complex_yaml_fixture('sub_json_directive') + + def test_parse_extra_vars_complex_yaml_int_directive(self): + self.extra_vars_complex_yaml_fixture('int_directive') + + def test_parse_extra_vars_complex_yaml_multi_directives_ast_int(self): + self.extra_vars_complex_yaml_fixture('multi_directives_ast_int') + + def test_parse_extra_vars_complex_yaml_multi_directives_json_int(self): + self.extra_vars_complex_yaml_fixture('multi_directives_json_int')