File 0002-Ensure-ANSIBLE_NO_LOG-is-respected-CVE-2024-0690-825.patch of Package ansible.37522
From f5cb50f79af310b917e6932a0c0d8e9a73261b7f Mon Sep 17 00:00:00 2001
From: Matt Martz <matt@sivel.net>
Date: Thu, 18 Jan 2024 17:17:23 -0600
Subject: [PATCH 2/2] [stable-2.14] Ensure ANSIBLE_NO_LOG is respected
(CVE-2024-0690) (#82565) (#82568)
(cherry picked from commit 6935c8e)
Force template module to use non-native Jinja2 (#68560)
Fixes #46169
Auto unroll generators produced by jinja filters (#68014)
* Auto unroll generators produced by jinja filters
* Unroll for native in finalize
* Fix indentation
Co-authored-by: Sam Doran <sdoran@redhat.com>
* Add changelog fragment
* ci_complete
* Always unroll regardless of jinja2
* ci_complete
Co-authored-by: Sam Doran <sdoran@redhat.com>
Skip literal_eval for string filters results in native jinja. (#70988) (#71313)
Fixes #70831
(cherry picked from commit b66d66027ece03f3f0a3fdb5fd6b8213965a2f1d)
Introduce context manager for temporary templar context changes (#60513)
* Introduce context manager for temporary templar context changes. Fixes #60106
* Rename and docstring
* Make set_temporary_context more generic, don't hardcode each thing you can set, apply to template action too
* not None
* linting fix
* Ignore invalid attrs
* Catch the right things, loop the right things
* Use set_temporary_context in a few extra action plugins
---
.../46169-non-native-template-module.yml | 2 +
.../60106-templar-contextmanager.yml | 4 +
.../68014-auto-unroll-jinja2-generators.yml | 3 +
...iteral_eval-string-filter-native-jinja.yml | 2 +
changelogs/fragments/cve-2024-0690.yml | 2 +
lib/ansible/config/base.yml | 2 +-
lib/ansible/playbook/base.py | 2 +-
lib/ansible/playbook/conditional.py | 4 +-
lib/ansible/playbook/play_context.py | 4 -
lib/ansible/plugins/action/ce_template.py | 4 +-
lib/ansible/plugins/action/network.py | 4 +-
lib/ansible/plugins/action/template.py | 32 ++-
lib/ansible/plugins/lookup/template.py | 50 ++--
lib/ansible/template/__init__.py | 227 ++++++++++++++++--
lib/ansible/template/native_helpers.py | 39 +++
lib/ansible/utils/native_jinja.py | 13 +
lib/ansible/utils/unsafe_proxy.py | 7 +
.../jinja2_native_types/test_casting.yml | 7 +
.../jinja2_native_types/test_dunder.yml | 2 +-
.../targets/no_log/no_log_config.yml | 13 +
test/integration/targets/no_log/runme.sh | 5 +
.../template_jinja2_non_native/46169.yml | 32 +++
.../template_jinja2_non_native/aliases | 1 +
.../template_jinja2_non_native/runme.sh | 7 +
.../templates/46169.json.j2 | 3 +
25 files changed, 398 insertions(+), 73 deletions(-)
create mode 100644 changelogs/fragments/46169-non-native-template-module.yml
create mode 100644 changelogs/fragments/60106-templar-contextmanager.yml
create mode 100644 changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
create mode 100644 changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
create mode 100644 changelogs/fragments/cve-2024-0690.yml
create mode 100644 lib/ansible/utils/native_jinja.py
create mode 100644 test/integration/targets/no_log/no_log_config.yml
create mode 100644 test/integration/targets/template_jinja2_non_native/46169.yml
create mode 100644 test/integration/targets/template_jinja2_non_native/aliases
create mode 100755 test/integration/targets/template_jinja2_non_native/runme.sh
create mode 100644 test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
diff --git a/changelogs/fragments/46169-non-native-template-module.yml b/changelogs/fragments/46169-non-native-template-module.yml
new file mode 100644
index 0000000000..7d004a6296
--- /dev/null
+++ b/changelogs/fragments/46169-non-native-template-module.yml
@@ -0,0 +1,2 @@
+minor_changes:
+ - Force the template module to use non-native Jinja2 (https://github.com/ansible/ansible/issues/46169)
diff --git a/changelogs/fragments/60106-templar-contextmanager.yml b/changelogs/fragments/60106-templar-contextmanager.yml
new file mode 100644
index 0000000000..45afc1544a
--- /dev/null
+++ b/changelogs/fragments/60106-templar-contextmanager.yml
@@ -0,0 +1,4 @@
+bugfixes:
+- template lookup - ensure changes to the templar in the lookup, do not
+ affect the templar context outside of the lookup
+ (https://github.com/ansible/ansible/issues/60106)
diff --git a/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
new file mode 100644
index 0000000000..211d2fd665
--- /dev/null
+++ b/changelogs/fragments/68014-auto-unroll-jinja2-generators.yml
@@ -0,0 +1,3 @@
+minor_changes:
+- Templating - Add support to auto unroll generators produced by jinja2 filters, to prevent the need of explicit use of ``|list``
+ (https://github.com/ansible/ansible/pull/68014)
diff --git a/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
new file mode 100644
index 0000000000..40b426e50b
--- /dev/null
+++ b/changelogs/fragments/70831-skip-literal_eval-string-filter-native-jinja.yml
@@ -0,0 +1,2 @@
+bugfixes:
+ - Skip literal_eval for string filters results in native jinja. (https://github.com/ansible/ansible/issues/70831)
diff --git a/changelogs/fragments/cve-2024-0690.yml b/changelogs/fragments/cve-2024-0690.yml
new file mode 100644
index 0000000000..0e030d8886
--- /dev/null
+++ b/changelogs/fragments/cve-2024-0690.yml
@@ -0,0 +1,2 @@
+security_fixes:
+- ANSIBLE_NO_LOG - Address issue where ANSIBLE_NO_LOG was ignored (CVE-2024-0690)
diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml
index 3d3916a7fc..96d38e7f51 100644
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@@ -1757,7 +1757,7 @@ SHOW_CUSTOM_STATS:
type: bool
STRING_TYPE_FILTERS:
name: Filters to preserve strings
- default: [string, to_json, to_nice_json, to_yaml, ppretty, json]
+ default: [string, to_json, to_nice_json, to_yaml, to_nice_yaml, ppretty, json]
description:
- "This list of filters avoids 'type conversion' when templating variables"
- Useful when you want to avoid conversion into lists or dictionaries for JSON strings, for example.
diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py
index 0f4dc4e430..172963a218 100644
--- a/lib/ansible/playbook/base.py
+++ b/lib/ansible/playbook/base.py
@@ -613,7 +613,7 @@ class Base(FieldAttributeBase):
# flags and misc. settings
_environment = FieldAttribute(isa='list', extend=True, prepend=True)
- _no_log = FieldAttribute(isa='bool')
+ _no_log = FieldAttribute(isa='bool', default=C.DEFAULT_NO_LOG)
_run_once = FieldAttribute(isa='bool')
_ignore_errors = FieldAttribute(isa='bool')
_ignore_unreachable = FieldAttribute(isa='bool')
diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py
index ac4fc0c568..be4b75986c 100644
--- a/lib/ansible/playbook/conditional.py
+++ b/lib/ansible/playbook/conditional.py
@@ -173,8 +173,8 @@ class Conditional:
)
try:
e = templar.environment.overlay()
- e.filters.update(templar._get_filters())
- e.tests.update(templar._get_tests())
+ e.filters.update(templar.environment.filters)
+ e.tests.update(templar.environment.tests)
res = e._parse(conditional, None, None)
res = generate(res, e, None, None)
diff --git a/lib/ansible/playbook/play_context.py b/lib/ansible/playbook/play_context.py
index 10dd57aa3f..5b8b28526c 100644
--- a/lib/ansible/playbook/play_context.py
+++ b/lib/ansible/playbook/play_context.py
@@ -318,10 +318,6 @@ class PlayContext(Base):
if not new_info.connection_user:
new_info.connection_user = new_info.remote_user
- # set no_log to default if it was not previously set
- if new_info.no_log is None:
- new_info.no_log = C.DEFAULT_NO_LOG
-
if task.check_mode is not None:
new_info.check_mode = task.check_mode
diff --git a/lib/ansible/plugins/action/ce_template.py b/lib/ansible/plugins/action/ce_template.py
index 8d62b25647..4a72fbbfa8 100644
--- a/lib/ansible/plugins/action/ce_template.py
+++ b/lib/ansible/plugins/action/ce_template.py
@@ -100,5 +100,5 @@ class ActionModule(_ActionModule):
for role in dep_chain:
searchpath.append(role._role_path)
searchpath.append(os.path.dirname(source))
- self._templar.environment.loader.searchpath = searchpath
- self._task.args['src'] = self._templar.template(template_data)
+ with self._templar.set_temporary_context(searchpath=searchpath):
+ self._task.args['src'] = self._templar.template(template_data)
diff --git a/lib/ansible/plugins/action/network.py b/lib/ansible/plugins/action/network.py
index f0d0ca3ba7..d91c9b2af9 100644
--- a/lib/ansible/plugins/action/network.py
+++ b/lib/ansible/plugins/action/network.py
@@ -160,8 +160,8 @@ class ActionModule(_ActionModule):
for role in dep_chain:
searchpath.append(role._role_path)
searchpath.append(os.path.dirname(source))
- self._templar.environment.loader.searchpath = searchpath
- self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data)
+ with self._templar.set_temporary_context(searchpath=searchpath):
+ self._task.args['src'] = self._templar.template(template_data, convert_data=convert_data)
def _get_network_os(self, task_vars):
if 'network_os' in self._task.args and self._task.args['network_os']:
diff --git a/lib/ansible/plugins/action/template.py b/lib/ansible/plugins/action/template.py
index 8fb7393ff9..cede680ca6 100644
--- a/lib/ansible/plugins/action/template.py
+++ b/lib/ansible/plugins/action/template.py
@@ -17,7 +17,7 @@ from ansible.module_utils._text import to_bytes, to_text, to_native
from ansible.module_utils.parsing.convert_bool import boolean
from ansible.module_utils.six import string_types
from ansible.plugins.action import ActionBase
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
class ActionModule(ActionBase):
@@ -127,27 +127,23 @@ class ActionModule(ActionBase):
newsearchpath.append(p)
searchpath = newsearchpath
- self._templar.environment.loader.searchpath = searchpath
- self._templar.environment.newline_sequence = newline_sequence
- if block_start_string is not None:
- self._templar.environment.block_start_string = block_start_string
- if block_end_string is not None:
- self._templar.environment.block_end_string = block_end_string
- if variable_start_string is not None:
- self._templar.environment.variable_start_string = variable_start_string
- if variable_end_string is not None:
- self._templar.environment.variable_end_string = variable_end_string
- self._templar.environment.trim_blocks = trim_blocks
- self._templar.environment.lstrip_blocks = lstrip_blocks
-
# add ansible 'template' vars
temp_vars = task_vars.copy()
temp_vars.update(generate_ansible_template_vars(source, dest))
- old_vars = self._templar.available_variables
- self._templar.available_variables = temp_vars
- resultant = self._templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
- self._templar.available_variables = old_vars
+ # force templar to use AnsibleEnvironment to prevent issues with native types
+ # https://github.com/ansible/ansible/issues/46169
+ templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment,
+ searchpath=searchpath,
+ newline_sequence=newline_sequence,
+ block_start_string=block_start_string,
+ block_end_string=block_end_string,
+ variable_start_string=variable_start_string,
+ variable_end_string=variable_end_string,
+ trim_blocks=trim_blocks,
+ lstrip_blocks=lstrip_blocks,
+ available_variables=temp_vars)
+ resultant = templar.do_template(template_data, preserve_trailing_newlines=True, escape_backslashes=False)
except AnsibleAction:
raise
except Exception as e:
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index 4fd3584b65..c04b5e0d6a 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -17,7 +17,9 @@ DOCUMENTATION = """
description: list of files to template
convert_data:
type: bool
- description: whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+ description:
+ - Whether to convert YAML into data. If False, strings that are YAML will be left untouched.
+ - Mutually exclusive with the jinja2_native option.
variable_start_string:
description: The string marking the beginning of a print statement.
default: '{{'
@@ -28,6 +30,16 @@ DOCUMENTATION = """
default: '}}'
version_added: '2.8'
type: str
+ jinja2_native:
+ description:
+ - Controls whether to use Jinja2 native types.
+ - It is off by default even if global jinja2_native is True.
+ - Has no effect if global jinja2_native is False.
+ - This offers more flexibility than the template module which does not use Jinja2 native types at all.
+ - Mutually exclusive with the convert_data option.
+ default: False
+ version_added: '2.11'
+ type: bool
"""
EXAMPLES = """
@@ -51,24 +63,31 @@ import os
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_bytes, to_text
-from ansible.template import generate_ansible_template_vars
+from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
from ansible.utils.display import Display
+if USE_JINJA2_NATIVE:
+ from ansible.utils.native_jinja import NativeJinjaText
+
+
display = Display()
class LookupModule(LookupBase):
def run(self, terms, variables, **kwargs):
-
convert_data_p = kwargs.get('convert_data', True)
lookup_template_vars = kwargs.get('template_vars', {})
+ jinja2_native = kwargs.get('jinja2_native', False)
ret = []
variable_start_string = kwargs.get('variable_start_string', None)
variable_end_string = kwargs.get('variable_end_string', None)
- old_vars = self._templar.available_variables
+ if USE_JINJA2_NATIVE and not jinja2_native:
+ templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
+ else:
+ templar = self._templar
for term in terms:
display.debug("File lookup term: %s" % term)
@@ -92,12 +111,6 @@ class LookupModule(LookupBase):
searchpath = newsearchpath
searchpath.insert(0, os.path.dirname(lookupfile))
- self._templar.environment.loader.searchpath = searchpath
- if variable_start_string is not None:
- self._templar.environment.variable_start_string = variable_start_string
- if variable_end_string is not None:
- self._templar.environment.variable_end_string = variable_end_string
-
# The template will have access to all existing variables,
# plus some added by ansible (e.g., template_{path,mtime}),
# plus anything passed to the lookup with the template_vars=
@@ -105,17 +118,20 @@ class LookupModule(LookupBase):
vars = deepcopy(variables)
vars.update(generate_ansible_template_vars(lookupfile))
vars.update(lookup_template_vars)
- self._templar.available_variables = vars
- # do the templating
- res = self._templar.template(template_data, preserve_trailing_newlines=True,
- convert_data=convert_data_p, escape_backslashes=False)
+ with templar.set_temporary_context(variable_start_string=variable_start_string,
+ variable_end_string=variable_end_string,
+ available_variables=vars, searchpath=searchpath):
+ res = templar.template(template_data, preserve_trailing_newlines=True,
+ convert_data=convert_data_p, escape_backslashes=False)
+
+ if USE_JINJA2_NATIVE and not jinja2_native:
+ # jinja2_native is true globally but off for the lookup, we need this text
+ # not to be processed by literal_eval anywhere in Ansible
+ res = NativeJinjaText(res)
ret.append(res)
else:
raise AnsibleError("the template file %s could not be found for the lookup" % term)
- # restore old variables
- self._templar.available_variables = old_vars
-
return ret
diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index 94ab31e58d..35c9dac194 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -28,6 +28,7 @@ import re
import time
from distutils.version import LooseVersion
+from contextlib import contextmanager
from numbers import Number
try:
@@ -42,8 +43,9 @@ from jinja2.runtime import Context, StrictUndefined
from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError
from ansible.module_utils.six import iteritems, string_types, text_type
+from ansible.module_utils.six.moves import range
from ansible.module_utils._text import to_native, to_text, to_bytes
-from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping
+from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping
from ansible.module_utils.common.collections import is_sequence
from ansible.module_utils.compat.importlib import import_module
from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
@@ -71,12 +73,16 @@ NON_TEMPLATED_TYPES = (bool, Number)
JINJA2_OVERRIDE = '#jinja2:'
from jinja2 import __version__ as j2_version
+from jinja2 import Environment
+from jinja2.utils import concat as j2_concat
+
USE_JINJA2_NATIVE = False
if C.DEFAULT_JINJA2_NATIVE:
try:
- from jinja2.nativetypes import NativeEnvironment as Environment
- from ansible.template.native_helpers import ansible_native_concat as j2_concat
+ from jinja2.nativetypes import NativeEnvironment
+ from ansible.template.native_helpers import ansible_native_concat
+ from ansible.utils.native_jinja import NativeJinjaText
USE_JINJA2_NATIVE = True
except ImportError:
from jinja2 import Environment
@@ -85,15 +91,15 @@ if C.DEFAULT_JINJA2_NATIVE:
'jinja2_native requires Jinja 2.10 and above. '
'Version detected: %s. Falling back to default.' % j2_version
)
-else:
- from jinja2 import Environment
- from jinja2.utils import concat as j2_concat
JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin'))
JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end'))
+RANGE_TYPE = type(range(0))
+
+
def generate_ansible_template_vars(path, dest_path=None):
b_path = to_bytes(path)
try:
@@ -230,6 +236,60 @@ def recursive_check_defined(item):
raise AnsibleFilterError("{0} is undefined".format(item))
+def _is_rolled(value):
+ """Helper method to determine if something is an unrolled generator,
+ iterator, or similar object
+ """
+ return (
+ isinstance(value, Iterator) or
+ isinstance(value, MappingView) or
+ isinstance(value, RANGE_TYPE)
+ )
+
+
+def _unroll_iterator(func):
+ """Wrapper function, that intercepts the result of a filter
+ and auto unrolls a generator, so that users are not required to
+ explicitly use ``|list`` to unroll.
+ """
+ def wrapper(*args, **kwargs):
+ ret = func(*args, **kwargs)
+ if _is_rolled(ret):
+ return list(ret)
+ return ret
+
+ return _update_wrapper(wrapper, func)
+
+
+def _update_wrapper(wrapper, func):
+ # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
+ # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
+ for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
+ try:
+ value = getattr(func, attr)
+ except AttributeError:
+ pass
+ else:
+ setattr(wrapper, attr, value)
+ for attr in ('__dict__',):
+ getattr(wrapper, attr).update(getattr(func, attr, {}))
+ wrapper.__wrapped__ = func
+ return wrapper
+
+
+def _wrap_native_text(func):
+ """Wrapper function, that intercepts the result of a filter
+ and wraps it into NativeJinjaText which is then used
+ in ``ansible_native_concat`` to indicate that it is a text
+ which should not be passed into ``literal_eval``.
+ """
+ def wrapper(*args, **kwargs):
+ ret = func(*args, **kwargs)
+ return NativeJinjaText(ret)
+
+ return _update_wrapper(wrapper, func)
+
+
class AnsibleUndefined(StrictUndefined):
'''
A custom Undefined class, which returns further Undefined objects on access,
@@ -350,10 +410,11 @@ class AnsibleContext(Context):
class JinjaPluginIntercept(MutableMapping):
- def __init__(self, delegatee, pluginloader, *args, **kwargs):
+ def __init__(self, delegatee, pluginloader, jinja2_native, *args, **kwargs):
super(JinjaPluginIntercept, self).__init__(*args, **kwargs)
self._delegatee = delegatee
self._pluginloader = pluginloader
+ self._jinja2_native = jinja2_native
if self._pluginloader.class_name == 'FilterModule':
self._method_map_name = 'filters'
@@ -406,10 +467,13 @@ class JinjaPluginIntercept(MutableMapping):
method_map = getattr(plugin_impl, self._method_map_name)
- for f in iteritems(method_map()):
- fq_name = '.'.join((parent_prefix, f[0]))
+ for func_name, func in iteritems(method_map()):
+ fq_name = '.'.join((parent_prefix, func_name))
# FIXME: detect/warn on intra-collection function name collisions
- self._collection_jinja_func_cache[fq_name] = f[1]
+ if self._jinja2_native and func_name in C.STRING_TYPE_FILTERS:
+ self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func)
+ else:
+ self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func)
function_impl = self._collection_jinja_func_cache[key]
return function_impl
@@ -433,6 +497,9 @@ class AnsibleEnvironment(Environment):
'''
Our custom environment, which simply allows us to override the class-level
values for the Template and Context classes used by jinja2 internally.
+
+ NOTE: Any changes to this class must be reflected in
+ :class:`AnsibleNativeEnvironment` as well.
'''
context_class = AnsibleContext
template_class = AnsibleJ2Template
@@ -440,8 +507,27 @@ class AnsibleEnvironment(Environment):
def __init__(self, *args, **kwargs):
super(AnsibleEnvironment, self).__init__(*args, **kwargs)
- self.filters = JinjaPluginIntercept(self.filters, filter_loader)
- self.tests = JinjaPluginIntercept(self.tests, test_loader)
+ self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=False)
+ self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False)
+
+
+if USE_JINJA2_NATIVE:
+ class AnsibleNativeEnvironment(NativeEnvironment):
+ '''
+ Our custom environment, which simply allows us to override the class-level
+ values for the Template and Context classes used by jinja2 internally.
+
+ NOTE: Any changes to this class must be reflected in
+ :class:`AnsibleEnvironment` as well.
+ '''
+ context_class = AnsibleContext
+ template_class = AnsibleJ2Template
+
+ def __init__(self, *args, **kwargs):
+ super(AnsibleNativeEnvironment, self).__init__(*args, **kwargs)
+
+ self.filters = JinjaPluginIntercept(self.filters, filter_loader, jinja2_native=True)
+ self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=True)
class Templar:
@@ -478,7 +564,9 @@ class Templar:
self._fail_on_filter_errors = True
self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
- self.environment = AnsibleEnvironment(
+ environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment
+
+ self.environment = environment_class(
trim_blocks=True,
undefined=AnsibleUndefined,
extensions=self._get_extensions(),
@@ -489,17 +577,50 @@ class Templar:
# the current rendering context under which the templar class is working
self.cur_context = None
+ # FIXME these regular expressions should be re-compiled each time variable_start_string and variable_end_string are changed
self.SINGLE_VAR = re.compile(r"^%s\s*(\w*)\s*%s$" % (self.environment.variable_start_string, self.environment.variable_end_string))
-
- self._clean_regex = re.compile(r'(?:%s|%s|%s|%s)' % (
- self.environment.variable_start_string,
- self.environment.block_start_string,
- self.environment.block_end_string,
- self.environment.variable_end_string
- ))
self._no_type_regex = re.compile(r'.*?\|\s*(?:%s)(?:\([^\|]*\))?\s*\)?\s*(?:%s)' %
('|'.join(C.STRING_TYPE_FILTERS), self.environment.variable_end_string))
+ @property
+ def jinja2_native(self):
+ return not isinstance(self.environment, AnsibleEnvironment)
+
+ def copy_with_new_env(self, environment_class=AnsibleEnvironment, **kwargs):
+ r"""Creates a new copy of Templar with a new environment. The new environment is based on
+ given environment class and kwargs.
+
+ :kwarg environment_class: Environment class used for creating a new environment.
+ :kwarg \*\*kwargs: Optional arguments for the new environment that override existing
+ environment attributes.
+
+ :returns: Copy of Templar with updated environment.
+ """
+ # We need to use __new__ to skip __init__, mainly not to create a new
+ # environment there only to override it below
+ new_env = object.__new__(environment_class)
+ new_env.__dict__.update(self.environment.__dict__)
+
+ new_templar = object.__new__(Templar)
+ new_templar.__dict__.update(self.__dict__)
+ new_templar.environment = new_env
+
+ mapping = {
+ 'available_variables': new_templar,
+ 'searchpath': new_env.loader,
+ }
+
+ for key, value in kwargs.items():
+ obj = mapping.get(key, new_env)
+ try:
+ if value is not None:
+ setattr(obj, key, value)
+ except AttributeError:
+ # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
+ pass
+
+ return new_templar
+
def _get_filters(self):
'''
Returns filter plugins, after loading and caching them if need be
@@ -513,6 +634,17 @@ class Templar:
for fp in self._filter_loader.all():
self._filters.update(fp.filters())
+ if self.jinja2_native:
+ for string_filter in C.STRING_TYPE_FILTERS:
+ try:
+ orig_filter = self._filters[string_filter]
+ except KeyError:
+ try:
+ orig_filter = self.environment.filters[string_filter]
+ except KeyError:
+ continue
+ self._filters[string_filter] = _wrap_native_text(orig_filter)
+
return self._filters.copy()
def _get_tests(self):
@@ -570,6 +702,36 @@ class Templar:
)
self.available_variables = variables
+ @contextmanager
+ def set_temporary_context(self, **kwargs):
+ """Context manager used to set temporary templating context, without having to worry about resetting
+ original values afterward
+
+ Use a keyword that maps to the attr you are setting. Applies to ``self.environment`` by default, to
+ set context on another object, it must be in ``mapping``.
+ """
+ mapping = {
+ 'available_variables': self,
+ 'searchpath': self.environment.loader,
+ }
+ original = {}
+
+ for key, value in kwargs.items():
+ obj = mapping.get(key, self.environment)
+ try:
+ original[key] = getattr(obj, key)
+ if value is not None:
+ setattr(obj, key, value)
+ except AttributeError:
+ # Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
+ pass
+
+ yield
+
+ for key in original:
+ obj = mapping.get(key, self.environment)
+ setattr(obj, key, original[key])
+
def template(self, variable, convert_bare=False, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None,
convert_data=True, static_vars=None, cache=True, disable_lookups=False):
'''
@@ -632,7 +794,7 @@ class Templar:
disable_lookups=disable_lookups,
)
- if not USE_JINJA2_NATIVE:
+ if not self.jinja2_native:
unsafe = hasattr(result, '__UNSAFE__')
if convert_data and not self._no_type_regex.match(variable):
# if this looks like a dictionary or list, convert it to such using the safe_eval method
@@ -746,8 +908,18 @@ class Templar:
If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always
'''
- if USE_JINJA2_NATIVE:
+ if _is_rolled(thing):
+ # Auto unroll a generator, so that users are not required to
+ # explicitly use ``|list`` to unroll
+ # This only affects the scenario where the final result of templating
+ # is a generator, and not where a filter creates a generator in the middle
+ # of a template. See ``_unroll_iterator`` for the other case. This is probably
+ # unncessary
+ return list(thing)
+
+ if self.jinja2_native:
return thing
+
return thing if thing is not None else ''
def _fail_lookup(self, name, *args, **kwargs):
@@ -802,7 +974,10 @@ class Templar:
ran = wrap_var(ran)
else:
try:
- ran = wrap_var(",".join(ran))
+ if self.jinja2_native and isinstance(ran[0], NativeJinjaText):
+ ran = wrap_var(NativeJinjaText(",".join(ran)))
+ else:
+ ran = wrap_var(",".join(ran))
except TypeError:
# Lookup Plugins should always return lists. Throw an error if that's not
# the case:
@@ -824,7 +999,7 @@ class Templar:
raise AnsibleError("lookup plugin (%s) not found" % name)
def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False):
- if USE_JINJA2_NATIVE and not isinstance(data, string_types):
+ if self.jinja2_native and not isinstance(data, string_types):
return data
# For preserving the number of input newlines in the output (used
@@ -853,6 +1028,8 @@ class Templar:
# Adds Ansible custom filters and tests
myenv.filters.update(self._get_filters())
+ for k in myenv.filters:
+ myenv.filters[k] = _unroll_iterator(myenv.filters[k])
myenv.tests.update(self._get_tests())
if escape_backslashes:
@@ -904,7 +1081,7 @@ class Templar:
display.debug("failing because of a type error, template data is: %s" % to_text(data))
raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te)))
- if USE_JINJA2_NATIVE and not isinstance(res, string_types):
+ if self.jinja2_native and not isinstance(res, string_types):
return res
if preserve_trailing_newlines:
diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py
index 11c14b7fa1..84296ad9b6 100644
--- a/lib/ansible/template/native_helpers.py
+++ b/lib/ansible/template/native_helpers.py
@@ -14,6 +14,34 @@ from jinja2._compat import text_type
from jinja2.runtime import StrictUndefined
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.native_jinja import NativeJinjaText
+
+
+def _fail_on_undefined(data):
+ """Recursively find an undefined value in a nested data structure
+ and properly raise the undefined exception.
+ """
+ if isinstance(data, Mapping):
+ for value in data.values():
+ _fail_on_undefined(value)
+ elif is_sequence(data):
+ for item in data:
+ _fail_on_undefined(item)
+ else:
+ if isinstance(data, StrictUndefined):
+ # To actually raise the undefined exception we need to
+ # access the undefined object otherwise the exception would
+ # be raised on the next access which might not be properly
+ # handled.
+ # See https://github.com/ansible/ansible/issues/52158
+ # and StrictUndefined implementation in upstream Jinja2.
+ str(data)
+
+ return data
+
+
+class NativeJinjaText(text_type):
+ pass
def ansible_native_concat(nodes):
@@ -49,9 +77,20 @@ def ansible_native_concat(nodes):
# We do that only here because it is taken care of by text_type() in the else block below already.
str(out)
+ if isinstance(out, NativeJinjaText):
+ # Sometimes (e.g. ``| string``) we need to mark variables
+ # in a special way so that they remain strings and are not
+ # passed into literal_eval.
+ # See:
+ # https://github.com/ansible/ansible/issues/70831
+ # https://github.com/pallets/jinja/issues/1200
+ # https://github.com/ansible/ansible/issues/70831#issuecomment-664190894
+ return out
+
# short circuit literal_eval when possible
if not isinstance(out, list):
return out
+
else:
if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes)
diff --git a/lib/ansible/utils/native_jinja.py b/lib/ansible/utils/native_jinja.py
new file mode 100644
index 0000000000..53ef14004a
--- /dev/null
+++ b/lib/ansible/utils/native_jinja.py
@@ -0,0 +1,13 @@
+# Copyright: (c) 2020, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+from ansible.module_utils.six import text_type
+
+
+class NativeJinjaText(text_type):
+ pass
diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py
index 0bfd6340ff..54bebd177a 100644
--- a/lib/ansible/utils/unsafe_proxy.py
+++ b/lib/ansible/utils/unsafe_proxy.py
@@ -57,6 +57,7 @@ from ansible.module_utils._text import to_bytes, to_text
from ansible.module_utils.common._collections_compat import Mapping, Set
from ansible.module_utils.common.collections import is_sequence
from ansible.module_utils.six import string_types, binary_type, text_type
+from ansible.utils.native_jinja import NativeJinjaText
__all__ = ['AnsibleUnsafe', 'wrap_var']
@@ -331,6 +332,10 @@ class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
pass
+class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
+ pass
+
+
class UnsafeProxy(object):
def __new__(cls, obj, *args, **kwargs):
from ansible.utils.display import Display
@@ -376,6 +381,8 @@ def wrap_var(v):
v = _wrap_set(v)
elif is_sequence(v):
v = _wrap_sequence(v)
+ elif isinstance(v, NativeJinjaText):
+ v = NativeJinjaUnsafeText(v)
elif isinstance(v, binary_type):
v = AnsibleUnsafeBytes(v)
elif isinstance(v, text_type):
diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml
index 5b4fe3ac0e..da06ab2e28 100644
--- a/test/integration/targets/jinja2_native_types/test_casting.yml
+++ b/test/integration/targets/jinja2_native_types/test_casting.yml
@@ -1,17 +1,22 @@
- name: cast things to other things
set_fact:
int_to_str: "{{ i_two|to_text }}"
+ int_to_str2: "{{ i_two | string }}"
str_to_int: "{{ s_two|int }}"
dict_to_str: "{{ dict_one|to_text }}"
list_to_str: "{{ list_one|to_text }}"
int_to_bool: "{{ i_one|bool }}"
str_true_to_bool: "{{ s_true|bool }}"
str_false_to_bool: "{{ s_false|bool }}"
+ list_to_json_str: "{{ list_one | to_json }}"
+ list_to_yaml_str: "{{ list_one | to_yaml }}"
- assert:
that:
- 'int_to_str == "2"'
- 'int_to_str|type_debug in ["str", "unicode"]'
+ - 'int_to_str2 == "2"'
+ - 'int_to_str2|type_debug in ["NativeJinjaText"]'
- 'str_to_int == 2'
- 'str_to_int|type_debug == "int"'
- 'dict_to_str|type_debug in ["str", "unicode"]'
@@ -22,3 +27,5 @@
- 'str_true_to_bool|type_debug == "bool"'
- 'str_false_to_bool is sameas false'
- 'str_false_to_bool|type_debug == "bool"'
+ - 'list_to_json_str|type_debug in ["NativeJinjaText"]'
+ - 'list_to_yaml_str|type_debug in ["NativeJinjaText"]'
diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml
index 46fd4d0a90..df5ea9276b 100644
--- a/test/integration/targets/jinja2_native_types/test_dunder.yml
+++ b/test/integration/targets/jinja2_native_types/test_dunder.yml
@@ -20,4 +20,4 @@
- assert:
that:
- - 'const_dunder|type_debug in ["str", "unicode"]'
+ - 'const_dunder|type_debug in ["str", "unicode", "NativeJinjaText"]'
diff --git a/test/integration/targets/no_log/no_log_config.yml b/test/integration/targets/no_log/no_log_config.yml
new file mode 100644
index 0000000000..8a5088059d
--- /dev/null
+++ b/test/integration/targets/no_log/no_log_config.yml
@@ -0,0 +1,13 @@
+- hosts: testhost
+ gather_facts: false
+ tasks:
+ - debug:
+ no_log: true
+
+ - debug:
+ no_log: false
+
+ - debug:
+
+ - debug:
+ loop: '{{ range(3) }}'
diff --git a/test/integration/targets/no_log/runme.sh b/test/integration/targets/no_log/runme.sh
index bb5c048fc9..8bfe019bb9 100755
--- a/test/integration/targets/no_log/runme.sh
+++ b/test/integration/targets/no_log/runme.sh
@@ -19,3 +19,8 @@ set -eux
# test invalid data passed to a suboption
[ "$(ansible-playbook no_log_suboptions_invalid.yml -i ../../inventory -vvvvv "$@" | grep -Ec '(SUPREME|IDIOM|MOCKUP|EDUCATED|FOOTREST|CRAFTY|FELINE|CRYSTAL|EXPECTANT|AGROUND|GOLIATH|FREEFALL)')" = "0" ]
+
+# test variations on ANSIBLE_NO_LOG
+[ "$(ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ]
+[ "$(ANSIBLE_NO_LOG=0 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "1" ]
+[ "$(ANSIBLE_NO_LOG=1 ansible-playbook no_log_config.yml -i ../../inventory -vvvvv "$@" | grep -Ec 'the output has been hidden')" = "6" ]
diff --git a/test/integration/targets/template_jinja2_non_native/46169.yml b/test/integration/targets/template_jinja2_non_native/46169.yml
new file mode 100644
index 0000000000..efb443eae0
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/46169.yml
@@ -0,0 +1,32 @@
+- hosts: localhost
+ gather_facts: no
+ tasks:
+ - set_fact:
+ output_dir: "{{ lookup('env', 'OUTPUT_DIR') }}"
+
+ - template:
+ src: templates/46169.json.j2
+ dest: "{{ output_dir }}/result.json"
+
+ - command: "diff templates/46169.json.j2 {{ output_dir }}/result.json"
+ register: diff_result
+
+ - assert:
+ that:
+ - diff_result.stdout == ""
+
+ - block:
+ - set_fact:
+ non_native_lookup: "{{ lookup('template', 'templates/46169.json.j2') }}"
+
+ - assert:
+ that:
+ - non_native_lookup | type_debug == 'NativeJinjaUnsafeText'
+
+ - set_fact:
+ native_lookup: "{{ lookup('template', 'templates/46169.json.j2', jinja2_native=true) }}"
+
+ - assert:
+ that:
+ - native_lookup | type_debug == 'dict'
+ when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')
diff --git a/test/integration/targets/template_jinja2_non_native/aliases b/test/integration/targets/template_jinja2_non_native/aliases
new file mode 100644
index 0000000000..b59832142f
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/aliases
@@ -0,0 +1 @@
+shippable/posix/group3
diff --git a/test/integration/targets/template_jinja2_non_native/runme.sh b/test/integration/targets/template_jinja2_non_native/runme.sh
new file mode 100755
index 0000000000..fe9d495a3e
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/runme.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export ANSIBLE_JINJA2_NATIVE=1
+ansible-playbook 46169.yml -v "$@"
+unset ANSIBLE_JINJA2_NATIVE
diff --git a/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2 b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
new file mode 100644
index 0000000000..a4fc3f6717
--- /dev/null
+++ b/test/integration/targets/template_jinja2_non_native/templates/46169.json.j2
@@ -0,0 +1,3 @@
+{
+ "key": "bar"
+}
--
2.44.0
