File make-sure-the-file-client-is-destroyed-upon-used.patch of Package salt.28967
From a1fc5287d501a1ecdbd259e5bbdd4f7d5d06dd13 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Fri, 28 Apr 2023 09:41:28 +0200
Subject: [PATCH] Make sure the file client is destroyed upon used
Backport of https://github.com/saltstack/salt/pull/64113
---
salt/client/ssh/wrapper/saltcheck.py | 108 +++----
salt/fileclient.py | 11 -
salt/modules/dockermod.py | 17 +-
salt/pillar/__init__.py | 6 +-
salt/states/ansiblegate.py | 11 +-
salt/utils/asynchronous.py | 2 +-
salt/utils/jinja.py | 53 ++-
salt/utils/mako.py | 7 +
salt/utils/templates.py | 303 +++++++++---------
.../integration/states/test_include.py | 40 +++
.../utils/jinja/test_salt_cache_loader.py | 47 ++-
11 files changed, 330 insertions(+), 275 deletions(-)
create mode 100644 tests/pytests/integration/states/test_include.py
diff --git a/salt/client/ssh/wrapper/saltcheck.py b/salt/client/ssh/wrapper/saltcheck.py
index d47b5cf6883..b0b94593809 100644
--- a/salt/client/ssh/wrapper/saltcheck.py
+++ b/salt/client/ssh/wrapper/saltcheck.py
@@ -9,6 +9,7 @@ import tarfile
import tempfile
from contextlib import closing
+import salt.fileclient
import salt.utils.files
import salt.utils.json
import salt.utils.url
@@ -28,65 +29,62 @@ def update_master_cache(states, saltenv="base"):
# Setup for copying states to gendir
gendir = tempfile.mkdtemp()
trans_tar = salt.utils.files.mkstemp()
- if "cp.fileclient_{}".format(id(__opts__)) not in __context__:
- __context__[
- "cp.fileclient_{}".format(id(__opts__))
- ] = salt.fileclient.get_file_client(__opts__)
-
- # generate cp.list_states output and save to gendir
- cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
- cp_output_file = os.path.join(gendir, "cp_output.txt")
- with salt.utils.files.fopen(cp_output_file, "w") as fp:
- fp.write(cp_output)
-
- # cp state directories to gendir
- already_processed = []
- sls_list = salt.utils.args.split_input(states)
- for state_name in sls_list:
- # generate low data for each state and save to gendir
- state_low_file = os.path.join(gendir, state_name + ".low")
- state_low_output = salt.utils.json.dumps(
- __salt__["state.show_low_sls"](state_name)
- )
- with salt.utils.files.fopen(state_low_file, "w") as fp:
- fp.write(state_low_output)
-
- state_name = state_name.replace(".", os.sep)
- if state_name in already_processed:
- log.debug("Already cached state for %s", state_name)
- else:
- file_copy_file = os.path.join(gendir, state_name + ".copy")
- log.debug("copying %s to %s", state_name, gendir)
- qualified_name = salt.utils.url.create(state_name, saltenv)
- # Duplicate cp.get_dir to gendir
- copy_result = __context__["cp.fileclient_{}".format(id(__opts__))].get_dir(
- qualified_name, gendir, saltenv
+ with salt.fileclient.get_file_client(__opts__) as cp_fileclient:
+
+ # generate cp.list_states output and save to gendir
+ cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]())
+ cp_output_file = os.path.join(gendir, "cp_output.txt")
+ with salt.utils.files.fopen(cp_output_file, "w") as fp:
+ fp.write(cp_output)
+
+ # cp state directories to gendir
+ already_processed = []
+ sls_list = salt.utils.args.split_input(states)
+ for state_name in sls_list:
+ # generate low data for each state and save to gendir
+ state_low_file = os.path.join(gendir, state_name + ".low")
+ state_low_output = salt.utils.json.dumps(
+ __salt__["state.show_low_sls"](state_name)
)
- if copy_result:
- copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
- copy_result_output = salt.utils.json.dumps(copy_result)
- with salt.utils.files.fopen(file_copy_file, "w") as fp:
- fp.write(copy_result_output)
- already_processed.append(state_name)
+ with salt.utils.files.fopen(state_low_file, "w") as fp:
+ fp.write(state_low_output)
+
+ state_name = state_name.replace(".", os.sep)
+ if state_name in already_processed:
+ log.debug("Already cached state for %s", state_name)
else:
- # If files were not copied, assume state.file.sls was given and just copy state
- state_name = os.path.dirname(state_name)
file_copy_file = os.path.join(gendir, state_name + ".copy")
- if state_name in already_processed:
- log.debug("Already cached state for %s", state_name)
+ log.debug("copying %s to %s", state_name, gendir)
+ qualified_name = salt.utils.url.create(state_name, saltenv)
+ # Duplicate cp.get_dir to gendir
+ copy_result = cp_fileclient.get_dir(qualified_name, gendir, saltenv)
+ if copy_result:
+ copy_result = [
+ dir.replace(gendir, state_cache) for dir in copy_result
+ ]
+ copy_result_output = salt.utils.json.dumps(copy_result)
+ with salt.utils.files.fopen(file_copy_file, "w") as fp:
+ fp.write(copy_result_output)
+ already_processed.append(state_name)
else:
- qualified_name = salt.utils.url.create(state_name, saltenv)
- copy_result = __context__[
- "cp.fileclient_{}".format(id(__opts__))
- ].get_dir(qualified_name, gendir, saltenv)
- if copy_result:
- copy_result = [
- dir.replace(gendir, state_cache) for dir in copy_result
- ]
- copy_result_output = salt.utils.json.dumps(copy_result)
- with salt.utils.files.fopen(file_copy_file, "w") as fp:
- fp.write(copy_result_output)
- already_processed.append(state_name)
+ # If files were not copied, assume state.file.sls was given and just copy state
+ state_name = os.path.dirname(state_name)
+ file_copy_file = os.path.join(gendir, state_name + ".copy")
+ if state_name in already_processed:
+ log.debug("Already cached state for %s", state_name)
+ else:
+ qualified_name = salt.utils.url.create(state_name, saltenv)
+ copy_result = cp_fileclient.get_dir(
+ qualified_name, gendir, saltenv
+ )
+ if copy_result:
+ copy_result = [
+ dir.replace(gendir, state_cache) for dir in copy_result
+ ]
+ copy_result_output = salt.utils.json.dumps(copy_result)
+ with salt.utils.files.fopen(file_copy_file, "w") as fp:
+ fp.write(copy_result_output)
+ already_processed.append(state_name)
# turn gendir into tarball and remove gendir
try:
diff --git a/salt/fileclient.py b/salt/fileclient.py
index fef5154a0be..f01a86dd0d4 100644
--- a/salt/fileclient.py
+++ b/salt/fileclient.py
@@ -849,7 +849,6 @@ class Client:
kwargs.pop("env")
kwargs["saltenv"] = saltenv
- url_data = urllib.parse.urlparse(url)
sfn = self.cache_file(url, saltenv, cachedir=cachedir)
if not sfn or not os.path.exists(sfn):
return ""
@@ -1165,13 +1164,8 @@ class RemoteClient(Client):
if not salt.utils.platform.is_windows():
hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
- try:
- mode_server = stat_server[0]
- except (IndexError, TypeError):
- mode_server = None
else:
hash_server = self.hash_file(path, saltenv)
- mode_server = None
# Check if file exists on server, before creating files and
# directories
@@ -1214,13 +1208,8 @@ class RemoteClient(Client):
if dest2check and os.path.isfile(dest2check):
if not salt.utils.platform.is_windows():
hash_local, stat_local = self.hash_and_stat_file(dest2check, saltenv)
- try:
- mode_local = stat_local[0]
- except (IndexError, TypeError):
- mode_local = None
else:
hash_local = self.hash_file(dest2check, saltenv)
- mode_local = None
if hash_local == hash_server:
return dest2check
diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py
index f7344b66ac6..69b722f0c95 100644
--- a/salt/modules/dockermod.py
+++ b/salt/modules/dockermod.py
@@ -6667,14 +6667,6 @@ def script_retcode(
)["retcode"]
-def _mk_fileclient():
- """
- Create a file client and add it to the context.
- """
- if "cp.fileclient" not in __context__:
- __context__["cp.fileclient"] = salt.fileclient.get_file_client(__opts__)
-
-
def _generate_tmp_path():
return os.path.join("/tmp", "salt.docker.{}".format(uuid.uuid4().hex[:6]))
@@ -6688,11 +6680,10 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs=""
# reuse it from salt.ssh, however this function should
# be somewhere else
refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs)
- _mk_fileclient()
- trans_tar = salt.client.ssh.state.prep_trans_tar(
- __context__["cp.fileclient"], chunks, refs, pillar, name
- )
- return trans_tar
+ with salt.fileclient.get_file_client(__opts__) as fileclient:
+ return salt.client.ssh.state.prep_trans_tar(
+ fileclient, chunks, refs, pillar, name
+ )
def _compile_state(sls_opts, mods=None):
diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py
index 0dfab4cc579..26312b3bd53 100644
--- a/salt/pillar/__init__.py
+++ b/salt/pillar/__init__.py
@@ -9,7 +9,6 @@ import logging
import os
import sys
import traceback
-import uuid
import salt.channel.client
import salt.ext.tornado.gen
@@ -1351,6 +1350,11 @@ class Pillar:
if hasattr(self, "_closing") and self._closing:
return
self._closing = True
+ if self.client:
+ try:
+ self.client.destroy()
+ except AttributeError:
+ pass
# pylint: disable=W1701
def __del__(self):
diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py
index 7fd4deb6c2a..9abd418c42c 100644
--- a/salt/states/ansiblegate.py
+++ b/salt/states/ansiblegate.py
@@ -32,12 +32,10 @@ state:
- state: installed
"""
-
import logging
import os
import sys
-# Import salt modules
import salt.fileclient
import salt.utils.decorators.path
from salt.utils.decorators import depends
@@ -108,13 +106,6 @@ def __virtual__():
return __virtualname__
-def _client():
- """
- Get a fileclient
- """
- return salt.fileclient.get_file_client(__opts__)
-
-
def _changes(plays):
"""
Find changes in ansible return data
@@ -171,7 +162,7 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
}
if git_repo:
if not isinstance(rundir, str) or not os.path.isdir(rundir):
- with _client() as client:
+ with salt.fileclient.get_file_client(__opts__) as client:
rundir = client._extrn_path(git_repo, "base")
log.trace("rundir set to %s", rundir)
if not isinstance(git_kwargs, dict):
diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py
index 2a858feee98..0c645bbc3bb 100644
--- a/salt/utils/asynchronous.py
+++ b/salt/utils/asynchronous.py
@@ -131,7 +131,7 @@ class SyncWrapper:
result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
results.append(True)
results.append(result)
- except Exception as exc: # pylint: disable=broad-except
+ except Exception: # pylint: disable=broad-except
results.append(False)
results.append(sys.exc_info())
diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py
index fcc5aec497e..a6a8a279605 100644
--- a/salt/utils/jinja.py
+++ b/salt/utils/jinja.py
@@ -58,19 +58,6 @@ class SaltCacheLoader(BaseLoader):
and only loaded once per loader instance.
"""
- _cached_pillar_client = None
- _cached_client = None
-
- @classmethod
- def shutdown(cls):
- for attr in ("_cached_client", "_cached_pillar_client"):
- client = getattr(cls, attr, None)
- if client is not None:
- # PillarClient and LocalClient objects do not have a destroy method
- if hasattr(client, "destroy"):
- client.destroy()
- setattr(cls, attr, None)
-
def __init__(
self,
opts,
@@ -93,8 +80,7 @@ class SaltCacheLoader(BaseLoader):
log.debug("Jinja search path: %s", self.searchpath)
self.cached = []
self._file_client = _file_client
- # Instantiate the fileclient
- self.file_client()
+ self._close_file_client = _file_client is None
def file_client(self):
"""
@@ -108,18 +94,10 @@ class SaltCacheLoader(BaseLoader):
or not hasattr(self._file_client, "opts")
or self._file_client.opts["file_roots"] != self.opts["file_roots"]
):
- attr = "_cached_pillar_client" if self.pillar_rend else "_cached_client"
- cached_client = getattr(self, attr, None)
- if (
- cached_client is None
- or not hasattr(cached_client, "opts")
- or cached_client.opts["file_roots"] != self.opts["file_roots"]
- ):
- cached_client = salt.fileclient.get_file_client(
- self.opts, self.pillar_rend
- )
- setattr(SaltCacheLoader, attr, cached_client)
- self._file_client = cached_client
+ self._file_client = salt.fileclient.get_file_client(
+ self.opts, self.pillar_rend
+ )
+ self._close_file_client = True
return self._file_client
def cache_file(self, template):
@@ -221,6 +199,27 @@ class SaltCacheLoader(BaseLoader):
# there is no template file within searchpaths
raise TemplateNotFound(template)
+ def destroy(self):
+ if self._close_file_client is False:
+ return
+ if self._file_client is None:
+ return
+ file_client = self._file_client
+ self._file_client = None
+
+ try:
+ file_client.destroy()
+ except AttributeError:
+ # PillarClient and LocalClient objects do not have a destroy method
+ pass
+
+ def __enter__(self):
+ self.file_client()
+ return self
+
+ def __exit__(self, *args):
+ self.destroy()
+
class PrintableDict(OrderedDict):
"""
diff --git a/salt/utils/mako.py b/salt/utils/mako.py
index 69618de9837..037d5d86deb 100644
--- a/salt/utils/mako.py
+++ b/salt/utils/mako.py
@@ -97,3 +97,10 @@ if HAS_MAKO:
self.cache[fpath] = self.file_client().get_file(
fpath, "", True, self.saltenv
)
+
+ def destroy(self):
+ if self.client:
+ try:
+ self.client.destroy()
+ except AttributeError:
+ pass
diff --git a/salt/utils/templates.py b/salt/utils/templates.py
index 4947b820a36..4a8adf2a14f 100644
--- a/salt/utils/templates.py
+++ b/salt/utils/templates.py
@@ -362,163 +362,169 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None):
elif tmplstr.endswith("\n"):
newline = "\n"
- if not saltenv:
- if tmplpath:
- loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath))
- else:
- loader = salt.utils.jinja.SaltCacheLoader(
- opts,
- saltenv,
- pillar_rend=context.get("_pillar_rend", False),
- _file_client=file_client,
- )
+ try:
+ if not saltenv:
+ if tmplpath:
+ loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath))
+ else:
+ loader = salt.utils.jinja.SaltCacheLoader(
+ opts,
+ saltenv,
+ pillar_rend=context.get("_pillar_rend", False),
+ _file_client=file_client,
+ )
- env_args = {"extensions": [], "loader": loader}
-
- if hasattr(jinja2.ext, "with_"):
- env_args["extensions"].append("jinja2.ext.with_")
- if hasattr(jinja2.ext, "do"):
- env_args["extensions"].append("jinja2.ext.do")
- if hasattr(jinja2.ext, "loopcontrols"):
- env_args["extensions"].append("jinja2.ext.loopcontrols")
- env_args["extensions"].append(salt.utils.jinja.SerializerExtension)
-
- opt_jinja_env = opts.get("jinja_env", {})
- opt_jinja_sls_env = opts.get("jinja_sls_env", {})
-
- opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {}
- opt_jinja_sls_env = opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {}
-
- # Pass through trim_blocks and lstrip_blocks Jinja parameters
- # trim_blocks removes newlines around Jinja blocks
- # lstrip_blocks strips tabs and spaces from the beginning of
- # line to the start of a block.
- if opts.get("jinja_trim_blocks", False):
- log.debug("Jinja2 trim_blocks is enabled")
- log.warning(
- "jinja_trim_blocks is deprecated and will be removed in a future release,"
- " please use jinja_env and/or jinja_sls_env instead"
- )
- opt_jinja_env["trim_blocks"] = True
- opt_jinja_sls_env["trim_blocks"] = True
- if opts.get("jinja_lstrip_blocks", False):
- log.debug("Jinja2 lstrip_blocks is enabled")
- log.warning(
- "jinja_lstrip_blocks is deprecated and will be removed in a future release,"
- " please use jinja_env and/or jinja_sls_env instead"
- )
- opt_jinja_env["lstrip_blocks"] = True
- opt_jinja_sls_env["lstrip_blocks"] = True
-
- def opt_jinja_env_helper(opts, optname):
- for k, v in opts.items():
- k = k.lower()
- if hasattr(jinja2.defaults, k.upper()):
- log.debug("Jinja2 environment %s was set to %s by %s", k, v, optname)
- env_args[k] = v
- else:
- log.warning("Jinja2 environment %s is not recognized", k)
+ env_args = {"extensions": [], "loader": loader}
- if "sls" in context and context["sls"] != "":
- opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env")
- else:
- opt_jinja_env_helper(opt_jinja_env, "jinja_env")
+ if hasattr(jinja2.ext, "with_"):
+ env_args["extensions"].append("jinja2.ext.with_")
+ if hasattr(jinja2.ext, "do"):
+ env_args["extensions"].append("jinja2.ext.do")
+ if hasattr(jinja2.ext, "loopcontrols"):
+ env_args["extensions"].append("jinja2.ext.loopcontrols")
+ env_args["extensions"].append(salt.utils.jinja.SerializerExtension)
- if opts.get("allow_undefined", False):
- jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args)
- else:
- jinja_env = jinja2.sandbox.SandboxedEnvironment(
- undefined=jinja2.StrictUndefined, **env_args
- )
+ opt_jinja_env = opts.get("jinja_env", {})
+ opt_jinja_sls_env = opts.get("jinja_sls_env", {})
- indent_filter = jinja_env.filters.get("indent")
- jinja_env.tests.update(JinjaTest.salt_jinja_tests)
- jinja_env.filters.update(JinjaFilter.salt_jinja_filters)
- if salt.utils.jinja.JINJA_VERSION >= Version("2.11"):
- # Use the existing indent filter on Jinja versions where it's not broken
- jinja_env.filters["indent"] = indent_filter
- jinja_env.globals.update(JinjaGlobal.salt_jinja_globals)
-
- # globals
- jinja_env.globals["odict"] = OrderedDict
- jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context
-
- jinja_env.tests["list"] = salt.utils.data.is_list
-
- decoded_context = {}
- for key, value in context.items():
- if not isinstance(value, str):
- if isinstance(value, NamedLoaderContext):
- decoded_context[key] = value.value()
- else:
- decoded_context[key] = value
- continue
+ opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {}
+ opt_jinja_sls_env = (
+ opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {}
+ )
- try:
- decoded_context[key] = salt.utils.stringutils.to_unicode(
- value, encoding=SLS_ENCODING
+ # Pass through trim_blocks and lstrip_blocks Jinja parameters
+ # trim_blocks removes newlines around Jinja blocks
+ # lstrip_blocks strips tabs and spaces from the beginning of
+ # line to the start of a block.
+ if opts.get("jinja_trim_blocks", False):
+ log.debug("Jinja2 trim_blocks is enabled")
+ log.warning(
+ "jinja_trim_blocks is deprecated and will be removed in a future release,"
+ " please use jinja_env and/or jinja_sls_env instead"
)
- except UnicodeDecodeError as ex:
- log.debug(
- "Failed to decode using default encoding (%s), trying system encoding",
- SLS_ENCODING,
+ opt_jinja_env["trim_blocks"] = True
+ opt_jinja_sls_env["trim_blocks"] = True
+ if opts.get("jinja_lstrip_blocks", False):
+ log.debug("Jinja2 lstrip_blocks is enabled")
+ log.warning(
+ "jinja_lstrip_blocks is deprecated and will be removed in a future release,"
+ " please use jinja_env and/or jinja_sls_env instead"
)
- decoded_context[key] = salt.utils.data.decode(value)
+ opt_jinja_env["lstrip_blocks"] = True
+ opt_jinja_sls_env["lstrip_blocks"] = True
+
+ def opt_jinja_env_helper(opts, optname):
+ for k, v in opts.items():
+ k = k.lower()
+ if hasattr(jinja2.defaults, k.upper()):
+ log.debug(
+ "Jinja2 environment %s was set to %s by %s", k, v, optname
+ )
+ env_args[k] = v
+ else:
+ log.warning("Jinja2 environment %s is not recognized", k)
- jinja_env.globals.update(decoded_context)
- try:
- template = jinja_env.from_string(tmplstr)
- output = template.render(**decoded_context)
- except jinja2.exceptions.UndefinedError as exc:
- trace = traceback.extract_tb(sys.exc_info()[2])
- line, out = _get_jinja_error(trace, context=decoded_context)
- if not line:
- tmplstr = ""
- raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr)
- except (
- jinja2.exceptions.TemplateRuntimeError,
- jinja2.exceptions.TemplateSyntaxError,
- jinja2.exceptions.SecurityError,
- ) as exc:
- trace = traceback.extract_tb(sys.exc_info()[2])
- line, out = _get_jinja_error(trace, context=decoded_context)
- if not line:
- tmplstr = ""
- raise SaltRenderError(
- "Jinja syntax error: {}{}".format(exc, out), line, tmplstr
- )
- except (SaltInvocationError, CommandExecutionError) as exc:
- trace = traceback.extract_tb(sys.exc_info()[2])
- line, out = _get_jinja_error(trace, context=decoded_context)
- if not line:
- tmplstr = ""
- raise SaltRenderError(
- "Problem running salt function in Jinja template: {}{}".format(exc, out),
- line,
- tmplstr,
- )
- except Exception as exc: # pylint: disable=broad-except
- tracestr = traceback.format_exc()
- trace = traceback.extract_tb(sys.exc_info()[2])
- line, out = _get_jinja_error(trace, context=decoded_context)
- if not line:
- tmplstr = ""
+ if "sls" in context and context["sls"] != "":
+ opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env")
else:
- tmplstr += "\n{}".format(tracestr)
- log.debug("Jinja Error")
- log.debug("Exception:", exc_info=True)
- log.debug("Out: %s", out)
- log.debug("Line: %s", line)
- log.debug("TmplStr: %s", tmplstr)
- log.debug("TraceStr: %s", tracestr)
+ opt_jinja_env_helper(opt_jinja_env, "jinja_env")
- raise SaltRenderError(
- "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr
- )
+ if opts.get("allow_undefined", False):
+ jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args)
+ else:
+ jinja_env = jinja2.sandbox.SandboxedEnvironment(
+ undefined=jinja2.StrictUndefined, **env_args
+ )
+
+ indent_filter = jinja_env.filters.get("indent")
+ jinja_env.tests.update(JinjaTest.salt_jinja_tests)
+ jinja_env.filters.update(JinjaFilter.salt_jinja_filters)
+ if salt.utils.jinja.JINJA_VERSION >= Version("2.11"):
+ # Use the existing indent filter on Jinja versions where it's not broken
+ jinja_env.filters["indent"] = indent_filter
+ jinja_env.globals.update(JinjaGlobal.salt_jinja_globals)
+
+ # globals
+ jinja_env.globals["odict"] = OrderedDict
+ jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context
+
+ jinja_env.tests["list"] = salt.utils.data.is_list
+
+ decoded_context = {}
+ for key, value in context.items():
+ if not isinstance(value, str):
+ if isinstance(value, NamedLoaderContext):
+ decoded_context[key] = value.value()
+ else:
+ decoded_context[key] = value
+ continue
+
+ try:
+ decoded_context[key] = salt.utils.stringutils.to_unicode(
+ value, encoding=SLS_ENCODING
+ )
+ except UnicodeDecodeError:
+ log.debug(
+ "Failed to decode using default encoding (%s), trying system encoding",
+ SLS_ENCODING,
+ )
+ decoded_context[key] = salt.utils.data.decode(value)
+
+ jinja_env.globals.update(decoded_context)
+ try:
+ template = jinja_env.from_string(tmplstr)
+ output = template.render(**decoded_context)
+ except jinja2.exceptions.UndefinedError as exc:
+ trace = traceback.extract_tb(sys.exc_info()[2])
+ line, out = _get_jinja_error(trace, context=decoded_context)
+ if not line:
+ tmplstr = ""
+ raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr)
+ except (
+ jinja2.exceptions.TemplateRuntimeError,
+ jinja2.exceptions.TemplateSyntaxError,
+ jinja2.exceptions.SecurityError,
+ ) as exc:
+ trace = traceback.extract_tb(sys.exc_info()[2])
+ line, out = _get_jinja_error(trace, context=decoded_context)
+ if not line:
+ tmplstr = ""
+ raise SaltRenderError(
+ "Jinja syntax error: {}{}".format(exc, out), line, tmplstr
+ )
+ except (SaltInvocationError, CommandExecutionError) as exc:
+ trace = traceback.extract_tb(sys.exc_info()[2])
+ line, out = _get_jinja_error(trace, context=decoded_context)
+ if not line:
+ tmplstr = ""
+ raise SaltRenderError(
+ "Problem running salt function in Jinja template: {}{}".format(
+ exc, out
+ ),
+ line,
+ tmplstr,
+ )
+ except Exception as exc: # pylint: disable=broad-except
+ tracestr = traceback.format_exc()
+ trace = traceback.extract_tb(sys.exc_info()[2])
+ line, out = _get_jinja_error(trace, context=decoded_context)
+ if not line:
+ tmplstr = ""
+ else:
+ tmplstr += "\n{}".format(tracestr)
+ log.debug("Jinja Error")
+ log.debug("Exception:", exc_info=True)
+ log.debug("Out: %s", out)
+ log.debug("Line: %s", line)
+ log.debug("TmplStr: %s", tmplstr)
+ log.debug("TraceStr: %s", tracestr)
+
+ raise SaltRenderError(
+ "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr
+ )
finally:
- if loader and hasattr(loader, "_file_client"):
- if hasattr(loader._file_client, "destroy"):
- loader._file_client.destroy()
+ if loader and isinstance(loader, salt.utils.jinja.SaltCacheLoader):
+ loader.destroy()
# Workaround a bug in Jinja that removes the final newline
# (https://github.com/mitsuhiko/jinja2/issues/75)
@@ -569,9 +575,8 @@ def render_mako_tmpl(tmplstr, context, tmplpath=None):
except Exception: # pylint: disable=broad-except
raise SaltRenderError(mako.exceptions.text_error_template().render())
finally:
- if lookup and hasattr(lookup, "_file_client"):
- if hasattr(lookup._file_client, "destroy"):
- lookup._file_client.destroy()
+ if lookup and isinstance(lookup, SaltMakoTemplateLookup):
+ lookup.destroy()
def render_wempy_tmpl(tmplstr, context, tmplpath=None):
diff --git a/tests/pytests/integration/states/test_include.py b/tests/pytests/integration/states/test_include.py
new file mode 100644
index 00000000000..f814328c5e4
--- /dev/null
+++ b/tests/pytests/integration/states/test_include.py
@@ -0,0 +1,40 @@
+"""
+Integration tests for the jinja includes in states
+"""
+import logging
+
+import pytest
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.slow_test
+def test_issue_64111(salt_master, salt_minion, salt_call_cli):
+ # This needs to be an integration test. A functional test does not trigger
+ # the issue fixed.
+
+ macros_jinja = """
+ {% macro a_jinja_macro(arg) -%}
+ {{ arg }}
+ {%- endmacro %}
+ """
+
+ init_sls = """
+ include:
+ - common.file1
+ """
+
+ file1_sls = """
+ {% from 'common/macros.jinja' import a_jinja_macro with context %}
+
+ a state id:
+ cmd.run:
+ - name: echo {{ a_jinja_macro("hello world") }}
+ """
+ tf = salt_master.state_tree.base.temp_file
+
+ with tf("common/macros.jinja", macros_jinja):
+ with tf("common/init.sls", init_sls):
+ with tf("common/file1.sls", file1_sls):
+ ret = salt_call_cli.run("state.apply", "common")
+ assert ret.returncode == 0
diff --git a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
index 38c5ce5b724..e0f5fa158ff 100644
--- a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
+++ b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py
@@ -15,7 +15,7 @@ import salt.utils.json # pylint: disable=unused-import
import salt.utils.stringutils # pylint: disable=unused-import
import salt.utils.yaml # pylint: disable=unused-import
from salt.utils.jinja import SaltCacheLoader
-from tests.support.mock import Mock, patch
+from tests.support.mock import Mock, call, patch
@pytest.fixture
@@ -224,14 +224,45 @@ def test_file_client_kwarg(minion_opts, mock_file_client):
assert loader._file_client is mock_file_client
-def test_cache_loader_shutdown(minion_opts, mock_file_client):
+def test_cache_loader_passed_file_client(minion_opts, mock_file_client):
"""
The shudown method can be called without raising an exception when the
file_client does not have a destroy method
"""
- assert not hasattr(mock_file_client, "destroy")
- mock_file_client.opts = minion_opts
- loader = SaltCacheLoader(minion_opts, _file_client=mock_file_client)
- assert loader._file_client is mock_file_client
- # Shutdown method should not raise any exceptions
- loader.shutdown()
+ # Test SaltCacheLoader creating and destroying the file client created
+ file_client = Mock()
+ with patch("salt.fileclient.get_file_client", return_value=file_client):
+ loader = SaltCacheLoader(minion_opts)
+ assert loader._file_client is None
+ with loader:
+ assert loader._file_client is file_client
+ assert loader._file_client is None
+ assert file_client.mock_calls == [call.destroy()]
+
+ # Test SaltCacheLoader reusing the file client passed
+ file_client = Mock()
+ file_client.opts = {"file_roots": minion_opts["file_roots"]}
+ with patch("salt.fileclient.get_file_client", return_value=Mock()):
+ loader = SaltCacheLoader(minion_opts, _file_client=file_client)
+ assert loader._file_client is file_client
+ with loader:
+ assert loader._file_client is file_client
+ assert loader._file_client is file_client
+ assert file_client.mock_calls == []
+
+ # Test SaltCacheLoader creating a client even though a file client was
+ # passed because the "file_roots" option is different, and, as such,
+ # the destroy method on the new file client is called, but not on the
+ # file client passed in.
+ file_client = Mock()
+ file_client.opts = {"file_roots": ""}
+ new_file_client = Mock()
+ with patch("salt.fileclient.get_file_client", return_value=new_file_client):
+ loader = SaltCacheLoader(minion_opts, _file_client=file_client)
+ assert loader._file_client is file_client
+ with loader:
+ assert loader._file_client is not file_client
+ assert loader._file_client is new_file_client
+ assert loader._file_client is None
+ assert file_client.mock_calls == []
+ assert new_file_client.mock_calls == [call.destroy()]
--
2.40.0
