File use-salt-bundle-in-dockermod.patch of Package salt
From b0891f83afa354c4b1f803af8a679ecf5a7fb63c Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 27 Jun 2022 17:59:24 +0300
Subject: [PATCH] Use Salt Bundle in dockermod
* Use Salt Bundle for salt calls in dockermod
* Add test of performing a call with the Salt Bundle
---
salt/modules/dockermod.py | 197 +++++++++++++++---
.../unit/modules/dockermod/test_module.py | 78 ++++++-
2 files changed, 241 insertions(+), 34 deletions(-)
diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py
index 6870c26b0e..8b6ab8058e 100644
--- a/salt/modules/dockermod.py
+++ b/salt/modules/dockermod.py
@@ -201,14 +201,19 @@ import copy
import fnmatch
import functools
import gzip
+import hashlib
import json
import logging
import os
+import pathlib
import pipes
import re
import shutil
import string
import subprocess
+import sys
+import tarfile
+import tempfile
import time
import uuid
@@ -6698,6 +6703,111 @@ def _compile_state(sls_opts, mods=None):
return st_.state.compile_high_data(high_data)
+def gen_venv_tar(cachedir, venv_dest_dir, venv_name):
+ """
+ Generate tarball with the Salt Bundle if required and return the path to it
+ """
+ exec_path = pathlib.Path(sys.executable).parts
+ venv_dir_name = "venv-salt-minion"
+ if venv_dir_name not in exec_path:
+ return None
+
+ venv_tar = os.path.join(cachedir, "venv-salt.tgz")
+ venv_hash = os.path.join(cachedir, "venv-salt.hash")
+ venv_lock = os.path.join(cachedir, ".venv-salt.lock")
+
+ venv_path = os.path.join(*exec_path[0 : exec_path.index(venv_dir_name)])
+
+ with __utils__["files.flopen"](venv_lock, "w"):
+ start_dir = os.getcwd()
+ venv_hash_file = os.path.join(venv_path, venv_dir_name, "venv-hash.txt")
+ try:
+ with __utils__["files.fopen"](venv_hash_file, "r") as fh:
+ venv_hash_src = fh.readline().strip()
+ except Exception: # pylint: disable=broad-except
+ # It makes no sense what caused the exception
+ # Just calculate the hash different way
+ for cmd in ("rpm -qi venv-salt-minion", "dpkg -s venv-salt-minion"):
+ ret = __salt__["cmd.run_all"](
+ cmd,
+ python_shell=True,
+ clean_env=True,
+ env={"LANG": "C", "LANGUAGE": "C", "LC_ALL": "C"},
+ )
+ if ret.get("retcode") == 0 and ret.get("stdout"):
+ venv_hash_src = hashlib.sha256(
+ "{}\n".format(ret.get("stdout")).encode()
+ ).hexdigest()
+ break
+ try:
+ with __utils__["files.fopen"](venv_hash, "r") as fh:
+ venv_hash_dest = fh.readline().strip()
+ except Exception: # pylint: disable=broad-except
+ # It makes no sense what caused the exception
+ # Set the hash to impossible value to force new tarball creation
+ venv_hash_dest = "UNKNOWN"
+ if venv_hash_src == venv_hash_dest and os.path.isfile(venv_tar):
+ return venv_tar
+ try:
+ tfd, tmp_venv_tar = tempfile.mkstemp(
+ dir=cachedir,
+ prefix=".venv-",
+ suffix=os.path.splitext(venv_tar)[1],
+ )
+ os.close(tfd)
+
+ os.chdir(venv_path)
+ tfp = tarfile.open(tmp_venv_tar, "w:gz")
+
+ for root, dirs, files in salt.utils.path.os_walk(
+ venv_dir_name, followlinks=True
+ ):
+ for name in files:
+ if name == "python" and pathlib.Path(root).parts == (
+ venv_dir_name,
+ "bin",
+ ):
+ tfd, tmp_python_file = tempfile.mkstemp(
+ dir=cachedir,
+ prefix=".python-",
+ )
+ os.close(tfd)
+ try:
+ with __utils__["files.fopen"](
+ os.path.join(root, name), "r"
+ ) as fh_in:
+ with __utils__["files.fopen"](
+ tmp_python_file, "w"
+ ) as fh_out:
+ rd_lines = fh_in.readlines()
+ rd_lines = [
+ 'export VIRTUAL_ENV="{}"\n'.format(
+ os.path.join(venv_dest_dir, venv_name)
+ )
+ if line.startswith("export VIRTUAL_ENV=")
+ else line
+ for line in rd_lines
+ ]
+ fh_out.write("".join(rd_lines))
+ os.chmod(tmp_python_file, 0o755)
+ tfp.add(tmp_python_file, arcname=os.path.join(root, name))
+ continue
+ finally:
+ if os.path.isfile(tmp_python_file):
+ os.remove(tmp_python_file)
+ if not name.endswith((".pyc", ".pyo")):
+ tfp.add(os.path.join(root, name))
+
+ tfp.close()
+ shutil.move(tmp_venv_tar, venv_tar)
+ with __utils__["files.fopen"](venv_hash, "w") as fh:
+ fh.write("{}\n".format(venv_hash_src))
+ finally:
+ os.chdir(start_dir)
+
+ return venv_tar
+
+
def call(name, function, *args, **kwargs):
"""
Executes a Salt function inside a running container
@@ -6733,47 +6843,68 @@ def call(name, function, *args, **kwargs):
if function is None:
raise CommandExecutionError("Missing function parameter")
- # move salt into the container
- thin_path = __utils__["thin.gen_thin"](
- __opts__["cachedir"],
- extra_mods=__salt__["config.option"]("thin_extra_mods", ""),
- so_mods=__salt__["config.option"]("thin_so_mods", ""),
- )
- ret = copy_to(
- name, thin_path, os.path.join(thin_dest_path, os.path.basename(thin_path))
- )
+ venv_dest_path = "/var/tmp"
+ venv_name = "venv-salt-minion"
+ venv_tar = gen_venv_tar(__opts__["cachedir"], venv_dest_path, venv_name)
- # figure out available python interpreter inside the container (only Python3)
- pycmds = ("python3", "/usr/libexec/platform-python")
- container_python_bin = None
- for py_cmd in pycmds:
- cmd = [py_cmd] + ["--version"]
- ret = run_all(name, subprocess.list2cmdline(cmd))
- if ret["retcode"] == 0:
- container_python_bin = py_cmd
- break
- if not container_python_bin:
- raise CommandExecutionError(
- "Python interpreter cannot be found inside the container. Make sure Python is installed in the container"
+ if venv_tar is not None:
+ venv_python_bin = os.path.join(venv_dest_path, venv_name, "bin", "python")
+ dest_venv_tar = os.path.join(venv_dest_path, os.path.basename(venv_tar))
+ copy_to(name, venv_tar, dest_venv_tar, overwrite=True, makedirs=True)
+ run_all(
+ name,
+ subprocess.list2cmdline(
+ ["tar", "zxf", dest_venv_tar, "-C", venv_dest_path]
+ ),
+ )
+ run_all(name, subprocess.list2cmdline(["rm", "-f", dest_venv_tar]))
+ container_python_bin = venv_python_bin
+ thin_dest_path = os.path.join(venv_dest_path, venv_name)
+ thin_salt_call = os.path.join(thin_dest_path, "bin", "salt-call")
+ else:
+ # move salt into the container
+ thin_path = __utils__["thin.gen_thin"](
+ __opts__["cachedir"],
+ extra_mods=__salt__["config.option"]("thin_extra_mods", ""),
+ so_mods=__salt__["config.option"]("thin_so_mods", ""),
)
- # untar archive
- untar_cmd = [
- container_python_bin,
- "-c",
- 'import tarfile; tarfile.open("{0}/{1}").extractall(path="{0}")'.format(
- thin_dest_path, os.path.basename(thin_path)
- ),
- ]
- ret = run_all(name, subprocess.list2cmdline(untar_cmd))
- if ret["retcode"] != 0:
- return {"result": False, "comment": ret["stderr"]}
+ ret = copy_to(
+ name, thin_path, os.path.join(thin_dest_path, os.path.basename(thin_path))
+ )
+
+ # figure out available python interpreter inside the container (only Python3)
+ pycmds = ("python3", "/usr/libexec/platform-python")
+ container_python_bin = None
+ for py_cmd in pycmds:
+ cmd = [py_cmd] + ["--version"]
+ ret = run_all(name, subprocess.list2cmdline(cmd))
+ if ret["retcode"] == 0:
+ container_python_bin = py_cmd
+ break
+ if not container_python_bin:
+ raise CommandExecutionError(
+ "Python interpreter cannot be found inside the container. Make sure Python is installed in the container"
+ )
+
+ # untar archive
+ untar_cmd = [
+ container_python_bin,
+ "-c",
+ 'import tarfile; tarfile.open("{0}/{1}").extractall(path="{0}")'.format(
+ thin_dest_path, os.path.basename(thin_path)
+ ),
+ ]
+ ret = run_all(name, subprocess.list2cmdline(untar_cmd))
+ if ret["retcode"] != 0:
+ return {"result": False, "comment": ret["stderr"]}
+ thin_salt_call = os.path.join(thin_dest_path, "salt-call")
try:
salt_argv = (
[
container_python_bin,
- os.path.join(thin_dest_path, "salt-call"),
+ thin_salt_call,
"--metadata",
"--local",
"--log-file",
diff --git a/tests/pytests/unit/modules/dockermod/test_module.py b/tests/pytests/unit/modules/dockermod/test_module.py
index 8fb7806497..1ac7dff52a 100644
--- a/tests/pytests/unit/modules/dockermod/test_module.py
+++ b/tests/pytests/unit/modules/dockermod/test_module.py
@@ -3,6 +3,7 @@ Unit tests for the docker module
"""
import logging
+import sys
import pytest
@@ -26,6 +27,7 @@ def configure_loader_modules(minion_opts):
whitelist=[
"args",
"docker",
+ "files",
"json",
"state",
"thin",
@@ -880,13 +882,16 @@ def test_call_success():
client = Mock()
client.put_archive = Mock()
get_client_mock = MagicMock(return_value=client)
+ gen_venv_tar_mock = MagicMock(return_value=None)
context = {"docker.exec_driver": "docker-exec"}
salt_dunder = {"config.option": docker_config_mock}
with patch.object(docker_mod, "run_all", docker_run_all_mock), patch.object(
docker_mod, "copy_to", docker_copy_to_mock
- ), patch.object(docker_mod, "_get_client", get_client_mock), patch.dict(
+ ), patch.object(docker_mod, "_get_client", get_client_mock), patch.object(
+ docker_mod, "gen_venv_tar", gen_venv_tar_mock
+ ), patch.dict(
docker_mod.__opts__, {"cachedir": "/tmp"}
), patch.dict(
docker_mod.__salt__, salt_dunder
@@ -931,6 +936,11 @@ def test_call_success():
!= docker_run_all_mock.mock_calls[9][1][1]
)
+ # check the parameters of gen_venv_tar call
+ assert gen_venv_tar_mock.mock_calls[0][1][0] == "/tmp"
+ assert gen_venv_tar_mock.mock_calls[0][1][1] == "/var/tmp"
+ assert gen_venv_tar_mock.mock_calls[0][1][2] == "venv-salt-minion"
+
assert {"retcode": 0, "comment": "container cmd"} == ret
@@ -1352,3 +1362,69 @@ def test_port():
"bar": {"6666/tcp": ports["bar"]["6666/tcp"]},
"baz": {},
}
+
+
+@pytest.mark.slow_test
+def test_call_with_gen_venv_tar():
+ """
+ test module calling inside containers with the Salt Bundle
+ """
+ ret = None
+ docker_run_all_mock = MagicMock(
+ return_value={
+ "retcode": 0,
+ "stdout": '{"retcode": 0, "comment": "container cmd"}',
+ "stderr": "err",
+ }
+ )
+ docker_copy_to_mock = MagicMock(return_value={"retcode": 0})
+ docker_config_mock = MagicMock(return_value="")
+ docker_cmd_run_mock = MagicMock(
+ return_value={
+ "retcode": 0,
+ "stdout": "test",
+ }
+ )
+ client = Mock()
+ client.put_archive = Mock()
+ get_client_mock = MagicMock(return_value=client)
+
+ context = {"docker.exec_driver": "docker-exec"}
+ salt_dunder = {
+ "config.option": docker_config_mock,
+ "cmd.run_all": docker_cmd_run_mock,
+ }
+
+ with patch.object(docker_mod, "run_all", docker_run_all_mock), patch.object(
+ docker_mod, "copy_to", docker_copy_to_mock
+ ), patch.object(docker_mod, "_get_client", get_client_mock), patch.object(
+ sys, "executable", "/tmp/venv-salt-minion/bin/python"
+ ), patch.dict(
+ docker_mod.__opts__, {"cachedir": "/tmp"}
+ ), patch.dict(
+ docker_mod.__salt__, salt_dunder
+ ), patch.dict(
+ docker_mod.__context__, context
+ ):
+ ret = docker_mod.call("ID", "test.arg", 1, 2, arg1="val1")
+
+ # Check that the directory is different each time
+ # [ call(name, [args]), ...
+ assert "mkdir" in docker_run_all_mock.mock_calls[0][1][1]
+
+ assert (
+ "tar zxf /var/tmp/venv-salt.tgz -C /var/tmp"
+ == docker_run_all_mock.mock_calls[1][1][1]
+ )
+
+ assert docker_run_all_mock.mock_calls[3][1][1].startswith(
+ "/var/tmp/venv-salt-minion/bin/python /var/tmp/venv-salt-minion/bin/salt-call "
+ )
+
+ # check remove the salt bundle tarball
+ assert docker_run_all_mock.mock_calls[2][1][1] == "rm -f /var/tmp/venv-salt.tgz"
+
+ # check directory cleanup
+ assert docker_run_all_mock.mock_calls[4][1][1] == "rm -rf /var/tmp/venv-salt-minion"
+
+ assert {"retcode": 0, "comment": "container cmd"} == ret
--
2.39.2
