File improvements-on-ansiblegate-module-354.patch of Package salt.26881
From b58056da2f5a12e3d614650904039c0655ce1221 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Tue, 18 Jan 2022 19:41:03 +0100
Subject: [PATCH] Improvements on "ansiblegate" module (#354)
* Allow collecting Ansible Inventory from a minion
* Prevent crashing if ansible-playbook doesn't return JSON
* Add new 'ansible.discover_playbooks' method
* Include custom inventory when discovering Ansible playbooks
* Enhance 'ansible.discover_playbooks' to accept a list of locations
* Remove unused constants from Ansible utils
* Avoid string concatenation to calculate extra cmd args
* Add unit test for ansible.targets
* Improve Ansible roster targetting
* Add tests for new ansiblegate module functions
* Fix issue dealing with ungrouped targets on inventory
* Enable ansible utils for ansible roster tests
* Remove unnecessary code from Ansible utils
* Fix pylint issue
* Fix issue in documentation
Fix issue parsing errors in ansiblegate state module
---
salt/modules/ansiblegate.py | 167 +++++++++++++++++-
salt/roster/ansible.py | 17 +-
salt/states/ansiblegate.py | 12 +-
salt/utils/ansible.py | 41 +++++
.../pytests/unit/modules/test_ansiblegate.py | 99 ++++++++++-
.../example_playbooks/example-playbook2/hosts | 7 +
.../example-playbook2/site.yml | 28 +++
.../playbooks/example_playbooks/playbook1.yml | 5 +
tests/unit/roster/test_ansible.py | 2 +-
9 files changed, 367 insertions(+), 11 deletions(-)
create mode 100644 salt/utils/ansible.py
create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts
create mode 100644 tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml
create mode 100644 tests/unit/files/playbooks/example_playbooks/playbook1.yml
diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py
index 328d9b7b0a..f33be6a00e 100644
--- a/salt/modules/ansiblegate.py
+++ b/salt/modules/ansiblegate.py
@@ -17,6 +17,7 @@ any Ansible module to respond.
import fnmatch
import json
import logging
+import os
import subprocess
import sys
from tempfile import NamedTemporaryFile
@@ -365,7 +366,171 @@ def playbooks(
}
ret = __salt__["cmd.run_all"](**cmd_kwargs)
log.debug("Ansible Playbook Return: %s", ret)
- retdata = json.loads(ret["stdout"])
+ try:
+ retdata = json.loads(ret["stdout"])
+ except ValueError:
+ retdata = ret
if "retcode" in ret:
__context__["retcode"] = retdata["retcode"] = ret["retcode"]
return retdata
+
+
+def targets(**kwargs):
+ """
+ Return the inventory from an Ansible inventory_file
+
+ :param inventory:
+ The inventory file to read the inventory from. Default: "/etc/ansible/hosts"
+
+ :param yaml:
+ Return the inventory as yaml output. Default: False
+
+ :param export:
+ Return inventory as export format. Default: False
+
+ CLI Example:
+
+ .. code-block:: bash
+
+ salt 'ansiblehost' ansible.targets
+ salt 'ansiblehost' ansible.targets inventory=my_custom_inventory
+
+ """
+ return __utils__["ansible.targets"](**kwargs)
+
+
+def discover_playbooks(path=None,
+ locations=None,
+ playbook_extension=None,
+ hosts_filename=None,
+ syntax_check=False):
+ """
+ Discover Ansible playbooks stored under the given path or from multiple paths (locations)
+
+ This will search for files matching with the playbook file extension under the given
+ root path and will also look for files inside the first level of directories in this path.
+
+ The return of this function would be a dict like this:
+
+ .. code-block:: python
+
+ {
+ "/home/foobar/": {
+ "my_ansible_playbook.yml": {
+ "fullpath": "/home/foobar/playbooks/my_ansible_playbook.yml",
+ "custom_inventory": "/home/foobar/playbooks/hosts"
+ },
+ "another_playbook.yml": {
+ "fullpath": "/home/foobar/playbooks/another_playbook.yml",
+ "custom_inventory": "/home/foobar/playbooks/hosts"
+ },
+ "lamp_simple/site.yml": {
+ "fullpath": "/home/foobar/playbooks/lamp_simple/site.yml",
+ "custom_inventory": "/home/foobar/playbooks/lamp_simple/hosts"
+ },
+ "lamp_proxy/site.yml": {
+ "fullpath": "/home/foobar/playbooks/lamp_proxy/site.yml",
+ "custom_inventory": "/home/foobar/playbooks/lamp_proxy/hosts"
+ }
+ },
+ "/srv/playbooks/": {
+ "example_playbook/example.yml": {
+ "fullpath": "/srv/playbooks/example_playbook/example.yml",
+ "custom_inventory": "/srv/playbooks/example_playbook/hosts"
+ }
+ }
+ }
+
+ :param path:
+ Path to discover playbooks from.
+
+ :param locations:
+ List of paths to discover playbooks from.
+
+ :param playbook_extension:
+ File extension of playbooks file to search for. Default: "yml"
+
+ :param hosts_filename:
+ Filename of custom playbook inventory to search for. Default: "hosts"
+
+ :param syntax_check:
+ Skip playbooks that do not pass "ansible-playbook --syntax-check" validation. Default: False
+
+ :return:
+ The discovered playbooks under the given paths
+
+ CLI Example:
+
+ .. code-block:: bash
+
+ salt 'ansiblehost' ansible.discover_playbooks path=/srv/playbooks/
+ salt 'ansiblehost' ansible.discover_playbooks locations='["/srv/playbooks/", "/srv/foobar"]'
+
+ """
+
+ if not path and not locations:
+ raise CommandExecutionError("You have to specify either 'path' or 'locations' arguments")
+
+ if path and locations:
+ raise CommandExecutionError("You cannot specify 'path' and 'locations' at the same time")
+
+ if not playbook_extension:
+ playbook_extension = "yml"
+ if not hosts_filename:
+ hosts_filename = "hosts"
+
+ if path:
+ if not os.path.isabs(path):
+ raise CommandExecutionError("The given path is not an absolute path: {}".format(path))
+ if not os.path.isdir(path):
+ raise CommandExecutionError("The given path is not a directory: {}".format(path))
+ return {path: _explore_path(path, playbook_extension, hosts_filename, syntax_check)}
+
+ if locations:
+ all_ret = {}
+ for location in locations:
+ all_ret[location] = _explore_path(location, playbook_extension, hosts_filename, syntax_check)
+ return all_ret
+
+
+def _explore_path(path, playbook_extension, hosts_filename, syntax_check):
+ ret = {}
+
+ if not os.path.isabs(path):
+ log.error("The given path is not an absolute path: {}".format(path))
+ return ret
+ if not os.path.isdir(path):
+ log.error("The given path is not a directory: {}".format(path))
+ return ret
+
+ try:
+ # Check files in the given path
+ for _f in os.listdir(path):
+ _path = os.path.join(path, _f)
+ if os.path.isfile(_path) and _path.endswith("." + playbook_extension):
+ ret[_f] = {"fullpath": _path}
+ # Check for custom inventory file
+ if os.path.isfile(os.path.join(path, hosts_filename)):
+ ret[_f].update({"custom_inventory": os.path.join(path, hosts_filename)})
+ elif os.path.isdir(_path):
+ # Check files in the 1st level of subdirectories
+ for _f2 in os.listdir(_path):
+ _path2 = os.path.join(_path, _f2)
+ if os.path.isfile(_path2) and _path2.endswith("." + playbook_extension):
+ ret[os.path.join(_f, _f2)] = {"fullpath": _path2}
+ # Check for custom inventory file
+ if os.path.isfile(os.path.join(_path, hosts_filename)):
+ ret[os.path.join(_f, _f2)].update({"custom_inventory": os.path.join(_path, hosts_filename)})
+ except Exception as exc:
+ raise CommandExecutionError("There was an exception while discovering playbooks: {}".format(exc))
+
+ # Run syntax check validation
+ if syntax_check:
+ check_command = ["ansible-playbook", "--syntax-check"]
+ try:
+ for pb in list(ret):
+ if __salt__["cmd.retcode"](check_command + [ret[pb]]):
+ del ret[pb]
+ except Exception as exc:
+ raise CommandExecutionError("There was an exception while checking syntax of playbooks: {}".format(exc))
+ return ret
diff --git a/salt/roster/ansible.py b/salt/roster/ansible.py
index 7beaaf2075..d3b352de27 100644
--- a/salt/roster/ansible.py
+++ b/salt/roster/ansible.py
@@ -117,27 +117,32 @@ def targets(tgt, tgt_type="glob", **kwargs):
Return the targets from the ansible inventory_file
Default: /etc/salt/roster
"""
- inventory = __runner__["salt.cmd"](
- "cmd.run", "ansible-inventory -i {} --list".format(get_roster_file(__opts__))
- )
- __context__["inventory"] = __utils__["json.loads"](
- __utils__["stringutils.to_str"](inventory)
+ __context__["inventory"] = __utils__["ansible.targets"](
+ inventory=get_roster_file(__opts__), **kwargs
)
if tgt_type == "glob":
hosts = [
host for host in _get_hosts_from_group("all") if fnmatch.fnmatch(host, tgt)
]
+ elif tgt_type == "list":
+ hosts = [host for host in _get_hosts_from_group("all") if host in tgt]
elif tgt_type == "nodegroup":
hosts = _get_hosts_from_group(tgt)
+ else:
+ hosts = []
+
return {host: _get_hostvars(host) for host in hosts}
def _get_hosts_from_group(group):
inventory = __context__["inventory"]
+ if group not in inventory:
+ return []
hosts = [host for host in inventory[group].get("hosts", [])]
for child in inventory[group].get("children", []):
- if child != "ungrouped":
+ child_info = _get_hosts_from_group(child)
+ if child_info not in hosts:
hosts.extend(_get_hosts_from_group(child))
return hosts
diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py
index 4afe6a020d..af5cb0f0e5 100644
--- a/salt/states/ansiblegate.py
+++ b/salt/states/ansiblegate.py
@@ -184,7 +184,11 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
checks = __salt__["ansible.playbooks"](
name, rundir=rundir, check=True, diff=True, **ansible_kwargs
)
- if all(
+ if "stats" not in checks:
+ ret["comment"] = checks.get("stderr", checks)
+ ret["result"] = False
+ ret["changes"] = {}
+ elif all(
not check["changed"]
and not check["failures"]
and not check["unreachable"]
@@ -213,7 +217,11 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs=
results = __salt__["ansible.playbooks"](
name, rundir=rundir, diff=True, **ansible_kwargs
)
- if all(
+ if "stats" not in results:
+ ret["comment"] = results.get("stderr", results)
+ ret["result"] = False
+ ret["changes"] = {}
+ elif all(
not check["changed"]
and not check["failures"]
and not check["unreachable"]
diff --git a/salt/utils/ansible.py b/salt/utils/ansible.py
new file mode 100644
index 0000000000..1e14037fd3
--- /dev/null
+++ b/salt/utils/ansible.py
@@ -0,0 +1,41 @@
+import logging
+import os
+
+# Import Salt libs
+import salt.utils.json
+import salt.utils.path
+import salt.utils.stringutils
+import salt.modules.cmdmod
+from salt.exceptions import CommandExecutionError
+
+__virtualname__ = "ansible"
+
+log = logging.getLogger(__name__)
+
+
+def __virtual__(): # pylint: disable=expected-2-blank-lines-found-0
+ if salt.utils.path.which("ansible-inventory"):
+ return __virtualname__
+ return (False, "Install `ansible` to use inventory")
+
+
+def targets(inventory="/etc/ansible/hosts", **kwargs):
+ """
+ Return the targets from the ansible inventory_file
+ Default: /etc/salt/roster
+ """
+ if not os.path.isfile(inventory):
+ raise CommandExecutionError("Inventory file not found: {}".format(inventory))
+
+ extra_cmd = []
+ if "export" in kwargs:
+ extra_cmd.append("--export")
+ if "yaml" in kwargs:
+ extra_cmd.append("--yaml")
+ inv = salt.modules.cmdmod.run(
+ "ansible-inventory -i {} --list {}".format(inventory, " ".join(extra_cmd))
+ )
+ if kwargs.get("yaml", False):
+ return salt.utils.stringutils.to_str(inv)
+ else:
+ return salt.utils.json.loads(salt.utils.stringutils.to_str(inv))
diff --git a/tests/pytests/unit/modules/test_ansiblegate.py b/tests/pytests/unit/modules/test_ansiblegate.py
index 44c9b12acb..f357133000 100644
--- a/tests/pytests/unit/modules/test_ansiblegate.py
+++ b/tests/pytests/unit/modules/test_ansiblegate.py
@@ -1,9 +1,15 @@
# Author: Bo Maryniuk <bo@suse.de>
+import os
import pytest
+
+import salt.config
+import salt.loader
import salt.modules.ansiblegate as ansiblegate
import salt.utils.json
+import salt.utils.path
from tests.support.mock import ANY, MagicMock, patch
+from tests.support.runtests import RUNTIME_VARS
pytestmark = [
pytest.mark.skip_on_windows(reason="Not supported on Windows"),
@@ -12,7 +18,7 @@ pytestmark = [
@pytest.fixture
def configure_loader_modules():
- return {ansiblegate: {}}
+ return {ansiblegate: {"__utils__": {}}}
def test_ansible_module_help():
@@ -133,3 +139,94 @@ def test_ansible_playbooks_return_retcode():
):
ret = ansiblegate.playbooks("fake-playbook.yml")
assert "retcode" in ret
+
+
+def test_ansible_targets():
+ """
+ Test ansible.targets execution module function.
+ :return:
+ """
+ ansible_inventory_ret = """
+{
+ "_meta": {
+ "hostvars": {
+ "uyuni-stable-ansible-centos7-1.tf.local": {
+ "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key"
+ },
+ "uyuni-stable-ansible-centos7-2.tf.local": {
+ "ansible_ssh_private_key_file": "/etc/ansible/my_ansible_private_key"
+ }
+ }
+ },
+ "all": {
+ "children": [
+ "ungrouped"
+ ]
+ },
+ "ungrouped": {
+ "hosts": [
+ "uyuni-stable-ansible-centos7-1.tf.local",
+ "uyuni-stable-ansible-centos7-2.tf.local"
+ ]
+ }
+}
+ """
+ ansible_inventory_mock = MagicMock(return_value=ansible_inventory_ret)
+ with patch("salt.utils.path.which", MagicMock(return_value=True)):
+ opts = salt.config.DEFAULT_MINION_OPTS.copy()
+ utils = salt.loader.utils(opts, whitelist=["ansible"])
+ with patch("salt.modules.cmdmod.run", ansible_inventory_mock), patch.dict(
+ ansiblegate.__utils__, utils
+ ), patch("os.path.isfile", MagicMock(return_value=True)):
+ ret = ansiblegate.targets()
+ assert ansible_inventory_mock.call_args
+ assert "_meta" in ret
+ assert "uyuni-stable-ansible-centos7-1.tf.local" in ret["_meta"]["hostvars"]
+ assert (
+ "ansible_ssh_private_key_file"
+ in ret["_meta"]["hostvars"]["uyuni-stable-ansible-centos7-1.tf.local"]
+ )
+ assert "all" in ret
+ assert len(ret["ungrouped"]["hosts"]) == 2
+
+
+def test_ansible_discover_playbooks_single_path():
+ playbooks_dir = os.path.join(
+ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/"
+ )
+ ret = ansiblegate.discover_playbooks(playbooks_dir)
+ assert playbooks_dir in ret
+ assert ret[playbooks_dir]["playbook1.yml"] == {
+ "fullpath": os.path.join(playbooks_dir, "playbook1.yml")
+ }
+ assert ret[playbooks_dir]["example-playbook2/site.yml"] == {
+ "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"),
+ "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"),
+ }
+
+
+def test_ansible_discover_playbooks_single_path_using_parameters():
+ playbooks_dir = os.path.join(
+ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/"
+ )
+ ret = ansiblegate.discover_playbooks(
+ playbooks_dir, playbook_extension="foobar", hosts_filename="deadbeaf"
+ )
+ assert playbooks_dir in ret
+ assert ret[playbooks_dir] == {}
+
+
+def test_ansible_discover_playbooks_multiple_locations():
+ playbooks_dir = os.path.join(
+ RUNTIME_VARS.TESTS_DIR, "unit/files/playbooks/example_playbooks/"
+ )
+ ret = ansiblegate.discover_playbooks(locations=[playbooks_dir, "/tmp/foobar"])
+ assert playbooks_dir in ret
+ assert "/tmp/foobar" in ret
+ assert ret[playbooks_dir]["playbook1.yml"] == {
+ "fullpath": os.path.join(playbooks_dir, "playbook1.yml")
+ }
+ assert ret[playbooks_dir]["example-playbook2/site.yml"] == {
+ "fullpath": os.path.join(playbooks_dir, "example-playbook2/site.yml"),
+ "custom_inventory": os.path.join(playbooks_dir, "example-playbook2/hosts"),
+ }
diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts
new file mode 100644
index 0000000000..75783285f6
--- /dev/null
+++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/hosts
@@ -0,0 +1,7 @@
+[databases]
+host1
+host2
+
+[webservers]
+host3
+host4
diff --git a/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml
new file mode 100644
index 0000000000..a64ebd5e18
--- /dev/null
+++ b/tests/unit/files/playbooks/example_playbooks/example-playbook2/site.yml
@@ -0,0 +1,28 @@
+---
+- name: update web servers
+ hosts: webservers
+ remote_user: root
+
+ tasks:
+ - name: ensure apache is at the latest version
+ yum:
+ name: httpd
+ state: latest
+ - name: write the apache config file
+ template:
+ src: /srv/httpd.j2
+ dest: /etc/httpd.conf
+
+- name: update db servers
+ hosts: databases
+ remote_user: root
+
+ tasks:
+ - name: ensure postgresql is at the latest version
+ yum:
+ name: postgresql
+ state: latest
+ - name: ensure that postgresql is started
+ service:
+ name: postgresql
+ state: started
diff --git a/tests/unit/files/playbooks/example_playbooks/playbook1.yml b/tests/unit/files/playbooks/example_playbooks/playbook1.yml
new file mode 100644
index 0000000000..e258a101e1
--- /dev/null
+++ b/tests/unit/files/playbooks/example_playbooks/playbook1.yml
@@ -0,0 +1,5 @@
+---
+- hosts: all
+ gather_facts: false
+ tasks:
+ - ping:
diff --git a/tests/unit/roster/test_ansible.py b/tests/unit/roster/test_ansible.py
index 7f1144454b..c4ab8b7639 100644
--- a/tests/unit/roster/test_ansible.py
+++ b/tests/unit/roster/test_ansible.py
@@ -63,7 +63,7 @@ class AnsibleRosterTestCase(TestCase, mixins.LoaderModuleMockMixin):
opts = salt.config.master_config(
os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "master")
)
- utils = salt.loader.utils(opts, whitelist=["json", "stringutils"])
+ utils = salt.loader.utils(opts, whitelist=["json", "stringutils", "ansible"])
runner = salt.loader.runner(opts, utils=utils, whitelist=["salt"])
return {ansible: {"__utils__": utils, "__opts__": {}, "__runner__": runner}}
--
2.34.1
