From c4edbf522da7e17aa069071c0e27910122e7bcfc Mon Sep 17 00:00:00 2001 From: Madhu Kanoor Date: Thu, 29 Jun 2023 16:51:51 -0400 Subject: [PATCH] feat: support files and directories for inventory This is based on PR https://github.com/ansible/ansible-rulebook/pull/523 --- ansible_rulebook/app.py | 10 +++- ansible_rulebook/builtin.py | 14 ++++-- ansible_rulebook/cli.py | 1 + ansible_rulebook/exception.py | 5 ++ ansible_rulebook/util.py | 34 +++++++------ docs/usage.rst | 2 +- tests/e2e/test_actions.py | 6 +-- tests/e2e/test_runtime.py | 2 +- tests/test_engine.py | 69 ++++++++++++++------------ tests/test_examples.py | 93 ++++++++++++++++++----------------- 10 files changed, 134 insertions(+), 102 deletions(-) diff --git a/ansible_rulebook/app.py b/ansible_rulebook/app.py index 735d35e4..7dc053bd 100644 --- a/ansible_rulebook/app.py +++ b/ansible_rulebook/app.py @@ -31,7 +31,6 @@ from ansible_rulebook.engine import run_rulesets, start_source from ansible_rulebook.job_template_runner import job_template_runner from ansible_rulebook.rule_types import RuleSet, RuleSetQueue -from ansible_rulebook.util import load_inventory from ansible_rulebook.validators import Validate from ansible_rulebook.websocket import ( request_workload, @@ -41,6 +40,7 @@ from .exception import ( ControllerNeededException, InventoryNeededException, + InventoryNotFound, RulebookNotFoundException, WebSocketExchangeException, ) @@ -80,7 +80,7 @@ async def run(parsed_args: argparse.ArgumentParser) -> None: parsed_args, startup_args.variables ) if parsed_args.inventory: - startup_args.inventory = load_inventory(parsed_args.inventory) + startup_args.inventory = parsed_args.inventory startup_args.project_data_file = parsed_args.project_tarball startup_args.controller_url = parsed_args.controller_url startup_args.controller_token = parsed_args.controller_token @@ -239,6 +239,12 @@ def validate_actions(startup_args: StartupArgs) -> None: "which needs inventory to be defined" ) + if action.action in INVENTORY_ACTIONS and not os.path.exists( + startup_args.inventory + ): + raise InventoryNotFound( + f"Inventory {startup_args.inventory} not found" + ) if ( action.action == "run_job_template" and not startup_args.controller_url diff --git a/ansible_rulebook/builtin.py b/ansible_rulebook/builtin.py index e25fc706..8911092c 100644 --- a/ansible_rulebook/builtin.py +++ b/ansible_rulebook/builtin.py @@ -45,7 +45,7 @@ ) from .job_template_runner import job_template_runner from .messages import Shutdown -from .util import get_horizontal_rule, run_at +from .util import create_inventory, get_horizontal_rule, run_at logger = logging.getLogger(__name__) @@ -360,6 +360,7 @@ async def run_playbook( temp_dir, dict(playbook=playbook_name), hosts, + inventory, verbosity, json_mode, ) @@ -471,6 +472,7 @@ async def run_module( module_args=module_args_str, ), hosts, + inventory, verbosity, json_mode, ) @@ -503,6 +505,7 @@ async def call_runner( private_data_dir: str, runner_args: Dict, hosts: List, + inventory: str, verbosity: int = 0, json_mode: Optional[bool] = False, ): @@ -555,6 +558,11 @@ def cancel_callback(): verbosity=verbosity, event_handler=event_callback, cancel_callback=cancel_callback, + inventory=os.path.join( + private_data_dir, + "inventory", + os.path.basename(inventory), + ), json_mode=json_mode, **runner_args, ), @@ -623,8 +631,8 @@ async def pre_process_runner( with open(os.path.join(env_dir, "extravars"), "w") as f: f.write(yaml.dump(playbook_extra_vars)) os.mkdir(inventory_dir) - with open(os.path.join(inventory_dir, "hosts"), "w") as f: - f.write(inventory) + if inventory: + create_inventory(inventory_dir, inventory) os.mkdir(project_dir) logger.debug("project_data_file: %s", project_data_file) diff --git a/ansible_rulebook/cli.py b/ansible_rulebook/cli.py index 5a10cf5f..d3f2393b 100644 --- a/ansible_rulebook/cli.py +++ b/ansible_rulebook/cli.py @@ -90,6 +90,7 @@ def get_parser() -> argparse.ArgumentParser: "-i", "--inventory", help="Inventory", + default=os.environ.get("ANSIBLE_INVENTORY", ""), ) parser.add_argument( "-W", diff --git a/ansible_rulebook/exception.py b/ansible_rulebook/exception.py index d5437597..1aaadf2e 100644 --- a/ansible_rulebook/exception.py +++ b/ansible_rulebook/exception.py @@ -146,3 +146,8 @@ class WebSocketExchangeException(Exception): class UnsupportedActionException(Exception): pass + + +class InventoryNotFound(Exception): + + pass diff --git a/ansible_rulebook/util.py b/ansible_rulebook/util.py index 2a0503cd..beefbcbf 100644 --- a/ansible_rulebook/util.py +++ b/ansible_rulebook/util.py @@ -33,7 +33,10 @@ from packaging.version import InvalidVersion from ansible_rulebook.conf import settings -from ansible_rulebook.exception import InvalidFilterNameException +from ansible_rulebook.exception import ( + InvalidFilterNameException, + InventoryNotFound, +) logger = logging.getLogger(__name__) @@ -81,24 +84,14 @@ def substitute_variables( return value -def load_inventory(inventory_spec: str) -> Any: - if os.path.exists(inventory_spec): - with open(inventory_spec) as f: - inventory_data = f.read() - return inventory_data - return inventory_spec - - def collect_ansible_facts(inventory: str) -> List[Dict]: hosts_facts = [] with tempfile.TemporaryDirectory( prefix="gather_facts" ) as private_data_dir: - os.mkdir(os.path.join(private_data_dir, "inventory")) - with open( - os.path.join(private_data_dir, "inventory", "hosts"), "w" - ) as f: - f.write(inventory) + inventory_dir = os.path.join(private_data_dir, "inventory") + os.mkdir(inventory_dir) + create_inventory(inventory_dir, inventory) r = ansible_runner.run( private_data_dir=private_data_dir, @@ -244,6 +237,19 @@ async def send_session_stats(event_log: asyncio.Queue, stats: Dict): ) +def create_inventory(runner_inventory_dir: str, inventory: str) -> None: + if os.path.isfile(inventory): + shutil.copy(os.path.abspath(inventory), runner_inventory_dir) + elif os.path.exists(inventory): + shutil.copytree( + os.path.abspath(inventory), + runner_inventory_dir, + dirs_exist_ok=True, + ) + else: + raise InventoryNotFound(f"Inventory {inventory} not found") + + def _builtin_filter_path(name: str) -> Tuple[bool, str]: if not name.startswith(EDA_BUILTIN_FILTER_PREFIX): return False, "" diff --git a/docs/usage.rst b/docs/usage.rst index a7896a9d..3457a994 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -21,7 +21,7 @@ The `ansible-rulebook` CLI supports the following options: -S SOURCE_DIR, --source-dir SOURCE_DIR Source dir -i INVENTORY, --inventory INVENTORY - Inventory + Inventory can be a file or a directory -W WEBSOCKET_URL, --websocket-url WEBSOCKET_ADDRESS Connect the event log to a websocket --websocket-ssl-verify How to verify the wss connection diff --git a/tests/e2e/test_actions.py b/tests/e2e/test_actions.py index 505b0a99..d028249f 100644 --- a/tests/e2e/test_actions.py +++ b/tests/e2e/test_actions.py @@ -2,7 +2,6 @@ Module with tests for operators """ import logging -import pprint import re import subprocess @@ -55,9 +54,6 @@ def test_actions_sanity(update_environment): "DEFAULT_STARTUP_DELAY", ) - with open(inventory) as f: - inventory_data = pprint.pformat(f.read()) - LOGGER.info(f"Running command: {cmd}") result = subprocess.run( cmd, @@ -100,7 +96,7 @@ def test_actions_sanity(update_environment): with check: expected_debug_lines = [ "'hosts': ['all']", - f"'inventory': {inventory_data}", + f"'inventory': '{inventory}'", "'project_data_file': None,", "'ruleset': 'Test actions sanity'", "'source_rule_name': 'debug',", diff --git a/tests/e2e/test_runtime.py b/tests/e2e/test_runtime.py index 1f5c2ff8..8fc12f62 100644 --- a/tests/e2e/test_runtime.py +++ b/tests/e2e/test_runtime.py @@ -30,7 +30,7 @@ id="successful_rc", ), pytest.param( - "hello_events_with_var.yml", + "actions/test_run_playbook.yml", Path("nonexistent.yml"), "0.1", 1, diff --git a/tests/test_engine.py b/tests/test_engine.py index 7d55de5c..d5459ebf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -14,6 +14,7 @@ import asyncio import os +import tempfile from pprint import pprint from unittest.mock import patch @@ -33,7 +34,6 @@ from ansible_rulebook.messages import Shutdown from ansible_rulebook.rule_types import EventSource, EventSourceFilter from ansible_rulebook.rules_parser import parse_rule_sets -from ansible_rulebook.util import load_inventory from ansible_rulebook.validators import Validate @@ -250,7 +250,7 @@ async def test_run_rulesets(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -319,7 +319,9 @@ async def test_run_rules_simple(): queue.put_nowait(dict(i=2)) queue.put_nowait(Shutdown()) - await run_rulesets(event_log, ruleset_queues, dict(), "localhost") + await run_rulesets( + event_log, ruleset_queues, dict(), "playbooks/inventory.yml" + ) assert event_log.get_nowait()["type"] == "Action", "0" assert event_log.get_nowait()["type"] == "Action", "0.2" @@ -358,7 +360,7 @@ async def test_run_multiple_hosts(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory1.yml"), + "playbooks/inventory1.yml", ) checks = { @@ -390,7 +392,7 @@ async def test_run_multiple_hosts2(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory1.yml"), + "playbooks/inventory1.yml", ) assert event_log.get_nowait()["type"] == "Action", "1" @@ -417,7 +419,7 @@ async def test_run_multiple_hosts3(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) assert event_log.get_nowait()["type"] == "Action", "1" @@ -435,7 +437,9 @@ async def test_filters(): queue.put_nowait(dict(i=2)) queue.put_nowait(Shutdown()) - await run_rulesets(event_log, ruleset_queues, dict(), "localhost") + await run_rulesets( + event_log, ruleset_queues, dict(), "playbooks/inventory.yml" + ) assert event_log.get_nowait()["type"] == "Action", "0" assert event_log.get_nowait()["type"] == "Action", "0.2" @@ -472,7 +476,7 @@ async def test_run_rulesets_on_hosts(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory1.yml"), + "playbooks/inventory1.yml", ) checks = { @@ -491,30 +495,33 @@ async def test_run_assert_facts(): inventory = dict( all=dict(hosts=dict(localhost=dict(ansible_connection="local"))) ) - queue = ruleset_queues[0][1] - queue.put_nowait(dict()) - queue.put_nowait(dict(i=1, meta=dict(hosts="localhost"))) - queue.put_nowait(Shutdown()) - await run_rulesets( - event_log, - ruleset_queues, - dict(Naboo="naboo"), - yaml.dump(inventory), - ) + with tempfile.NamedTemporaryFile(mode="w+") as temp: + temp.write(yaml.dump(inventory)) + temp.flush() + queue = ruleset_queues[0][1] + queue.put_nowait(dict()) + queue.put_nowait(dict(i=1, meta=dict(hosts="localhost"))) + queue.put_nowait(Shutdown()) + await run_rulesets( + event_log, + ruleset_queues, + dict(Naboo="naboo"), + temp.name, + ) - assert event_log.get_nowait()["type"] == "EmptyEvent", "0" - assert event_log.get_nowait()["type"] == "Action", "0.1" - assert event_log.get_nowait()["type"] == "Job", "1.0" - for i in range(47): - assert event_log.get_nowait()["type"] == "AnsibleEvent", f"1.{i}" - - event = event_log.get_nowait() - assert event["type"] == "Action", "2.1" - assert event["action"] == "run_playbook", "2.2" - assert event["rc"] == 0, "2.3" - assert event["status"] == "successful", "2.4" - assert event_log.get_nowait()["type"] == "Shutdown", "4" - assert event_log.empty() + assert event_log.get_nowait()["type"] == "EmptyEvent", "0" + assert event_log.get_nowait()["type"] == "Action", "0.1" + assert event_log.get_nowait()["type"] == "Job", "1.0" + for i in range(47): + assert event_log.get_nowait()["type"] == "AnsibleEvent", f"1.{i}" + + event = event_log.get_nowait() + assert event["type"] == "Action", "2.1" + assert event["action"] == "run_playbook", "2.2" + assert event["rc"] == 0, "2.3" + assert event["status"] == "successful", "2.4" + assert event_log.get_nowait()["type"] == "Shutdown", "4" + assert event_log.empty() @pytest.mark.asyncio diff --git a/tests/test_examples.py b/tests/test_examples.py index 79100895..09b3856b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -26,7 +26,6 @@ ) from ansible_rulebook.job_template_runner import job_template_runner from ansible_rulebook.messages import Shutdown -from ansible_rulebook.util import load_inventory from .test_engine import get_queue_item, load_rulebook, validate_events @@ -637,7 +636,7 @@ async def test_21_run_playbook(rule, ansible_events): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -777,7 +776,7 @@ async def test_27_var_root(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) for _ in range(2): @@ -838,6 +837,7 @@ async def test_29_run_module(): event_log, ruleset_queues, dict(), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -875,6 +875,7 @@ async def test_30_run_module_missing(): event_log, ruleset_queues, dict(), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -907,6 +908,7 @@ async def test_31_run_module_missing_args(): event_log, ruleset_queues, dict(), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -939,6 +941,7 @@ async def test_32_run_module_fail(): event_log, ruleset_queues, dict(), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -1008,7 +1011,7 @@ async def test_36_multiple_rulesets_both_fired(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 4, @@ -1033,7 +1036,7 @@ async def test_37_hosts_facts(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -1056,7 +1059,7 @@ async def test_38_shutdown_action(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = event_log.get_nowait() @@ -1084,7 +1087,7 @@ async def test_40_in(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1106,7 +1109,7 @@ async def test_41_not_in(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1128,7 +1131,7 @@ async def test_42_contains(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1151,7 +1154,7 @@ async def test_43_not_contains(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1173,7 +1176,7 @@ async def test_44_in_and(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1196,7 +1199,7 @@ async def test_45_in_or(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 5, @@ -1221,7 +1224,7 @@ async def test_47_generic_plugin(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 3, @@ -1245,7 +1248,7 @@ async def test_48_echo(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1277,7 +1280,7 @@ async def test_49_float(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = event_log.get_nowait() assert event["type"] == "Action", "1" @@ -1322,7 +1325,7 @@ async def test_50_negation(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = event_log.get_nowait() assert event["type"] == "Action", "1" @@ -1386,7 +1389,7 @@ async def test_51_vars_namespace(): event_log, ruleset_queues, person, - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 6, @@ -1430,7 +1433,7 @@ async def test_51_vars_namespace_missing_key(): event_log, ruleset_queues, person, - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) assert str(exc_info.value) == "vars does not contain key: person.age" @@ -1446,7 +1449,7 @@ async def test_52_once_within(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1473,7 +1476,7 @@ async def test_53_once_within_multiple_hosts(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1500,7 +1503,7 @@ async def test_54_time_window(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) event = await get_queue_item(event_log, 10, 2) @@ -1536,7 +1539,7 @@ async def test_55_not_all(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1563,7 +1566,7 @@ async def test_56_once_after(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 2, @@ -1591,7 +1594,7 @@ async def test_57_once_after_multiple(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 7, @@ -1611,7 +1614,7 @@ async def test_58_string_search(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 7, @@ -1641,7 +1644,7 @@ async def test_59_multiple_actions(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { "max_events": 6, @@ -1668,7 +1671,7 @@ async def test_60_json_filter(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1692,7 +1695,7 @@ async def test_61_select_1(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1716,7 +1719,7 @@ async def test_62_select_2(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1742,7 +1745,7 @@ async def test_63_selectattr_1(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1767,7 +1770,7 @@ async def test_64_selectattr_2(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1791,7 +1794,7 @@ async def test_65_selectattr_3(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1823,7 +1826,7 @@ async def test_66_sleepy_playbook(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1855,7 +1858,7 @@ async def test_67_shutdown_now(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1881,7 +1884,7 @@ async def test_68_disabled_rule(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1905,7 +1908,7 @@ async def test_69_enhanced_debug(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1932,7 +1935,7 @@ async def test_70_null(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1960,7 +1963,7 @@ async def test_72_set_fact_with_type(): event_log, ruleset_queues, dict(my_bool=True, my_int=2, my_float=3.123), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -1991,7 +1994,7 @@ async def test_73_mix_and_match_list(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -2021,7 +2024,7 @@ async def test_74_self_referential(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -2045,7 +2048,7 @@ async def test_75_all_conditions(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -2069,7 +2072,7 @@ async def test_76_all_conditions(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = { @@ -2101,7 +2104,7 @@ async def test_46_job_template(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) while not event_log.empty(): @@ -2136,7 +2139,7 @@ async def test_46_job_template_exception(err_msg, err): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) while not event_log.empty(): @@ -2176,7 +2179,7 @@ async def test_77_default_events_ttl(): event_log, ruleset_queues, dict(), - load_inventory("playbooks/inventory.yml"), + "playbooks/inventory.yml", ) checks = {