diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 258dde48..02fc5ef8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.13.0 +current_version = 1.0.0 commit = True tag = True search = {current_version} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0acdc068..72872146 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 7880bf48..813970a8 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -16,7 +16,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 610f9ed9..0c71e62b 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -42,7 +42,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f05a474f..6748b7c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,4 @@ repos: hooks: - id: pretty-format-json language_version: python3 + args: ['--autofix', '--no-sort-keys', '--indent', '4'] diff --git a/CHANGELOG.md b/CHANGELOG.md index 8186e981..6aacf363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ ## [Unreleased] +### Added + +### Fixed + +### Removed +- Remove official support for Python 3.8 + +## [1.0.0] - 2023-06-13 + ### Added - Sending heartbeat to the server with the session stats - Added command line option --execution-strategy @@ -12,6 +21,11 @@ - In a collection look for playbook in playbooks directory - Support .yaml and .yml extension for playbooks - Retract fact for partial and complete matches +- Checking of controller url and token at startup +- rule_uuid and ruleset_uuid provided even when an action fails +- Drools intermittently misses firing of rules +- Resend events lost during websocket disconnect +- limits the number of simultaneously open connection to controller to 30 ### Removed diff --git a/ansible_rulebook/__init__.py b/ansible_rulebook/__init__.py index 8baff7a7..63a08dfa 100644 --- a/ansible_rulebook/__init__.py +++ b/ansible_rulebook/__init__.py @@ -14,4 +14,4 @@ """Top-level package for Ansible Events.""" -__version__ = "__version__ = '0.13.0'" +__version__ = "__version__ = '1.0.0'" diff --git a/ansible_rulebook/app.py b/ansible_rulebook/app.py index b98a31c0..735d35e4 100644 --- a/ansible_rulebook/app.py +++ b/ansible_rulebook/app.py @@ -87,7 +87,9 @@ async def run(parsed_args: argparse.ArgumentParser) -> None: startup_args.controller_ssl_verify = parsed_args.controller_ssl_verify validate_actions(startup_args) - set_controller_params(startup_args) + + if startup_args.check_controller_connection: + await validate_controller_params(startup_args) if parsed_args.websocket_address: event_log = asyncio.Queue() @@ -139,6 +141,7 @@ async def run(parsed_args: argparse.ArgumentParser) -> None: logger.info("Main complete") await event_log.put(dict(type="Exit")) + await job_template_runner.close_session() if error_found: raise Exception("One of the source plugins failed") @@ -225,6 +228,8 @@ def validate_actions(startup_args: StartupArgs) -> None: for ruleset in startup_args.rulesets: for rule in ruleset.rules: for action in rule.actions: + if action.action == "run_job_template": + startup_args.check_controller_connection = True if ( action.action in INVENTORY_ACTIONS and not startup_args.inventory @@ -245,9 +250,12 @@ def validate_actions(startup_args: StartupArgs) -> None: ) -def set_controller_params(startup_args: StartupArgs) -> None: +async def validate_controller_params(startup_args: StartupArgs) -> None: if startup_args.controller_url: job_template_runner.host = startup_args.controller_url job_template_runner.token = startup_args.controller_token if startup_args.controller_ssl_verify: job_template_runner.verify_ssl = startup_args.controller_ssl_verify + + data = await job_template_runner.get_config() + logger.info("AAP Version %s", data["version"]) diff --git a/ansible_rulebook/builtin.py b/ansible_rulebook/builtin.py index b8a10b6f..e25fc706 100644 --- a/ansible_rulebook/builtin.py +++ b/ansible_rulebook/builtin.py @@ -825,7 +825,7 @@ async def run_job_template( rule_run_at=rule_run_at, ) if "error" in controller_job: - a_log["reason"] = dict(error=controller_job["error"]) + a_log["message"] = controller_job["error"] await event_log.put(a_log) if set_facts or post_events: diff --git a/ansible_rulebook/common.py b/ansible_rulebook/common.py index bc880e5d..60b64330 100644 --- a/ansible_rulebook/common.py +++ b/ansible_rulebook/common.py @@ -27,3 +27,4 @@ class StartupArgs: controller_ssl_verify: str = field(default="") project_data_file: str = field(default="") inventory: str = field(default="") + check_controller_connection: bool = field(default=False) diff --git a/ansible_rulebook/job_template_runner.py b/ansible_rulebook/job_template_runner.py index cbd1a3f4..a0ecdca0 100644 --- a/ansible_rulebook/job_template_runner.py +++ b/ansible_rulebook/job_template_runner.py @@ -34,7 +34,7 @@ class JobTemplateRunner: JOB_TEMPLATE_SLUG = "/api/v2/job_templates" - VALID_POST_CODES = [200, 201, 202] + CONFIG_SLUG = "/api/v2/config" JOB_COMPLETION_STATUSES = ["successful", "failed", "error", "canceled"] def __init__( @@ -46,27 +46,36 @@ def __init__( self.refresh_delay = int( os.environ.get("EDA_JOB_TEMPLATE_REFRESH_DELAY", 10) ) - - async def _get_page( - self, session: aiohttp.ClientSession, href_slug: str, params: dict - ) -> dict: - url = urljoin(self.host, href_slug) - async with session.get( - url, params=params, ssl=self._sslcontext - ) as response: - response_text = dict( - status=response.status, body=await response.text() + self._session = None + + async def close_session(self): + if self._session and not self._session.closed: + await self._session.close() + + def _create_session(self): + if self._session is None: + limit = int(os.getenv("EDA_CONTROLLER_CONNECTION_LIMIT", "30")) + self._session = aiohttp.ClientSession( + connector=aiohttp.TCPConnector(limit=limit), + headers=self._auth_headers(), + raise_for_status=True, ) - if response_text["status"] != 200: - raise ControllerApiException( - "Failed to get from %s. Status: %s, Body: %s" - % ( - url, - response_text["status"], - response_text.get("body", "empty"), - ) - ) - return response_text + + async def _get_page(self, href_slug: str, params: dict) -> dict: + try: + url = urljoin(self.host, href_slug) + self._create_session() + async with self._session.get( + url, params=params, ssl=self._sslcontext + ) as response: + return json.loads(await response.text()) + except aiohttp.ClientError as e: + logger.error("Error connecting to controller %s", str(e)) + raise ControllerApiException(str(e)) + + async def get_config(self) -> dict: + logger.info("Attempting to connect to Controller %s", self.host) + return await self._get_page(self.CONFIG_SLUG, {}) def _auth_headers(self) -> dict: return dict(Authorization=f"Bearer {self.token}") @@ -84,26 +93,20 @@ async def _get_job_template_id(self, name: str, organization: str) -> int: slug = f"{self.JOB_TEMPLATE_SLUG}/" params = {"name": name} - async with aiohttp.ClientSession( - headers=self._auth_headers() - ) as session: - while True: - response = await self._get_page(session, slug, params) - json_body = json.loads(response["body"]) - for jt in json_body["results"]: - if ( - jt["name"] == name - and dpath.get( - jt, "summary_fields.organization.name", "." - ) - == organization - ): - return jt["id"] - - if json_body.get("next", None): - params["page"] = params.get("page", 1) + 1 - else: - break + while True: + json_body = await self._get_page(slug, params) + for jt in json_body["results"]: + if ( + jt["name"] == name + and dpath.get(jt, "summary_fields.organization.name", ".") + == organization + ): + return jt["id"] + + if json_body.get("next", None): + params["page"] = params.get("page", 1) + 1 + else: + break raise JobTemplateNotFoundException( ( @@ -123,18 +126,14 @@ async def run_job_template( url = job["url"] params = {} - async with aiohttp.ClientSession( - headers=self._auth_headers() - ) as session: - while True: - # fetch and process job status - response = await self._get_page(session, url, params) - json_body = json.loads(response["body"]) - job_status = json_body["status"] - if job_status in self.JOB_COMPLETION_STATUSES: - return json_body + while True: + # fetch and process job status + json_body = await self._get_page(url, params) + job_status = json_body["status"] + if job_status in self.JOB_COMPLETION_STATUSES: + return json_body - await asyncio.sleep(self.refresh_delay) + await asyncio.sleep(self.refresh_delay) async def launch( self, name: str, organization: str, job_params: dict @@ -142,28 +141,14 @@ async def launch( jt_id = await self._get_job_template_id(name, organization) url = urljoin(self.host, f"{self.JOB_TEMPLATE_SLUG}/{jt_id}/launch/") - async with aiohttp.ClientSession( - headers=self._auth_headers() - ) as session: - async with session.post( + try: + async with self._session.post( url, json=job_params, ssl=self._sslcontext ) as post_response: - response = dict( - status=post_response.status, - body=await post_response.text(), - ) - - if response["status"] not in self.VALID_POST_CODES: - raise ControllerApiException( - "Failed to post to %s. Status: %s, Body: %s" - % ( - url, - response["status"], - response.get("body", "empty"), - ) - ) - json_body = json.loads(response["body"]) - return json_body + return json.loads(await post_response.text()) + except aiohttp.ClientError as e: + logger.error("Error connecting to controller %s", str(e)) + raise ControllerApiException(str(e)) job_template_runner = JobTemplateRunner() diff --git a/ansible_rulebook/rule_set_runner.py b/ansible_rulebook/rule_set_runner.py index d53f457b..ecf7a28a 100644 --- a/ansible_rulebook/rule_set_runner.py +++ b/ansible_rulebook/rule_set_runner.py @@ -15,7 +15,7 @@ import asyncio import gc import logging -from datetime import datetime +import uuid from pprint import PrettyPrinter, pformat from types import MappingProxyType from typing import Dict, List, Optional, Union, cast @@ -446,11 +446,17 @@ async def _call_action( dict( type="Action", action=action, + action_uuid=str(uuid.uuid4()), activation_id=settings.identifier, playbook_name=action_args.get("name"), status="failed", - run_at=str(datetime.utcnow()), - reason=dict(error=str(error)), + run_at=run_at(), + rule_run_at=rule_run_at, + message=str(error), + rule=rule, + ruleset=ruleset, + rule_uuid=rule_uuid, + ruleset_uuid=ruleset_uuid, ) ) diff --git a/ansible_rulebook/schema/ruleset_schema.json b/ansible_rulebook/schema/ruleset_schema.json index 6c441cd3..9be5d409 100644 --- a/ansible_rulebook/schema/ruleset_schema.json +++ b/ansible_rulebook/schema/ruleset_schema.json @@ -30,7 +30,10 @@ }, "execution_strategy": { "type": "string", - "enum": ["sequential", "parallel"], + "enum": [ + "parallel", + "sequential" + ], "default": "sequential" }, "sources": { diff --git a/ansible_rulebook/websocket.py b/ansible_rulebook/websocket.py index 68f06905..ee830fc7 100644 --- a/ansible_rulebook/websocket.py +++ b/ansible_rulebook/websocket.py @@ -89,21 +89,32 @@ async def send_event_log_to_websocket( event_log, websocket_address, websocket_ssl_verify ): logger.info("websocket %s connecting", websocket_address) + event = None async for websocket in websockets.connect( websocket_address, logger=logger, ssl=_sslcontext(websocket_address, websocket_ssl_verify), ): logger.info("websocket %s connected", websocket_address) - event = None try: + if event: + logger.info("Resending last event...") + await websocket.send(json.dumps(event)) + event = None + while True: event = await event_log.get() await websocket.send(json.dumps(event)) + if event == dict(type="Shutdown"): return - except websockets.ConnectionClosed: - logger.warning("websocket %s connection closed", websocket_address) + + event = None + except websockets.exceptions.ConnectionClosed: + logger.warning( + "websocket %s connection closed, will retry...", + websocket_address, + ) except CancelledError: logger.info("closing websocket due to task cancelled") return diff --git a/docs/collections.rst b/docs/collections.rst index 8807d9f2..edef3fc3 100644 --- a/docs/collections.rst +++ b/docs/collections.rst @@ -1,5 +1,52 @@ -=========== -Collections -=========== +.. _rulebook-collections: -Work in progress +======================== +Rulebook and Collections +======================== + +It's entirely possible to build and track simple Rulebooks and Playbooks in source repos. If you find yourself building more complex +and repeatable rulebooks that depends on other content, capabilities, or modules then you may want to consider packaging them into +a `Collection `_ + +Collections are an existing Ansible packaging concept that have been extended to support Ansible Rulebook Content. Rulebook Collections +also work particularly well with :ref:`Decision Environments ` + +The structure of a Collection with Rulebook content +--------------------------------------------------- + +Collections already have an `existing structure `_ +supporting Ansible Roles, Modules, Plugins, and Documentation. Lets look at what we can add to that structure to support ansible-rulebook content:: + + collection/ + ├ ... + ├── extensions/ + │ ├── eda/ + │ │ ├── rulebooks/ + │ │ └── plugins/ + │ │ ├── event_source/ + │ │ └── event_filter/ + └ ... + +There's more to a collection but these are the things added to a collection that ansible-rulebook itself is looking for. You can and will put +roles, playbooks, and other content in the collection as well. Especially if you will be calling them and making use of them from your rulebooks. + +You'll initialize the Collection the same way you would any other collection:: + + ansible-galaxy collection init my_collection + +Then you can add the directories above and start populating it with content. + +Not every Collection you write will have its own plugins but if you find yourself building your own :ref:`event sources ` +or :ref:`event filters ` then you'll want to put them in the collection as shown above. + +Using a rulebook included in a collection +----------------------------------------- + +The ansible-rulebook command can take a path to a rulebook file directly but once you've put a rulebook into a collection and it's available in +the environment then you can refer to it by its fully qualified name:: + + ansible-rulebook -r my_namespace.my_collection.my_rulebook + +.. note:: + For more details on how to build, and publish collections see + the `Developing Ansible Collections `_ documentation. diff --git a/docs/conf.py b/docs/conf.py index 0c4187be..8de8b679 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,6 +33,7 @@ # import os import sys +import datetime sys.path.insert(0, os.path.abspath("..")) @@ -66,8 +67,8 @@ # General information about the project. project = "ansible-rulebook" -copyright = "2022, Ben Thomasson" -author = "Ben Thomasson" +copyright = f"2022-{datetime.datetime.today().year}, Red Hat, Inc" +author = "Red Hat, Inc" # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout @@ -154,7 +155,7 @@ master_doc, "ansible_rulebook.tex", "ansible-rulebook Documentation", - "Ben Thomasson", + "Red Hat Ansible", "manual", ), ] diff --git a/docs/decision_environment.rst b/docs/decision_environment.rst index 9d96f23b..7f4e5eba 100644 --- a/docs/decision_environment.rst +++ b/docs/decision_environment.rst @@ -1,7 +1,14 @@ +.. _decision-environment: + ==================== Decision Environment ==================== +.. note:: + + Some of the examples in this section refer to Rulebooks store in collections. If you are interested in packaging your event driven automation + in collections, please see the :ref:`Rulebook and Collections ` section. + Decision Environments are `Execution Environments `_ tailored towards running Ansible Rulebook tasks. These represent container images that launch and run the rulebook process and contain all of the dependencies, collections, and configuration needed to run a rulebook. @@ -62,7 +69,7 @@ The minimal decision environment is a good starting point, but you will likely w This shows an example where you may have your own Collection that contains rulebooks and playbooks but need to bring them together with some other collections and some python and system dependencies. -You could also use Builder to add your own rulebooks and playbooks to the decision environment via `additional-build-steps`_ +You could also use Builder to add your own rulebooks and playbooks to the decision environment via `additional-build-steps `_ and then making use of Containerfile commands to ADD or COPY to get the files into the environment. .. code-block:: yaml diff --git a/docs/environments.rst b/docs/environments.rst deleted file mode 100644 index 9f8f05a5..00000000 --- a/docs/environments.rst +++ /dev/null @@ -1,5 +0,0 @@ -============ -Environments -============ - -Work in progress diff --git a/docs/filters.rst b/docs/filters.rst index fbf8ed76..34d4c9c6 100644 --- a/docs/filters.rst +++ b/docs/filters.rst @@ -1,3 +1,5 @@ +.. _event-filter: + ============= Event Filters ============= diff --git a/docs/index.rst b/docs/index.rst index 4eca768b..073cf36c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,6 @@ Welcome to Ansible Rulebook documentation filters runner collections - environments decision_environment Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst index 73ed0051..250e7d30 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -8,7 +8,7 @@ Please ensure you have installed all components listed in the **Requirements** s Requirements ------------ -* Python >= 3.8 +* Python >= 3.9 * Python 3 pip * Java development kit >= 17 diff --git a/docs/rules.rst b/docs/rules.rst index eb6b8da0..f95c7fc7 100644 --- a/docs/rules.rst +++ b/docs/rules.rst @@ -47,7 +47,7 @@ Example: A single action name: remediate_outage.yml - name: Print event with linux - condition: event.target_os == "linux" or + condition: event.target_os == "linux" action: debug: @@ -81,7 +81,7 @@ Example: Disable a rule name: remediate_outage.yml - name: Print event with linux - condition: event.target_os == "linux" or + condition: event.target_os == "linux" action: debug: diff --git a/docs/sources.rst b/docs/sources.rst index dab3ef3a..6d163b8d 100644 --- a/docs/sources.rst +++ b/docs/sources.rst @@ -1,3 +1,5 @@ +.. _event-source-plugins: + ==================== Event Source Plugins ==================== @@ -44,14 +46,56 @@ These include: Mainly used for development and testing + How to Develop a Custom Plugin ------------------------------ You can build your own event source plugin in python. A plugin is a single -python file. You can start with this example: +python file but before we get to that lets take a look at some best practices and patterns: + +Best Practices and Patterns +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are 3 basic patterns that you'll be developing against when considering a new source plugin: + +#. Event Bus Plugins + These are plugins that listen to a stream of events from a source where the connection + is established by the plugin itself. Examples of this are the ``kafka`` and ``mqtt`` plugins. + + This is the most ideal and reliable pattern to follow. Durability and Reliability of the data + is the responsibility of the event source and availability of the data can follow the patterns + of the event source and its own internal configuration. + +#. Scraper Plugins + These plugins connect to a source and scrape the data from the source usually after a given amount of time + has passed. Examples of this are the ``url_check`` and ``watchdog`` plugins. + + These plugins can be reliable but may require extract logic for handling duplication. It's also possible + to miss data if the scraper is not running at the time the data is available. + +#. Callback Plugins + These plugins provide a callback endpoint that the event source can call when data is available. + Examples of this are the ``webhook`` and ``alertmanager`` plugins. + + These plugins are the least reliable as they are dependent on the event source to call the callback + endpoint and are highly sensitive to data loss. If the event source is not available or the callback + endpoint is not available then there may not be another opportunity to receive the data. + + These can also require other ingress policies and firewall rules to be available and configured properly + to operate. + +It's strongly recommended to adopt one of the first two patterns and only consider callback plugins in the absence +of any other solution. + +When deciding whether to build a dedicated plugin you may consider configuring the data source to send data to a +system where a more general plugin exists already. For example, if you have a system that can send data to a kafka +topic then you can use the ``kafka`` plugin to receive the data. There are many connectors for tying systems to other +message buses and this is a great way to leverage existing plugins. Plugin template ^^^^^^^^^^^^^^^ +Lets take a look at a very basic example that you could use in the form of a template for producing other plugins: + .. code-block:: python """ diff --git a/pyproject.toml b/pyproject.toml index eefc5910..9329969a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 79 -target-version = ["py38", "py39", "py310"] +target-version = ["py39", "py310"] extend-exclude = "docs" [tool.isort] diff --git a/requirements_test.txt b/requirements_test.txt index 24d878ba..f6773b5d 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -19,3 +19,4 @@ freezegun oauthlib>=3.2.0 kubernetes urllib3<2 +aioresponses diff --git a/setup.cfg b/setup.cfg index eff3fabc..9b8e99b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,19 +1,18 @@ [metadata] name = ansible_rulebook -version = 0.13.0 +version = 1.0.0 description = Event driven automation for Ansible url = https://github.com/ansible/ansible-rulebook license = Apache-2.0 keywords = ansible_rulebook long_description = file: README.rst, HISTORY.rst long_description_content_type = text/x-rst; charset=UTF-8 -classifiers = +classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers License :: OSI Approved :: Apache Software License Natural Language :: English Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -22,8 +21,8 @@ classifiers = zip_safe = False include_package_data = True packages = find: -python_requires = >=3.8 -install_requires = +python_requires = >=3.9 +install_requires = aiohttp pyparsing >= 3.0 jsonschema @@ -32,15 +31,15 @@ install_requires = janus ansible-runner websockets - drools_jpy == 0.3.3 + drools_jpy == 0.3.4 [options.packages.find] -include = +include = ansible_rulebook ansible_rulebook.* [options.entry_points] -console_scripts = +console_scripts = ansible-rulebook = ansible_rulebook.cli:main [bumpversion:file:setup.cfg] @@ -53,5 +52,5 @@ replace = __version__ = '{new_version}' [flake8] extend-exclude = docs, venv, .venv -extend-ignore = +extend-ignore = E203, # Whitespace before ':' (false positive in slices, handled by black. diff --git a/tests/test_app.py b/tests/test_app.py index 6b33e2a0..9c3625ec 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,6 +1,7 @@ import os from contextlib import nullcontext as does_not_raise from unittest import mock +from unittest.mock import patch import pytest @@ -156,13 +157,16 @@ async def test_run_with_websocket(create_ruleset): controller_url="abc", controller_token="token", controller_ssl_verify="no", + check_controller_connection=True, ) - - await run(cmdline_args) - - assert mock_start_source.call_count == 1 - assert mock_run_rulesets.call_count == 1 - assert mock_request_workload.call_count == 1 + with patch( + "ansible_rulebook.app.job_template_runner.get_config", + return_value=dict(version="4.4.1"), + ): + await run(cmdline_args) + assert mock_start_source.call_count == 1 + assert mock_run_rulesets.call_count == 1 + assert mock_request_workload.call_count == 1 @pytest.mark.asyncio diff --git a/tests/test_controller.py b/tests/test_controller.py index bbe1f148..4f88a0e6 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -12,11 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from pathlib import Path import pytest +from aiohttp import ClientError +from aioresponses import aioresponses -from ansible_rulebook.exception import JobTemplateNotFoundException +from ansible_rulebook.exception import ( + ControllerApiException, + JobTemplateNotFoundException, +) from ansible_rulebook.job_template_runner import job_template_runner @@ -50,3 +56,34 @@ async def test_job_template_not_exist(): job_template_runner.token = "DUMMY" with pytest.raises(JobTemplateNotFoundException): await job_template_runner.run_job_template("Hello World", "no-org", {}) + + +@pytest.mark.asyncio +async def test_job_template_get_config(): + text = json.dumps(dict(version="4.4.1")) + with aioresponses() as mocked: + job_template_runner.host = "https://example.com" + job_template_runner.token = "DUMMY" + mocked.get("https://example.com/api/v2/config", status=200, body=text) + data = await job_template_runner.get_config() + assert data["version"] == "4.4.1" + + +@pytest.mark.asyncio +async def test_job_template_get_config_error(): + with aioresponses() as mocked: + job_template_runner.host = "https://example.com" + job_template_runner.token = "DUMMY" + mocked.get("https://example.com/api/v2/config", exception=ClientError) + with pytest.raises(ControllerApiException): + await job_template_runner.get_config() + + +@pytest.mark.asyncio +async def test_job_template_get_config_auth_error(): + with aioresponses() as mocked: + job_template_runner.host = "https://example.com" + job_template_runner.token = "DUMMY" + mocked.get("https://example.com/api/v2/config", status=401) + with pytest.raises(ControllerApiException): + await job_template_runner.get_config() diff --git a/tests/test_examples.py b/tests/test_examples.py index c25117d3..79100895 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -2116,6 +2116,7 @@ async def test_46_job_template(): JOB_TEMPLATE_ERRORS = [ ("api error", ControllerApiException("api error")), ("jt does not exist", JobTemplateNotFoundException("jt does not exist")), + ("Kaboom", RuntimeError("Kaboom")), ] @@ -2144,7 +2145,22 @@ async def test_46_job_template_exception(err_msg, err): action = event assert action["action"] == "run_job_template" - assert action["reason"] == {"error": err_msg} + assert action["message"] == err_msg + required_keys = { + "action", + "action_uuid", + "activation_id", + "message", + "rule_run_at", + "run_at", + "rule", + "ruleset", + "rule_uuid", + "ruleset_uuid", + "status", + "type", + } + assert set(action.keys()).issuperset(required_keys) @pytest.mark.asyncio diff --git a/tests/test_websocket.py b/tests/test_websocket.py index daa8810d..215cf5c6 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -4,9 +4,11 @@ import json import os from typing import Dict, List +from unittest import mock from unittest.mock import AsyncMock, patch import pytest +import websockets from ansible_rulebook.websocket import ( request_workload, @@ -120,3 +122,32 @@ def my_func(data): mo.return_value.send.side_effect = my_func await send_event_log_to_websocket(queue, "dummy", "yes") assert data_sent == ['{"a": 1}', '{"b": 1}', '{"type": "Shutdown"}'] + + +@pytest.mark.asyncio +@mock.patch("ansible_rulebook.websocket.websockets.connect") +async def test_send_event_log_to_websocket_with_exception( + socket_mock: AsyncMock, +): + queue = asyncio.Queue() + queue.put_nowait({"a": 1}) + queue.put_nowait({"b": 2}) + queue.put_nowait(dict(type="Shutdown")) + + data_sent = [] + + mock_object = AsyncMock() + socket_mock.return_value = mock_object + socket_mock.return_value.__aenter__.return_value = mock_object + socket_mock.return_value.__anext__.return_value = mock_object + socket_mock.return_value.__aiter__.side_effect = [mock_object] + + socket_mock.return_value.send.side_effect = [ + websockets.exceptions.ConnectionClosed(rcvd=None, sent=None), + data_sent.append({"a": 1}), + data_sent.append({"b": 2}), + data_sent.append({"type": "Shutdown"}), + ] + + await send_event_log_to_websocket(queue, "dummy", "yes") + assert data_sent == [{"a": 1}, {"b": 2}, {"type": "Shutdown"}] diff --git a/tools/convert_to_ast.py b/tools/convert_to_ast.py new file mode 100644 index 00000000..e0f8a2ea --- /dev/null +++ b/tools/convert_to_ast.py @@ -0,0 +1,83 @@ +import argparse +import json +import os +import sys +from pathlib import Path +from typing import List + +import yaml + +from ansible_rulebook.json_generator import visit_ruleset +from ansible_rulebook.rules_parser import parse_rule_sets + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + + parser.add_argument( + "-r", + "--rulebook", + help="The rulebook file or rulebook from a collection", + required=True, + ) + parser.add_argument( + "-e", + "--vars", + help="Variables file", + ) + parser.add_argument( + "-o", + "--output", + help="output ast file", + ) + parser.add_argument( + "-E", + "--env-vars", + help=( + "Comma separated list of variables to import from the environment" + ), + ) + return parser + + +def load_rules(rules_file, variables): + with open(rules_file) as f: + data = yaml.safe_load(f.read()) + + return parse_rule_sets(data, variables) + + +def main(args: List[str] = None) -> int: + parser = get_parser() + cmdline_args = parser.parse_args(args) + + variables = {} + if cmdline_args.vars: + with open(cmdline_args.vars) as f: + variables = yaml.safe_load(f.read()) + + if cmdline_args.env_vars: + for var in cmdline_args.env_vars.split(","): + variables[var] = os.environ.get(var) + + file_name = cmdline_args.rulebook + if cmdline_args.output: + ast_file_name = cmdline_args.output + ast_file_json = f"{ast_file_name}.json" + else: + ast_file_name = f"{Path(file_name).stem}_ast.yml" + ast_file_json = f"{Path(file_name).stem}_ast.json" + + ruleset_asts = [] + for ruleset in load_rules(file_name, variables): + ruleset_asts.append(visit_ruleset(ruleset, variables)) + + with open(ast_file_name, "w") as f: + yaml.dump(ruleset_asts, f) + + with open(ast_file_json, "w") as outfile: + json.dump(ruleset_asts, outfile, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tox.ini b/tox.ini index aed26804..b6e0791b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = flake8,py38,py39,py310,py311 +envlist = flake8,py39,py310,py311 isolated_build = True [travis] python = - 3.8: py38 3.9: py39 3.10: py310 3.11: py311