From 031271c2a03f3b48f42a0528005acb91d62abb6e Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 18 Jan 2021 15:41:13 -0600 Subject: [PATCH 01/90] Setting up for #107 --- tests/test_commands_yaml_set.py | 59 +++++++++++++++++++++++++++++++++ yamlpath/processor.py | 43 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index bb28be7f..4ee1deb3 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1218,3 +1218,62 @@ def test_assign_null(self, script_runner, tmp_path_factory): with open(yaml_file, 'r') as fhnd: filedat = fhnd.read() assert filedat == yamlout + + def test_assign_to_nonexistent_and_empty_nodes(self, script_runner, tmp_path_factory): + # Inspiration: https://github.com/wwkimball/yamlpath/issues/107 + # Test: cat testbed.yaml | yaml-set --change='/devices/*/[os!=~/.+/]/os' --value=generic + yamlin = """--- +devices: + R1: + os: ios + type: router + platform: asr1k + + R2: + type: switch + platform: cat3k + + R3: + type: access-point + platform: wrt + os: + + R4: + type: tablet + os: null + platform: java +""" + yamlout = """--- +devices: + R1: + os: ios + type: router + platform: asr1k + + R2: + type: switch + platform: cat3k + + os: generic + R3: + type: access-point + platform: wrt + os: generic + + R4: + type: tablet + os: null + platform: java +""" + yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) + result = script_runner.run( + self.command, + "--change=/devices/*/[os!=~/.+/]/os", + "--value=generic", + yaml_file + ) + assert result.success, result.stderr + + with open(yaml_file, 'r') as fhnd: + filedat = fhnd.read() + assert filedat == yamlout diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 42d41edb..43e95bed 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -538,6 +538,7 @@ def _get_nodes_by_search( term = terms.term matches = False desc_path = YAMLPath(attr) + debug_matched = "NO MATCHES YIELDED" if isinstance(data, list): if not traverse_lists: self.logger.debug( @@ -562,6 +563,7 @@ def _get_nodes_by_search( break if (matches and not invert) or (invert and not matches): + debug_matched = "one list match yielded" self.logger.debug( "Yielding list match at index {}:".format(lstidx), data=ele, @@ -576,6 +578,7 @@ def _get_nodes_by_search( for key, val in data.items(): matches = Searches.search_matches(method, term, key) if (matches and not invert) or (invert and not matches): + debug_matched = "one dictionary key name match yielded" self.logger.debug( "Yielding dictionary key name match against '{}':" .format(key), @@ -590,6 +593,7 @@ def _get_nodes_by_search( value = data[attr] matches = Searches.search_matches(method, term, value) if (matches and not invert) or (invert and not matches): + debug_matched = "one dictionary attribute match yielded" self.logger.debug( "Yielding dictionary attribute match against '{}':" .format(attr), @@ -611,17 +615,30 @@ def _get_nodes_by_search( break if (matches and not invert) or (invert and not matches): + debug_matched = "one descendant search match yielded" + self.logger.debug( + "Yielding descendant match against '{}':" + .format(attr), + data=data, + prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(data, parent, parentref, translated_path) else: # Check the passed data itself for a match matches = Searches.search_matches(method, term, data) if (matches and not invert) or (invert and not matches): + debug_matched = "query source data itself yielded" self.logger.debug( "Yielding the queried data itself because it matches.", prefix="Processor::_get_nodes_by_search: ") yield NodeCoords(data, parent, parentref, translated_path) + self.logger.debug( + "Finished seeking SEARCH nodes matching {} in data with {}:" + .format(terms, debug_matched), + data=data, + prefix="Processor::_get_nodes_by_search: ") + # pylint: disable=locally-disabled def _get_nodes_by_collector( self, data: Any, yaml_path: YAMLPath, segment_index: int, @@ -1066,6 +1083,8 @@ def _get_optional_nodes( matched_nodes < 1 and segment_type is not PathSegmentTypes.SEARCH ): + at_terminus = len(yaml_path) <= depth + 1 + # Add the missing element self.logger.debug( ("Processor::_get_optional_nodes: Element <{}>{} is" @@ -1169,7 +1188,31 @@ def _get_optional_nodes( str(yaml_path), except_segment ) + + # elif at_terminus and isinstance(parent, (dict, list)): + # self.logger.debug( + # "Setting a {} terminal value at path {} ({} segments)" + # " at depth {}, to value {}, when:" + # .format( + # str(segment_type), str(yaml_path), + # str(len(yaml_path)), str(depth + 1), + # str(value)), + # prefix="Processor::_get_optional_nodes: ", + # data={"data": data, "parent": parent, + # "parentref": parentref}) + # parent[parentref] = value + # data = value + # yield NodeCoords(data, parent, parentref, translated_path) + else: + self.logger.debug( + "Assuming data is scalar and cannot receive a {}" + " subreference at {} ({}/{}):".format( + str(segment_type), str(yaml_path), str(depth + 1), + str(len(yaml_path))), + prefix="Processor::_get_optional_nodes: ", + data={"data": data, "parent": parent, + "parentref": parentref, "(default_)value": value}) raise YAMLPathException( "Cannot add {} subreference to scalars".format( str(segment_type) From fbd49a7889a76be264fe3bd6cfcd9dff8917576f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 12 Apr 2021 17:19:57 -0500 Subject: [PATCH 02/90] Traverse children of matched search parents --- tests/test_commands_yaml_set.py | 2 +- yamlpath/processor.py | 47 ++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index b19cea76..c4b28af6 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1277,7 +1277,7 @@ def test_assign_to_nonexistent_and_empty_nodes(self, script_runner, tmp_path_fac R3: type: access-point platform: wrt - os: generic + os: R4: type: tablet os: null diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 2351fa11..a9ace43c 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1393,6 +1393,13 @@ def _get_optional_nodes( ] = segments[depth][1] except_segment = str(unstripped_attrs) + prior_was_search = False + if depth > 0: + prior_was_search = segments[depth - 1][0] in [ + PathSegmentTypes.SEARCH, + PathSegmentTypes.TRAVERSE + ] + self.logger.debug( "Seeking element <{}>{} in data of type {}:" .format(segment_type, except_segment, type(data)), @@ -1428,8 +1435,16 @@ def _get_optional_nodes( # Add the missing element self.logger.debug( ("Processor::_get_optional_nodes: Element <{}>{} is" - + " unknown in the data! Applying default, <{}>{}." - ).format(segment_type, except_segment, type(value), value) + " unknown in the data! Applying default, <{}>{} to" + " data:" + ).format(segment_type, except_segment, type(value), value), + data=data + ) + self.logger.debug( + "Processor::_get_optional_nodes: Considering application" + " of unknown element to parent while at_terminus={}:" + .format(at_terminus), + data=parent ) if isinstance(data, list): self.logger.debug( @@ -1529,20 +1544,20 @@ def _get_optional_nodes( except_segment ) - # elif at_terminus and isinstance(parent, (dict, list)): - # self.logger.debug( - # "Setting a {} terminal value at path {} ({} segments)" - # " at depth {}, to value {}, when:" - # .format( - # str(segment_type), str(yaml_path), - # str(len(yaml_path)), str(depth + 1), - # str(value)), - # prefix="Processor::_get_optional_nodes: ", - # data={"data": data, "parent": parent, - # "parentref": parentref}) - # parent[parentref] = value - # data = value - # yield NodeCoords(data, parent, parentref, translated_path) + elif prior_was_search and isinstance(parent, (dict, list)): + self.logger.debug( + "Setting a {} terminal value at path {} ({} segments)" + " at depth {}, to value {}, when:" + .format( + str(segment_type), str(yaml_path), + str(len(yaml_path)), str(depth + 1), + str(value)), + prefix="Processor::_get_optional_nodes: ", + data={"data": data, "parent": parent, + "parentref": parentref}) + parent[parentref] = value + data = value + yield NodeCoords(data, parent, parentref, translated_path) else: self.logger.debug( From 9aec758254f0f4a91f22adc923122dca1a175bf8 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Apr 2021 13:24:57 -0500 Subject: [PATCH 03/90] Minor debug code removal --- yamlpath/processor.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index a9ace43c..1c68bcc2 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1430,8 +1430,6 @@ def _get_optional_nodes( matched_nodes < 1 and segment_type is not PathSegmentTypes.SEARCH ): - at_terminus = len(yaml_path) <= depth + 1 - # Add the missing element self.logger.debug( ("Processor::_get_optional_nodes: Element <{}>{} is" @@ -1440,12 +1438,6 @@ def _get_optional_nodes( ).format(segment_type, except_segment, type(value), value), data=data ) - self.logger.debug( - "Processor::_get_optional_nodes: Considering application" - " of unknown element to parent while at_terminus={}:" - .format(at_terminus), - data=parent - ) if isinstance(data, list): self.logger.debug( "Processor::_get_optional_nodes: Dealing with a list" @@ -1546,8 +1538,8 @@ def _get_optional_nodes( elif prior_was_search and isinstance(parent, (dict, list)): self.logger.debug( - "Setting a {} terminal value at path {} ({} segments)" - " at depth {}, to value {}, when:" + ("Setting a post-search {} value at path {} ({}" + " segments) at depth {}, to value {}, when:") .format( str(segment_type), str(yaml_path), str(len(yaml_path)), str(depth + 1), From 16a27d20779a4dca2a78e52e9568209869a3470f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 13 Apr 2021 17:55:20 -0500 Subject: [PATCH 04/90] WIP: Introducing keyword searches --- yamlpath/enums/__init__.py | 1 + yamlpath/enums/pathsearchkeywords.py | 38 +++++++++++++ yamlpath/enums/pathsearchmethods.py | 4 +- yamlpath/enums/pathsegmenttypes.py | 4 ++ yamlpath/path/__init__.py | 1 + yamlpath/path/searchkeywordterms.py | 71 ++++++++++++++++++++++++ yamlpath/processor.py | 81 +++++++++++++++++++++++++++- yamlpath/yamlpath.py | 48 ++++++++++++++--- 8 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 yamlpath/enums/pathsearchkeywords.py create mode 100644 yamlpath/path/searchkeywordterms.py diff --git a/yamlpath/enums/__init__.py b/yamlpath/enums/__init__.py index 9420583d..2050c804 100644 --- a/yamlpath/enums/__init__.py +++ b/yamlpath/enums/__init__.py @@ -2,6 +2,7 @@ from .anchormatches import AnchorMatches from .collectoroperators import CollectorOperators from .includealiases import IncludeAliases +from .pathsearchkeywords import PathSearchKeywords from .pathsearchmethods import PathSearchMethods from .pathsegmenttypes import PathSegmentTypes from .pathseperators import PathSeperators diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py new file mode 100644 index 00000000..3da8e2e7 --- /dev/null +++ b/yamlpath/enums/pathsearchkeywords.py @@ -0,0 +1,38 @@ +""" +Implements the PathSearchKeywords enumeration. + +Copyright 2021 William W. Kimball, Jr. MBA MSIS +""" +from enum import Enum, auto +from typing import List + + +class PathSearchKeywords(Enum): + """ + Supported keyword methods for searching YAML Path segments. + + These include: + + `HAS_CHILD` + Matches when the node has a direct child with a given name. + """ + + HAS_CHILD = auto() + + def __str__(self) -> str: + """Get a String representation of an employed value of this enum.""" + keyword = '' + if self is PathSearchKeywords.HAS_CHILD: + keyword = 'has_child' + + return keyword + + @staticmethod + def get_keywords() -> List[str]: + """Return the full list of supported search keywords.""" + return [str(o).lower() for o in PathSearchKeywords] + + @staticmethod + def is_keyword(keyword: str) -> bool: + """Indicate whether keyword is known.""" + return keyword in PathSearchKeywords.get_keywords() diff --git a/yamlpath/enums/pathsearchmethods.py b/yamlpath/enums/pathsearchmethods.py index edf181fb..9e6246ad 100644 --- a/yamlpath/enums/pathsearchmethods.py +++ b/yamlpath/enums/pathsearchmethods.py @@ -9,7 +9,7 @@ class PathSearchMethods(Enum): """ - Supported selfs for searching YAML Path segments. + Supported methods for searching YAML Path segments. These include: @@ -77,7 +77,7 @@ def __str__(self) -> str: @staticmethod def get_operators() -> List[str]: - """Return the full list of suppoerted symbolic search operators.""" + """Return the full list of supported symbolic search operators.""" return [str(o) for o in PathSearchMethods] @staticmethod diff --git a/yamlpath/enums/pathsegmenttypes.py b/yamlpath/enums/pathsegmenttypes.py index ee21f8c0..489d9e00 100644 --- a/yamlpath/enums/pathsegmenttypes.py +++ b/yamlpath/enums/pathsegmenttypes.py @@ -23,6 +23,9 @@ class PathSegmentTypes(Enum): `INDEX` A list element index. + `KEYWORD_SEARCH` + A search based on PathSearchKeywords. + `KEY` A dictionary key name. @@ -41,3 +44,4 @@ class PathSegmentTypes(Enum): KEY = auto() SEARCH = auto() TRAVERSE = auto() + KEYWORD_SEARCH = auto() diff --git a/yamlpath/path/__init__.py b/yamlpath/path/__init__.py index d68851ed..7d06a926 100644 --- a/yamlpath/path/__init__.py +++ b/yamlpath/path/__init__.py @@ -1,3 +1,4 @@ """Make all of the YAML Path components available.""" from .collectorterms import CollectorTerms +from .searchkeywordterms import SearchKeywordTerms from .searchterms import SearchTerms diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py new file mode 100644 index 00000000..94e8056c --- /dev/null +++ b/yamlpath/path/searchkeywordterms.py @@ -0,0 +1,71 @@ +""" +YAML path Keyword Search segment terms. + +Copyright 2019, 2020 William W. Kimball, Jr. MBA MSIS +""" +from typing import List + +from yamlpath.enums import PathSearchKeywords + + +class SearchKeywordTerms: + """YAML path Keyword Search segment terms.""" + + def __init__(self, inverted: bool, keyword: PathSearchKeywords, + parameters: List[str]) -> None: + """ + Instantiate a Keyword Search Term segment. + + Parameters: + 1. inverted (bool) true = invert the search operation; false, otherwise + 2. keyword (PathSearchKeywords) the search keyword + 3. parameters (str) the parameters to the keyword-named operation + """ + self._inverted: bool = inverted + self._keyword: PathSearchKeywords = keyword + self._parameters: str = parameters + + def __str__(self) -> str: + """Get a String representation of this Keyword Search Term.""" + # Replace unescaped spaces with escaped spaces + safe_parameters = ", ".join( + r"\ ".join( + list(map( + lambda ele: ele.replace(" ", r"\ ") + , self.parameters.split(r"\ ") + )) + ).replace(",", "\\,")) + + return ( + "[" + + ("!" if self.inverted else "") + + str(self.keyword) + + "(" + + safe_parameters + + ")]" + ) + + @property + def inverted(self) -> bool: + """ + Access the inversion flag for this Keyword Search. + + This indicates whether the search logic is to be inverted. + """ + return self._inverted + + @property + def keyword(self) -> PathSearchKeywords: + """ + Access the search keyword. + + This indicates what kind of search logic is to be performed. + """ + return self._keyword + + @property + def parameters(self) -> str: + """ + Accessor for the parameters being fed to the search operation. + """ + return self._parameters diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 1c68bcc2..6275870a 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -8,11 +8,12 @@ from yamlpath.common import Anchors, Nodes, Searches from yamlpath import YAMLPath -from yamlpath.path import SearchTerms, CollectorTerms +from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms from yamlpath.wrappers import ConsolePrinter, NodeCoords from yamlpath.exceptions import YAMLPathException from yamlpath.enums import ( YAMLValueFormats, + PathSearchKeywords, PathSegmentTypes, CollectorOperators, PathSeperators, @@ -604,6 +605,13 @@ def _get_nodes_by_path_segment(self, data: Any, node_coords = self._get_nodes_by_anchor( data, yaml_path, segment_index, translated_path=translated_path) + elif ( + segment_type == PathSegmentTypes.KEYWORD_SEARCH + and isinstance(stripped_attrs, SearchKeywordTerms) + ): + node_coords = self._get_nodes_by_keyword_search( + data, stripped_attrs, parent=parent, parentref=parentref, + traverse_lists=traverse_lists, translated_path=translated_path) elif ( segment_type == PathSegmentTypes.SEARCH and isinstance(stripped_attrs, SearchTerms) @@ -843,6 +851,77 @@ def _get_nodes_by_anchor( and stripped_attrs == val.anchor.value): yield NodeCoords(val, data, key, next_translated_path) + def _get_nodes_by_keyword_search( + self, data: Any, terms: SearchKeywordTerms, **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """ + Perform a search identified by a keyword and its parameters. + + Parameters: + 1. data (Any) The parsed YAML data to process + 2. terms (SearchKeywordTerms) The search terms + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * traverse_lists (Boolean) Indicate whether searches against lists are + permitted to automatically traverse into the list; Default=True + + Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they + are matched + + Raises: N/A + """ + self.logger.debug( + "Seeking KEYWORD_SEARCH nodes matching {} in data:".format(terms), + data=data, + prefix="Processor::_get_nodes_by_keyword_search: ") + + parent = kwargs.pop("parent", None) + parentref = kwargs.pop("parentref", None) + traverse_lists = kwargs.pop("traverse_lists", True) + translated_path = kwargs.pop("translated_path", YAMLPath("")) + invert = terms.inverted + keyword = terms.keyword + parameters = terms.parameters + matches = False + + if keyword is PathSearchKeywords.HAS_CHILD: + # Against a map, this will return nodes which have an immediate + # child key exactly named as per parameters. When inverted, only + # parents with no such key are yielded. + if isinstance(data, dict): + child_present = parameters in data + if ( + (invert and not child_present) or + (child_present and not invert) + ): + self.logger.debug( + "Yielding dictionary with child keyword-matched" + " against '{}':".format(parameters), + data=data, + prefix="Processor::_get_nodes_by_search: ") + yield NodeCoords( + data, parent, parentref, + translated_path) + + # Against a list, this will merely require an exact match between + # parameters and any list elements. When inverted, every + # non-matching element is yielded. + elif isinstance(data, list): + if not traverse_lists: + self.logger.debug( + "Processor::_get_nodes_by_keyword_search: Refusing to" + " traverse a list.") + return + + # Against an AoH, this will scan each element's immediate children, + # treating and yielding as if this search were performed directly + # against each map in the list. + else: + raise NotImplementedError + # pylint: disable=too-many-statements def _get_nodes_by_search( self, data: Any, terms: SearchTerms, **kwargs: Any diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index a1c586eb..5405dee3 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -10,11 +10,12 @@ from yamlpath.exceptions import YAMLPathException from yamlpath.enums import ( PathSegmentTypes, + PathSearchKeywords, PathSearchMethods, PathSeperators, CollectorOperators, ) -from yamlpath.path import SearchTerms, CollectorTerms +from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms class YAMLPath: @@ -268,6 +269,7 @@ def _parse_path(self, search_inverted: bool = False search_method: Optional[PathSearchMethods] = None search_attr: str = "" + search_keyword: Optional[PathSearchKeywords] = None seeking_regex_delim: bool = False capturing_regex: bool = False pathsep: str = str(self.seperator) @@ -384,6 +386,25 @@ def _parse_path(self, continue elif char == "(": + if demarc_count > 0 and demarc_stack[-1] == "[" and segment_id: + if PathSearchKeywords.is_keyword(segment_id): + demarc_stack.append(char) + demarc_count += 1 + segment_type = PathSegmentTypes.KEYWORD_SEARCH + search_keyword = PathSearchKeywords[segment_id.upper()] + segment_id = "" + continue + + raise YAMLPathException( + ("Unknown search keyword, {}; allowed: {}." + " Encountered in YAML Path") + .format( + segment_id, + ','.join(PathSearchKeywords.get_keywords()) + ) + , yaml_path + ) + seeking_collector_operator = False collector_level += 1 demarc_stack.append(char) @@ -394,12 +415,12 @@ def _parse_path(self, if collector_level == 1: continue - elif collector_level > 0: - if ( - demarc_count > 0 - and char == ")" - and demarc_stack[-1] == "(" - ): + if ( + demarc_count > 0 + and char == ")" + and demarc_stack[-1] == "(" + ): + if collector_level > 0: collector_level -= 1 demarc_count -= 1 demarc_stack.pop() @@ -413,6 +434,19 @@ def _parse_path(self, seeking_collector_operator = True continue + if segment_type is PathSegmentTypes.KEYWORD_SEARCH: + demarc_count -= 1 + demarc_stack.pop() + path_segments.append(( + segment_type, + SearchKeywordTerms(search_inverted, search_keyword, + segment_id) + )) + search_inverted = False + search_keyword = None + segment_id = "" + continue + elif demarc_count == 0 and char == "[": # Array INDEX/SLICE or SEARCH if segment_id: From 61cc3fcc618725d10ce24ad97fb5f4b3f3d698a2 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 00:50:22 -0500 Subject: [PATCH 05/90] WIP: Fleshing out the first search keyword --- yamlpath/path/searchkeywordterms.py | 39 +++++++++++++++++++++++------ yamlpath/processor.py | 34 ++++++++++++++++++------- yamlpath/yamlpath.py | 3 +++ 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index 94e8056c..de97543c 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -24,17 +24,13 @@ def __init__(self, inverted: bool, keyword: PathSearchKeywords, self._inverted: bool = inverted self._keyword: PathSearchKeywords = keyword self._parameters: str = parameters + self._lparameters: List[str] = [] + self._parameters_parsed: bool = False def __str__(self) -> str: """Get a String representation of this Keyword Search Term.""" # Replace unescaped spaces with escaped spaces - safe_parameters = ", ".join( - r"\ ".join( - list(map( - lambda ele: ele.replace(" ", r"\ ") - , self.parameters.split(r"\ ") - )) - ).replace(",", "\\,")) + safe_parameters = ", ".join(self.parameters) return ( "[" @@ -68,4 +64,31 @@ def parameters(self) -> str: """ Accessor for the parameters being fed to the search operation. """ - return self._parameters + if self._parameters_parsed: + return self._lparameters + + param = "" + params = [] + escape_next = False + for char in self._parameters: + if escape_next: + escape_next = False + + elif char == "\\": + escape_next = True + continue + + elif char == ",": + params.append(param) + param = "" + continue + + param = param + char + + # Add the last parameter, if there is one + if param: + params.append(param) + + self._lparameters = params + self._parameters_parsed = True + return self._lparameters diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 6275870a..e0a50462 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -610,8 +610,9 @@ def _get_nodes_by_path_segment(self, data: Any, and isinstance(stripped_attrs, SearchKeywordTerms) ): node_coords = self._get_nodes_by_keyword_search( - data, stripped_attrs, parent=parent, parentref=parentref, - traverse_lists=traverse_lists, translated_path=translated_path) + data, yaml_path, stripped_attrs, parent=parent, + parentref=parentref, traverse_lists=traverse_lists, + translated_path=translated_path) elif ( segment_type == PathSegmentTypes.SEARCH and isinstance(stripped_attrs, SearchTerms) @@ -852,14 +853,16 @@ def _get_nodes_by_anchor( yield NodeCoords(val, data, key, next_translated_path) def _get_nodes_by_keyword_search( - self, data: Any, terms: SearchKeywordTerms, **kwargs: Any + self, data: Any, yaml_path: YAMLPath, terms: SearchKeywordTerms, + **kwargs: Any ) -> Generator[NodeCoords, None, None]: """ Perform a search identified by a keyword and its parameters. Parameters: 1. data (Any) The parsed YAML data to process - 2. terms (SearchKeywordTerms) The search terms + 2. yaml_path (Path) The YAML Path being processed + 3. terms (SearchKeywordTerms) The keyword search terms Keyword Arguments: * parent (ruamel.yaml node) The parent node from which this query @@ -885,21 +888,29 @@ def _get_nodes_by_keyword_search( invert = terms.inverted keyword = terms.keyword parameters = terms.parameters - matches = False if keyword is PathSearchKeywords.HAS_CHILD: + # There must be exactly one parameter + param_count = len(parameters) + if param_count != 1: + raise YAMLPathException( + ("Invalid parameter count to {}; {} required, got {} in" + " YAML Path").format(keyword, 1, param_count), + yaml_path) + match_key = parameters[0] + # Against a map, this will return nodes which have an immediate # child key exactly named as per parameters. When inverted, only # parents with no such key are yielded. if isinstance(data, dict): - child_present = parameters in data + child_present = match_key in data if ( (invert and not child_present) or (child_present and not invert) ): self.logger.debug( "Yielding dictionary with child keyword-matched" - " against '{}':".format(parameters), + " against '{}':".format(match_key), data=data, prefix="Processor::_get_nodes_by_search: ") yield NodeCoords( @@ -1376,8 +1387,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, translated_path=translated_path ): self.logger.debug( - "Found node of type {} at <{}>{} in the data and recursing" - " into it..." + "Got data of type {} at <{}>{} in the data." .format( type(segment_node_coords.node if hasattr(segment_node_coords, "node") @@ -1400,11 +1410,17 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, # such, it must be treated as a virtual DOM element that # cannot itself be parented to the real DOM, though each # of its elements has a real parent. + self.logger.debug( + "Processor::_get_required_nodes: Got a list:", + data=segment_node_coords) for subnode_coord in self._get_required_nodes( segment_node_coords, yaml_path, depth + 1, translated_path=translated_path): yield subnode_coord else: + self.logger.debug( + "Recursing into the retrieved data...", + prefix="Processor::_get_required_nodes: ") for subnode_coord in self._get_required_nodes( segment_node_coords.node, yaml_path, depth + 1, parent=segment_node_coords.parent, diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 5405dee3..5730fee6 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -445,6 +445,7 @@ def _parse_path(self, search_inverted = False search_keyword = None segment_id = "" + segment_type = None continue elif demarc_count == 0 and char == "[": @@ -789,6 +790,8 @@ def _stringify_yamlpath_segments( ppath += "[&{}]".format(segment_attrs) else: ppath += "&{}".format(segment_attrs) + elif segment_type == PathSegmentTypes.KEYWORD_SEARCH: + ppath += str(segment_attrs) elif segment_type == PathSegmentTypes.SEARCH: ppath += str(segment_attrs) elif segment_type == PathSegmentTypes.COLLECTOR: From 6e31d267c8c3db0b7d9d60ab2cf2d28d0b32720c Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 14:49:32 -0500 Subject: [PATCH 06/90] WIP: Parsing keyword searches may need to return the parent of the match --- yamlpath/processor.py | 7 +++++ yamlpath/wrappers/consoleprinter.py | 3 +- yamlpath/yamlpath.py | 46 +++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index e0a50462..a43bd90f 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -77,6 +77,13 @@ def get_nodes(self, yaml_path: Union[YAMLPath, str], elif pathsep is not PathSeperators.AUTO: yaml_path.seperator = pathsep + self.logger.debug( + "Processing YAML Path:", + prefix="Processor::get_nodes: ", data={ + 'path': yaml_path, + 'segments': yaml_path.escaped + }) + if mustexist: matched_nodes: int = 0 for node_coords in self._get_required_nodes(self.data, yaml_path): diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 55e9151b..4656cd9a 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -14,6 +14,7 @@ Copyright 2018, 2019, 2020 William W. Kimball, Jr. MBA MSIS """ import sys +from collections import deque from typing import Any, Dict, Generator, List, Set, Tuple, Union from ruamel.yaml.comments import ( @@ -231,7 +232,7 @@ def _debug_dump(data: Any, **kwargs) -> Generator[str, None, None]: data, prefix=prefix, **kwargs ): yield line - elif isinstance(data, (list, set, tuple)): + elif isinstance(data, (list, set, tuple, deque)): for line in ConsolePrinter._debug_list( data, prefix=prefix, **kwargs ): diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 5730fee6..faa3e9e0 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -276,6 +276,7 @@ def _parse_path(self, collector_level: int = 0 collector_operator: CollectorOperators = CollectorOperators.NONE seeking_collector_operator: bool = False + next_char_must_be = None # Empty paths yield empty queues if not yaml_path: @@ -291,6 +292,8 @@ def _parse_path(self, # pylint: disable=locally-disabled,too-many-nested-blocks for char in yaml_path: demarc_count = len(demarc_stack) + if next_char_must_be and char == next_char_must_be: + next_char_must_be = None if escape_next: # Pass-through; capture this escaped character @@ -351,6 +354,11 @@ def _parse_path(self, collector_operator = CollectorOperators.SUBTRACTION continue + elif next_char_must_be and char != next_char_must_be: + raise YAMLPathException( + "Invalid YAML Path at {}, which must be {} in YAML Path" + .format(char, next_char_must_be), yaml_path) + elif char in ['"', "'"]: # Found a string demarcation mark if demarc_count > 0: @@ -437,15 +445,7 @@ def _parse_path(self, if segment_type is PathSegmentTypes.KEYWORD_SEARCH: demarc_count -= 1 demarc_stack.pop() - path_segments.append(( - segment_type, - SearchKeywordTerms(search_inverted, search_keyword, - segment_id) - )) - search_inverted = False - search_keyword = None - segment_id = "" - segment_type = None + next_char_must_be = "]" continue elif demarc_count == 0 and char == "[": @@ -592,7 +592,7 @@ def _parse_path(self, and char == "]" and demarc_stack[-1] == "[" ): - # Store the INDEX, SLICE, or SEARCH parameters + # Store the INDEX, SLICE, SEARCH, or KEYWORD_SEARCH parameters if ( segment_type is PathSegmentTypes.INDEX and ':' not in segment_id @@ -621,6 +621,15 @@ def _parse_path(self, SearchTerms(search_inverted, search_method, search_attr, segment_id) )) + elif ( + segment_type is PathSegmentTypes.KEYWORD_SEARCH + and search_keyword + ): + path_segments.append(( + segment_type, + SearchKeywordTerms(search_inverted, search_keyword, + segment_id) + )) else: path_segments.append((segment_type, segment_id)) @@ -629,6 +638,8 @@ def _parse_path(self, demarc_stack.pop() demarc_count -= 1 search_method = None + search_inverted = False + search_keyword = None continue elif demarc_count < 1 and char == pathsep: @@ -792,8 +803,19 @@ def _stringify_yamlpath_segments( ppath += "&{}".format(segment_attrs) elif segment_type == PathSegmentTypes.KEYWORD_SEARCH: ppath += str(segment_attrs) - elif segment_type == PathSegmentTypes.SEARCH: - ppath += str(segment_attrs) + elif (segment_type == PathSegmentTypes.SEARCH + and isinstance(segment_attrs, SearchTerms)): + terms: SearchTerms = segment_attrs + if (terms.method == PathSearchMethods.REGEX + and terms.attribute == "." + and terms.term == ".*" + and not terms.inverted + ): + if add_sep: + ppath += pathsep + ppath += "*" + else: + ppath += str(segment_attrs) elif segment_type == PathSegmentTypes.COLLECTOR: ppath += str(segment_attrs) elif segment_type == PathSegmentTypes.TRAVERSE: From 385f8301c5788c6f57c9f2f5286843395ac75a77 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:01:50 -0500 Subject: [PATCH 07/90] yaml-set must change gathered nodes lest kw searches alter the DOM and then fail to match nodes on the 2nd scan --- yamlpath/commands/yaml_set.py | 11 ++++------- yamlpath/path/searchkeywordterms.py | 8 +++----- yamlpath/processor.py | 21 +++++++++++++-------- yamlpath/wrappers/consoleprinter.py | 4 ++-- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index b9c80e04..923f2119 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -28,7 +28,7 @@ from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.eyaml.enums import EYAMLOutputFormats from yamlpath.eyaml import EYAMLProcessor -from yamlpath.wrappers import ConsolePrinter +from yamlpath.wrappers import ConsolePrinter, NodeCoords def processcli(): """Process command-line arguments.""" @@ -584,12 +584,9 @@ def main(): except EYAMLCommandException as ex: log.critical(ex, 2) elif has_new_value: - try: - processor.set_value( - change_path, new_value, value_format=args.format, - mustexist=must_exist, tag=args.tag) - except YAMLPathException as ex: - log.critical(ex, 1) + change_node: NodeCoords = None + for change_node in change_node_coordinates: + change_node.parent[change_node.parentref] = new_value elif args.tag: processor.tag_gathered_nodes(change_node_coordinates, args.tag) diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index de97543c..792aec7d 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -12,7 +12,7 @@ class SearchKeywordTerms: """YAML path Keyword Search segment terms.""" def __init__(self, inverted: bool, keyword: PathSearchKeywords, - parameters: List[str]) -> None: + parameters: str) -> None: """ Instantiate a Keyword Search Term segment. @@ -60,10 +60,8 @@ def keyword(self) -> PathSearchKeywords: return self._keyword @property - def parameters(self) -> str: - """ - Accessor for the parameters being fed to the search operation. - """ + def parameters(self) -> List[str]: + """Accessor for the parameters being fed to the search operation.""" if self._parameters_parsed: return self._lparameters diff --git a/yamlpath/processor.py b/yamlpath/processor.py index a43bd90f..16b3cf0d 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -746,7 +746,7 @@ def _get_nodes_by_index( Parameters: 1. data (Any) The parsed YAML data to process - 2. yaml_path (Path) The YAML Path being processed + 2. yaml_path (YAMLPath) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they @@ -827,7 +827,7 @@ def _get_nodes_by_anchor( Parameters: 1. data (Any) The parsed YAML data to process - 2. yaml_path (Path) The YAML Path being processed + 2. yaml_path (YAMLPath) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they @@ -868,7 +868,7 @@ def _get_nodes_by_keyword_search( Parameters: 1. data (Any) The parsed YAML data to process - 2. yaml_path (Path) The YAML Path being processed + 2. yaml_path (YAMLPath) The YAML Path being processed 3. terms (SearchKeywordTerms) The keyword search terms Keyword Arguments: @@ -903,7 +903,7 @@ def _get_nodes_by_keyword_search( raise YAMLPathException( ("Invalid parameter count to {}; {} required, got {} in" " YAML Path").format(keyword, 1, param_count), - yaml_path) + str(yaml_path)) match_key = parameters[0] # Against a map, this will return nodes which have an immediate @@ -919,7 +919,7 @@ def _get_nodes_by_keyword_search( "Yielding dictionary with child keyword-matched" " against '{}':".format(match_key), data=data, - prefix="Processor::_get_nodes_by_search: ") + prefix="Processor::_get_nodes_by_keyword_search: ") yield NodeCoords( data, parent, parentref, translated_path) @@ -933,10 +933,13 @@ def _get_nodes_by_keyword_search( "Processor::_get_nodes_by_keyword_search: Refusing to" " traverse a list.") return + raise NotImplementedError # Against an AoH, this will scan each element's immediate children, # treating and yielding as if this search were performed directly # against each map in the list. + else: + raise NotImplementedError else: raise NotImplementedError @@ -1096,7 +1099,7 @@ def _get_nodes_by_collector( Parameters: 1. data (ruamel.yaml data) The parsed YAML data to process - 2. yaml_path (Path) The YAML Path being processed + 2. yaml_path (YAMLPath) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process 4. terms (CollectorTerms) The collector terms @@ -1356,7 +1359,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, Parameters: 1. data (Any) The parsed YAML data to process - 2. yaml_path (Path) The pre-parsed YAML Path to follow + 2. yaml_path (YAMLPath) The pre-parsed YAML Path to follow 3. depth (int) Index within yaml_path to process; default=0 4. parent (ruamel.yaml node) The parent node from which this query originates @@ -1462,7 +1465,7 @@ def _get_optional_nodes( Parameters: 1. data (Any) The parsed YAML data to process - 2. yaml_path (Path) The pre-parsed YAML Path to follow + 2. yaml_path (YAMLPath) The pre-parsed YAML Path to follow 3. value (Any) The value to assign to the element 4. depth (int) For recursion, this identifies which segment of yaml_path to evaluate; default=0 @@ -1498,6 +1501,7 @@ def _get_optional_nodes( prior_was_search = False if depth > 0: prior_was_search = segments[depth - 1][0] in [ + PathSegmentTypes.KEYWORD_SEARCH, PathSegmentTypes.SEARCH, PathSegmentTypes.TRAVERSE ] @@ -1531,6 +1535,7 @@ def _get_optional_nodes( if ( matched_nodes < 1 and segment_type is not PathSegmentTypes.SEARCH + and segment_type is not PathSegmentTypes.KEYWORD_SEARCH ): # Add the missing element self.logger.debug( diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 4656cd9a..01239e07 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -15,7 +15,7 @@ """ import sys from collections import deque -from typing import Any, Dict, Generator, List, Set, Tuple, Union +from typing import Any, Deque, Dict, Generator, List, Set, Tuple, Union from ruamel.yaml.comments import ( CommentedBase, @@ -310,7 +310,7 @@ def _debug_node_coord( @staticmethod def _debug_list( - data: Union[List[Any], Tuple[Any, ...], Set[Any]], **kwargs + data: Union[List[Any], Set[Any], Tuple[Any, ...], Deque[Any]], **kwargs ) -> Generator[str, None, None]: """Helper for debug.""" prefix = kwargs.pop("prefix", "") From 85b5d23b3e9ecf1a9f049bb0db4edf83295621a3 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:10:44 -0500 Subject: [PATCH 08/90] WIP: direct modification to node_coords causes too many tests to fail --- yamlpath/commands/yaml_set.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index 923f2119..b9c80e04 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -28,7 +28,7 @@ from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.eyaml.enums import EYAMLOutputFormats from yamlpath.eyaml import EYAMLProcessor -from yamlpath.wrappers import ConsolePrinter, NodeCoords +from yamlpath.wrappers import ConsolePrinter def processcli(): """Process command-line arguments.""" @@ -584,9 +584,12 @@ def main(): except EYAMLCommandException as ex: log.critical(ex, 2) elif has_new_value: - change_node: NodeCoords = None - for change_node in change_node_coordinates: - change_node.parent[change_node.parentref] = new_value + try: + processor.set_value( + change_path, new_value, value_format=args.format, + mustexist=must_exist, tag=args.tag) + except YAMLPathException as ex: + log.critical(ex, 1) elif args.tag: processor.tag_gathered_nodes(change_node_coordinates, args.tag) From 713b1f7f6d45b20a0d038ff8d697c5469508c14a Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:25:08 -0500 Subject: [PATCH 09/90] Try a soft-fail approach to yaml-set's first pass --- yamlpath/commands/yaml_set.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index b9c80e04..7bf7e1d7 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -391,6 +391,7 @@ def _get_nodes(log, processor, yaml_path, **kwargs): """Gather requested nodes.""" must_exist = kwargs.pop("must_exist", False) default_value = kwargs.pop("default_value", " ") + ignore_fail = kwargs.pop("ignore_fail", False) gathered_nodes = [] try: @@ -402,6 +403,11 @@ def _get_nodes(log, processor, yaml_path, **kwargs): data=node_coordinate, prefix="yaml_set::_get_nodes: ") gathered_nodes.append(node_coordinate) except YAMLPathException as ex: + if ignore_fail: + log.debug( + "Ignoring failure to gather nodes due to: {}".format(ex), + prefix="yaml_set::_get_nodes: ") + return [] log.critical(ex, 1) log.debug( @@ -478,7 +484,7 @@ def main(): log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) change_node_coordinates = _get_nodes( - log, processor, change_path, must_exist=must_exist, + log, processor, change_path, must_exist=True, ignore_fail=True, default_value=("" if new_value else " ")) old_format = YAMLValueFormats.DEFAULT From 143056b3c9eea66c12192d30d92def9711daa2a5 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 17:50:19 -0500 Subject: [PATCH 10/90] yaml-set --mustexist error message changed --- tests/test_commands_yaml_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index c4b28af6..2cbd5aeb 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -178,12 +178,12 @@ def test_bad_yaml_path(self, script_runner, tmp_path_factory): # Explicit --mustexist result = script_runner.run(self.command, "--change=key2", "--random=1", "--mustexist", yaml_file) assert not result.success, result.stderr - assert "Required YAML Path does not match any nodes" in result.stderr + assert "No nodes matched required YAML Path" in result.stderr # Implicit --mustexist via --saveto result = script_runner.run(self.command, "--change=key3", "--random=1", "--saveto=save_here", yaml_file) assert not result.success, result.stderr - assert "Required YAML Path does not match any nodes" in result.stderr + assert "No nodes matched required YAML Path" in result.stderr def test_checked_replace(self, script_runner, tmp_path_factory): content = """--- From 45e1a85e2071ae1262f90b2fb4c6ba9d6beaf0fb Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:04:26 -0500 Subject: [PATCH 11/90] Reverse last change; update has_child test --- tests/test_commands_yaml_set.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index 2cbd5aeb..aa157e02 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -178,12 +178,12 @@ def test_bad_yaml_path(self, script_runner, tmp_path_factory): # Explicit --mustexist result = script_runner.run(self.command, "--change=key2", "--random=1", "--mustexist", yaml_file) assert not result.success, result.stderr - assert "No nodes matched required YAML Path" in result.stderr + assert "Required YAML Path does not match any nodes" in result.stderr # Implicit --mustexist via --saveto result = script_runner.run(self.command, "--change=key3", "--random=1", "--saveto=save_here", yaml_file) assert not result.success, result.stderr - assert "No nodes matched required YAML Path" in result.stderr + assert "Required YAML Path does not match any nodes" in result.stderr def test_checked_replace(self, script_runner, tmp_path_factory): content = """--- @@ -1243,9 +1243,8 @@ def test_change_null(self, script_runner, tmp_path_factory): filedat = fhnd.read() assert filedat == yamlout - def test_assign_to_nonexistent_and_empty_nodes(self, script_runner, tmp_path_factory): + def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): # Inspiration: https://github.com/wwkimball/yamlpath/issues/107 - # Test: cat testbed.yaml | yaml-set --change='/devices/*/[os!=~/.+/]/os' --value=generic yamlin = """--- devices: R1: @@ -1286,7 +1285,7 @@ def test_assign_to_nonexistent_and_empty_nodes(self, script_runner, tmp_path_fac yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) result = script_runner.run( self.command, - "--change=/devices/*/[os!=~/.+/]/os", + "--change=/devices/*/[!has_child(os)]/os", "--value=generic", yaml_file ) From 270fc392881860431cdd51bc87d8dd8a05ce52ea Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:04:56 -0500 Subject: [PATCH 12/90] Invert yaml-set node cathering error-sensitivity --- yamlpath/commands/yaml_set.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index 7bf7e1d7..1e4a307d 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -483,8 +483,9 @@ def main(): processor = EYAMLProcessor( log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) + ignore_fail = not must_exist change_node_coordinates = _get_nodes( - log, processor, change_path, must_exist=True, ignore_fail=True, + log, processor, change_path, must_exist=True, ignore_fail=ignore_fail, default_value=("" if new_value else " ")) old_format = YAMLValueFormats.DEFAULT From 366cff677d2737f97b3caaf7dddac8d450010a06 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:53:29 -0500 Subject: [PATCH 13/90] str(YAMLPath('*')) now == '*' --- tests/test_yamlpath.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_yamlpath.py b/tests/test_yamlpath.py index c12e1266..124d145a 100644 --- a/tests/test_yamlpath.py +++ b/tests/test_yamlpath.py @@ -4,7 +4,7 @@ from yamlpath.enums import PathSegmentTypes, PathSeperators from yamlpath import YAMLPath -class Test_path_Path(): +class Test_YAMLPath(): """Tests for the Path class.""" @pytest.mark.parametrize("yamlpath,pathsep,output", [ @@ -32,8 +32,8 @@ class Test_path_Path(): ("a*f", PathSeperators.AUTO, "[.=~/^a.*f$/]"), ("a*f*z", PathSeperators.AUTO, "[.=~/^a.*f.*z$/]"), ("a*f*z*", PathSeperators.AUTO, "[.=~/^a.*f.*z.*$/]"), - ("*", PathSeperators.AUTO, "[.=~/.*/]"), - ("*.*", PathSeperators.AUTO, "[.=~/.*/][.=~/.*/]"), + ("*", PathSeperators.AUTO, "*"), + ("*.*", PathSeperators.AUTO, "*.*"), ("**", PathSeperators.AUTO, "**"), ("/**/def", PathSeperators.AUTO, "/**/def"), ("abc.**.def", PathSeperators.AUTO, "abc.**.def"), From 18f83ba48a984b14008637364be9a476bc402c72 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 21:53:39 -0500 Subject: [PATCH 14/90] Minor typing and pass-through cleanup --- yamlpath/yamlpath.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index faa3e9e0..b8724e99 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -276,7 +276,7 @@ def _parse_path(self, collector_level: int = 0 collector_operator: CollectorOperators = CollectorOperators.NONE seeking_collector_operator: bool = False - next_char_must_be = None + next_char_must_be: Optional[str] = None # Empty paths yield empty queues if not yaml_path: @@ -300,20 +300,17 @@ def _parse_path(self, escape_next = False elif capturing_regex: - if char == demarc_stack[-1]: - # Stop the RegEx capture - capturing_regex = False - demarc_stack.pop() - continue - # Pass-through; capture everything that isn't the present # RegEx delimiter. This deliberately means users cannot # escape the RegEx delimiter itself should it occur within # the RegEx; thus, users must select a delimiter that won't # appear within the RegEx (which is exactly why the user # gets to choose the delimiter). - # pylint: disable=unnecessary-pass - pass # pragma: no cover + if char == demarc_stack[-1]: + # Stop the RegEx capture + capturing_regex = False + demarc_stack.pop() + continue # The escape test MUST come AFTER the RegEx capture test so users # won't be forced into "The Backslash Plague". From f3f0b6eb15e2162d334521536a141a0b6a174f88 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 22:00:15 -0500 Subject: [PATCH 15/90] Corrected bad if/elif breakpoint --- yamlpath/yamlpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index b8724e99..ee57f9b9 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -420,7 +420,7 @@ def _parse_path(self, if collector_level == 1: continue - if ( + elif ( demarc_count > 0 and char == ")" and demarc_stack[-1] == "(" From 32ea3384ad624ad82d7082ee4d6181bb63616849 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 14 Apr 2021 22:33:27 -0500 Subject: [PATCH 16/90] Slightly expanded has_child test --- tests/test_commands_yaml_set.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index aa157e02..3a485aad 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1262,6 +1262,10 @@ def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): type: tablet os: null platform: java + R5: + type: tablet + os: "" + platform: objective-c """ yamlout = """--- devices: @@ -1279,8 +1283,12 @@ def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): os: R4: type: tablet - os: null + os: platform: java + R5: + type: tablet + os: "" + platform: objective-c """ yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) result = script_runner.run( From a1e3bbc2e7316d9e89c3da5d9f7ce37c49786552 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 15 Apr 2021 00:04:17 -0500 Subject: [PATCH 17/90] Improved keyword search parameter parsing --- yamlpath/path/searchkeywordterms.py | 47 ++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index 792aec7d..bac0f588 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -60,15 +60,21 @@ def keyword(self) -> PathSearchKeywords: return self._keyword @property + # pylint: disable=locally-disabled,too-many-branches def parameters(self) -> List[str]: """Accessor for the parameters being fed to the search operation.""" if self._parameters_parsed: return self._lparameters - param = "" - params = [] - escape_next = False + param: str = "" + params: List[str] = [] + escape_next: bool = False + demarc_stack: List[str] = [] + + # pylint: disable=locally-disabled,too-many-nested-blocks for char in self._parameters: + demarc_count = len(demarc_stack) + if escape_next: escape_next = False @@ -76,13 +82,46 @@ def parameters(self) -> List[str]: escape_next = True continue - elif char == ",": + elif ( + char == " " + and (demarc_count < 1 + or demarc_stack[-1] not in ["'", '"']) + ): + # Ignore unescaped, non-demarcated whitespace + continue + + elif char in ['"', "'"]: + # Found a string demarcation mark + if demarc_count > 0: + # Already appending to an ongoing demarcated value + if char == demarc_stack[-1]: + # Close a matching pair + demarc_stack.pop() + demarc_count -= 1 + continue + + # Embed a nested, demarcated component + demarc_stack.append(char) + demarc_count += 1 + else: + # Fresh demarcated value + demarc_stack.append(char) + demarc_count += 1 + continue + + elif demarc_count < 1 and char == ",": params.append(param) param = "" continue param = param + char + # Check for mismatched demarcations + if demarc_count > 0: + raise ValueError( + "Keyword search parameters contain one or more unmatched" + " demarcation symbol(s): {}".format(" ".join(demarc_stack))) + # Add the last parameter, if there is one if param: params.append(param) From 151fb397a3586d993b7e3692cb26df2d082c26b2 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 15 Apr 2021 00:04:36 -0500 Subject: [PATCH 18/90] Started tests for SearchKeywordTerms --- tests/test_path_searchkeywordterms.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/test_path_searchkeywordterms.py diff --git a/tests/test_path_searchkeywordterms.py b/tests/test_path_searchkeywordterms.py new file mode 100644 index 00000000..ac6395f9 --- /dev/null +++ b/tests/test_path_searchkeywordterms.py @@ -0,0 +1,17 @@ +import pytest + +from yamlpath.enums import PathSearchKeywords +from yamlpath.path import SearchKeywordTerms + +class Test_path_SearchKeywordTerms(): + """Tests for the SearchKeywordTerms class.""" + + @pytest.mark.parametrize("invert,keyword,parameters,output", [ + (True, PathSearchKeywords.HAS_CHILD, "abc", "[!has_child(abc)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc", "[has_child(abc)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc\\,def", "[has_child(abc\\,def)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc, def", "[has_child(abc, def)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc,' def'", "[has_child(abc,\\ def)]"), + ]) + def test_str(self, invert, keyword, parameters, output): + assert output == str(SearchKeywordTerms(invert, keyword, parameters)) From d68b70967effc7e8b2cf1431dd438804e01009ab Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 15 Apr 2021 16:45:35 -0500 Subject: [PATCH 19/90] Fully test SearchKeywordTerms --- tests/test_path_searchkeywordterms.py | 45 +++++++++++++++++++++------ yamlpath/path/searchkeywordterms.py | 30 +++++++++++------- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/tests/test_path_searchkeywordterms.py b/tests/test_path_searchkeywordterms.py index ac6395f9..a4334eea 100644 --- a/tests/test_path_searchkeywordterms.py +++ b/tests/test_path_searchkeywordterms.py @@ -4,14 +4,39 @@ from yamlpath.path import SearchKeywordTerms class Test_path_SearchKeywordTerms(): - """Tests for the SearchKeywordTerms class.""" + """Tests for the SearchKeywordTerms class.""" - @pytest.mark.parametrize("invert,keyword,parameters,output", [ - (True, PathSearchKeywords.HAS_CHILD, "abc", "[!has_child(abc)]"), - (False, PathSearchKeywords.HAS_CHILD, "abc", "[has_child(abc)]"), - (False, PathSearchKeywords.HAS_CHILD, "abc\\,def", "[has_child(abc\\,def)]"), - (False, PathSearchKeywords.HAS_CHILD, "abc, def", "[has_child(abc, def)]"), - (False, PathSearchKeywords.HAS_CHILD, "abc,' def'", "[has_child(abc,\\ def)]"), - ]) - def test_str(self, invert, keyword, parameters, output): - assert output == str(SearchKeywordTerms(invert, keyword, parameters)) + @pytest.mark.parametrize("invert,keyword,parameters,output", [ + (True, PathSearchKeywords.HAS_CHILD, "abc", "[!has_child(abc)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc", "[has_child(abc)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc\\,def", "[has_child(abc\\,def)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc, def", "[has_child(abc, def)]"), + (False, PathSearchKeywords.HAS_CHILD, "abc,' def'", "[has_child(abc,' def')]"), + ]) + def test_str(self, invert, keyword, parameters, output): + assert output == str(SearchKeywordTerms(invert, keyword, parameters)) + + @pytest.mark.parametrize("parameters,output", [ + ("abc", ["abc"]), + ("abc\\,def", ["abc,def"]), + ("abc, def", ["abc", "def"]), + ("abc,' def'", ["abc", " def"]), + ("1,'1', 1, '1', 1 , ' 1', '1 ', ' 1 '", ["1", "1", "1", "1", "1", " 1", "1 ", " 1 "]), + ("true, False,'True','false'", ["true", "False", "True", "false"]), + ("'',,\"\", '', ,,\"\\'\",'\\\"'", ["", "", "", "", "", "", "'", "\""]), + ("'And then, she said, \"Quote!\"'", ["And then, she said, \"Quote!\""]), + (None, []), + ]) + def test_parameter_parsing(self, parameters, output): + skt = SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, parameters) + assert output == skt.parameters + + @pytest.mark.parametrize("parameters", [ + ("','a'"), + ("a,\"b,"), + ]) + def test_unmatched_demarcation(self, parameters): + skt = SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, parameters) + with pytest.raises(ValueError) as ex: + parmlist = skt.parameters + assert -1 < str(ex.value).find("one or more unmatched demarcation symbol") diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index bac0f588..8e7f780e 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -29,15 +29,12 @@ def __init__(self, inverted: bool, keyword: PathSearchKeywords, def __str__(self) -> str: """Get a String representation of this Keyword Search Term.""" - # Replace unescaped spaces with escaped spaces - safe_parameters = ", ".join(self.parameters) - return ( "[" - + ("!" if self.inverted else "") - + str(self.keyword) + + ("!" if self._inverted else "") + + str(self._keyword) + "(" - + safe_parameters + + self._parameters + ")]" ) @@ -66,6 +63,11 @@ def parameters(self) -> List[str]: if self._parameters_parsed: return self._lparameters + if self._parameters is None: + self._parameters_parsed = True + self._lparameters = [] + return self._lparameters + param: str = "" params: List[str] = [] escape_next: bool = False @@ -76,6 +78,7 @@ def parameters(self) -> List[str]: demarc_count = len(demarc_stack) if escape_next: + # Pass-through; capture this escaped character escape_next = False elif char == "\\": @@ -84,8 +87,7 @@ def parameters(self) -> List[str]: elif ( char == " " - and (demarc_count < 1 - or demarc_stack[-1] not in ["'", '"']) + and (demarc_count < 1) ): # Ignore unescaped, non-demarcated whitespace continue @@ -98,11 +100,15 @@ def parameters(self) -> List[str]: # Close a matching pair demarc_stack.pop() demarc_count -= 1 - continue - # Embed a nested, demarcated component - demarc_stack.append(char) - demarc_count += 1 + if demarc_count < 1: + # Final close; seek the next delimiter + continue + + else: + # Embed a nested, demarcated component + demarc_stack.append(char) + demarc_count += 1 else: # Fresh demarcated value demarc_stack.append(char) From 5a045475428172a1c6a7ff4f2f291325bb61e6e9 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 16 Apr 2021 17:10:03 -0500 Subject: [PATCH 20/90] WIP: Start seperating kw-search logic from proc --- yamlpath/common/keywordsearches.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 yamlpath/common/keywordsearches.py diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py new file mode 100644 index 00000000..8412585c --- /dev/null +++ b/yamlpath/common/keywordsearches.py @@ -0,0 +1,27 @@ +""" +Implement KeywordSearches. + +This is a static library of generally-useful code for searching data based on +pre-defined keywords (in the programming language sense). + +Copyright 2020 William W. Kimball, Jr. MBA MSIS +""" +from typing import Any, List + +from yamlpath.enums import PathSearchKeywords +from yamlpath.path import SearchKeywordTerms + + +class KeywordSearches: + """Helper methods for common data searching operations.""" + + @staticmethod + def search_matches( + terms: SearchKeywordTerms, + haystack: Any + ) -> bool: + """Performs a keyword search.""" + invert: bool = terms.inverted + keyword: PathSearchKeywords = terms.keyword + parameters: List[str] = terms.parameters + From 071890624959b09c6569d4705eec1878b707404c Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 17 Apr 2021 14:44:21 -0500 Subject: [PATCH 21/90] WIP: Move KWSearch logic to static class --- yamlpath/common/__init__.py | 1 + yamlpath/common/keywordsearches.py | 69 +++++++++++++++++++++++++++--- yamlpath/processor.py | 61 +++----------------------- 3 files changed, 70 insertions(+), 61 deletions(-) diff --git a/yamlpath/common/__init__.py b/yamlpath/common/__init__.py index 3270d771..765d000d 100644 --- a/yamlpath/common/__init__.py +++ b/yamlpath/common/__init__.py @@ -1,5 +1,6 @@ """Common library methods.""" from .anchors import Anchors +from .keywordsearches import KeywordSearches from .nodes import Nodes from .parsers import Parsers from .searches import Searches diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 8412585c..1e7b4b0e 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -6,22 +6,81 @@ Copyright 2020 William W. Kimball, Jr. MBA MSIS """ -from typing import Any, List +from typing import Any, Generator, List from yamlpath.enums import PathSearchKeywords from yamlpath.path import SearchKeywordTerms - +from yamlpath.exceptions import YAMLPathException +from yamlpath.wrappers import NodeCoords +from yamlpath import YAMLPath class KeywordSearches: """Helper methods for common data searching operations.""" @staticmethod def search_matches( - terms: SearchKeywordTerms, - haystack: Any - ) -> bool: + terms: SearchKeywordTerms, haystack: Any, yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: """Performs a keyword search.""" invert: bool = terms.inverted keyword: PathSearchKeywords = terms.keyword parameters: List[str] = terms.parameters + nc_matches: Generator[NodeCoords, None, None] = False + + if keyword is PathSearchKeywords.HAS_CHILD: + nc_matches = KeywordSearches.has_child( + haystack, invert, parameters, yaml_path, **kwargs) + else: + raise NotImplementedError + + for nc_match in nc_matches: + yield nc_match + + @staticmethod + def has_child( + data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """Indicate whether data has a named child.""" + parent = kwargs.pop("parent", None) + parentref = kwargs.pop("parentref", None) + traverse_lists = kwargs.pop("traverse_lists", True) + translated_path = kwargs.pop("translated_path", YAMLPath("")) + + # There must be exactly one parameter + param_count = len(parameters) + if param_count != 1: + raise YAMLPathException( + ("Invalid parameter count to {}; {} required, got {} in" + " YAML Path").format( + PathSearchKeywords.HAS_CHILD, 1, param_count), + str(yaml_path)) + match_key = parameters[0] + + # Against a map, this will return nodes which have an immediate + # child key exactly named as per parameters. When inverted, only + # parents with no such key are yielded. + if isinstance(data, dict): + child_present = match_key in data + if ( + (invert and not child_present) or + (child_present and not invert) + ): + yield NodeCoords( + data, parent, parentref, + translated_path) + + # Against a list, this will merely require an exact match between + # parameters and any list elements. When inverted, every + # non-matching element is yielded. + elif isinstance(data, list): + if not traverse_lists: + return + raise NotImplementedError + # Against an AoH, this will scan each element's immediate children, + # treating and yielding as if this search were performed directly + # against each map in the list. + else: + raise NotImplementedError diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 16b3cf0d..37fe677e 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -6,14 +6,13 @@ """ from typing import Any, Dict, Generator, List, Union -from yamlpath.common import Anchors, Nodes, Searches +from yamlpath.common import Anchors, KeywordSearches, Nodes, Searches from yamlpath import YAMLPath from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms from yamlpath.wrappers import ConsolePrinter, NodeCoords from yamlpath.exceptions import YAMLPathException from yamlpath.enums import ( YAMLValueFormats, - PathSearchKeywords, PathSegmentTypes, CollectorOperators, PathSeperators, @@ -888,60 +887,10 @@ def _get_nodes_by_keyword_search( data=data, prefix="Processor::_get_nodes_by_keyword_search: ") - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - traverse_lists = kwargs.pop("traverse_lists", True) - translated_path = kwargs.pop("translated_path", YAMLPath("")) - invert = terms.inverted - keyword = terms.keyword - parameters = terms.parameters - - if keyword is PathSearchKeywords.HAS_CHILD: - # There must be exactly one parameter - param_count = len(parameters) - if param_count != 1: - raise YAMLPathException( - ("Invalid parameter count to {}; {} required, got {} in" - " YAML Path").format(keyword, 1, param_count), - str(yaml_path)) - match_key = parameters[0] - - # Against a map, this will return nodes which have an immediate - # child key exactly named as per parameters. When inverted, only - # parents with no such key are yielded. - if isinstance(data, dict): - child_present = match_key in data - if ( - (invert and not child_present) or - (child_present and not invert) - ): - self.logger.debug( - "Yielding dictionary with child keyword-matched" - " against '{}':".format(match_key), - data=data, - prefix="Processor::_get_nodes_by_keyword_search: ") - yield NodeCoords( - data, parent, parentref, - translated_path) - - # Against a list, this will merely require an exact match between - # parameters and any list elements. When inverted, every - # non-matching element is yielded. - elif isinstance(data, list): - if not traverse_lists: - self.logger.debug( - "Processor::_get_nodes_by_keyword_search: Refusing to" - " traverse a list.") - return - raise NotImplementedError - - # Against an AoH, this will scan each element's immediate children, - # treating and yielding as if this search were performed directly - # against each map in the list. - else: - raise NotImplementedError - else: - raise NotImplementedError + for res_nc in KeywordSearches.search_matches( + terms, data, yaml_path, **kwargs + ): + yield res_nc # pylint: disable=too-many-statements def _get_nodes_by_search( From b717d2ce00747db3cfcb01425ec0e730779618d9 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 17 Apr 2021 15:03:28 -0500 Subject: [PATCH 22/90] Do not allow searches to create nodes --- yamlpath/processor.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 37fe677e..48d6ae81 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1447,14 +1447,6 @@ def _get_optional_nodes( ] = segments[depth][1] except_segment = str(unstripped_attrs) - prior_was_search = False - if depth > 0: - prior_was_search = segments[depth - 1][0] in [ - PathSegmentTypes.KEYWORD_SEARCH, - PathSegmentTypes.SEARCH, - PathSegmentTypes.TRAVERSE - ] - self.logger.debug( "Seeking element <{}>{} in data of type {}:" .format(segment_type, except_segment, type(data)), @@ -1592,21 +1584,6 @@ def _get_optional_nodes( except_segment ) - elif prior_was_search and isinstance(parent, (dict, list)): - self.logger.debug( - ("Setting a post-search {} value at path {} ({}" - " segments) at depth {}, to value {}, when:") - .format( - str(segment_type), str(yaml_path), - str(len(yaml_path)), str(depth + 1), - str(value)), - prefix="Processor::_get_optional_nodes: ", - data={"data": data, "parent": parent, - "parentref": parentref}) - parent[parentref] = value - data = value - yield NodeCoords(data, parent, parentref, translated_path) - else: self.logger.debug( "Assuming data is scalar and cannot receive a {}" From 27badd960c7ee0700d95e6c0614402a67698d15b Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 17 Apr 2021 21:06:53 -0500 Subject: [PATCH 23/90] BUGFIX: bool = bool never worked --- tests/test_common_searches.py | 33 +++++++++++++++++++++++++++------ yamlpath/common/nodes.py | 5 ++++- yamlpath/common/searches.py | 30 +++++++++++++++++++----------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/tests/test_common_searches.py b/tests/test_common_searches.py index fcaa0f27..873172d3 100644 --- a/tests/test_common_searches.py +++ b/tests/test_common_searches.py @@ -12,12 +12,33 @@ class Test_common_searches(): ### # search_matches ### - def test_search_matches(self): - method = PathSearchMethods.CONTAINS - needle = "a" - haystack = "parents" - assert Searches.search_matches(method, needle, haystack) == True - + @pytest.mark.parametrize("match, method, needle, haystack", [ + (True, PathSearchMethods.CONTAINS, "a", "parents"), + (True, PathSearchMethods.ENDS_WITH, "ts", "parents"), + (True, PathSearchMethods.EQUALS, "parents", "parents"), + (True, PathSearchMethods.EQUALS, 42, 42), + (True, PathSearchMethods.EQUALS, "42", 42), + (True, PathSearchMethods.EQUALS, 3.14159265385, 3.14159265385), + (True, PathSearchMethods.EQUALS, "3.14159265385", 3.14159265385), + (True, PathSearchMethods.EQUALS, True, True), + (True, PathSearchMethods.EQUALS, "True", True), + (True, PathSearchMethods.EQUALS, "true", True), + (True, PathSearchMethods.EQUALS, False, False), + (True, PathSearchMethods.EQUALS, "False", False), + (True, PathSearchMethods.EQUALS, "false", False), + (True, PathSearchMethods.GREATER_THAN, 2, 4), + (True, PathSearchMethods.GREATER_THAN, "2", 4), + (True, PathSearchMethods.GREATER_THAN_OR_EQUAL, 2, 4), + (True, PathSearchMethods.GREATER_THAN_OR_EQUAL, "2", 4), + (True, PathSearchMethods.LESS_THAN, 4, 2), + (True, PathSearchMethods.LESS_THAN, "4", 2), + (True, PathSearchMethods.LESS_THAN_OR_EQUAL, 4, 2), + (True, PathSearchMethods.LESS_THAN_OR_EQUAL, "4", 2), + (True, PathSearchMethods.REGEX, ".+", "a"), + (True, PathSearchMethods.STARTS_WITH, "p", "parents") + ]) + def test_search_matches(self, match, method, needle, haystack): + assert match == Searches.search_matches(method, needle, haystack) ### # search_anchor diff --git a/yamlpath/common/nodes.py b/yamlpath/common/nodes.py index 637283ef..5e653bfb 100644 --- a/yamlpath/common/nodes.py +++ b/yamlpath/common/nodes.py @@ -259,7 +259,10 @@ def wrap_type(value: Any) -> Any: wrapped_value = value try: - ast_value = ast.literal_eval(value) + cased_value = value + if str(value).lower() in ("true", "false"): + cased_value = str(value).title() + ast_value = ast.literal_eval(cased_value) except ValueError: ast_value = value except SyntaxError: diff --git a/yamlpath/common/searches.py b/yamlpath/common/searches.py index b495e2df..0abfbfde 100644 --- a/yamlpath/common/searches.py +++ b/yamlpath/common/searches.py @@ -4,6 +4,7 @@ Copyright 2020 William W. Kimball, Jr. MBA MSIS """ import re +from ast import literal_eval from typing import Any, List from yamlpath.enums import ( @@ -24,21 +25,28 @@ def search_matches( method: PathSearchMethods, needle: str, haystack: Any ) -> bool: """Perform a search.""" + try: + cased_needle = needle + lower_needle = str(needle).lower() + if lower_needle in ("true", "false"): + cased_needle = str(needle).title() + typed_needle = literal_eval(cased_needle) + except ValueError: + typed_needle = needle + except SyntaxError: + typed_needle = needle + needle_type = type(typed_needle) matches: bool = False if method is PathSearchMethods.EQUALS: - if isinstance(haystack, int): - try: - matches = haystack == int(needle) - except ValueError: - matches = False - elif isinstance(haystack, float): - try: - matches = haystack == float(needle) - except ValueError: - matches = False + if isinstance(haystack, bool) and needle_type is bool: + matches = haystack == typed_needle + elif isinstance(haystack, int) and needle_type is int: + matches = haystack == typed_needle + elif isinstance(haystack, float) and needle_type is float: + matches = haystack == typed_needle else: - matches = haystack == needle + matches = str(haystack) == str(needle) elif method is PathSearchMethods.STARTS_WITH: matches = str(haystack).startswith(needle) elif method is PathSearchMethods.ENDS_WITH: From e6b3bc3f64d555967c56b9df35a713612b832202 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 17 Apr 2021 23:48:54 -0500 Subject: [PATCH 24/90] Keyword Search refinements --- yamlpath/common/keywordsearches.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 1e7b4b0e..ec08da28 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -22,11 +22,11 @@ def search_matches( terms: SearchKeywordTerms, haystack: Any, yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Performs a keyword search.""" + """Perform a keyword search.""" invert: bool = terms.inverted keyword: PathSearchKeywords = terms.keyword parameters: List[str] = terms.parameters - nc_matches: Generator[NodeCoords, None, None] = False + nc_matches: Generator[NodeCoords, None, None] if keyword is PathSearchKeywords.HAS_CHILD: nc_matches = KeywordSearches.has_child( @@ -77,10 +77,20 @@ def has_child( elif isinstance(data, list): if not traverse_lists: return - raise NotImplementedError + + child_present = match_key in data + if ( + (invert and not child_present) or + (child_present and not invert) + ): + yield NodeCoords( + data, parent, parentref, + translated_path) # Against an AoH, this will scan each element's immediate children, # treating and yielding as if this search were performed directly # against each map in the list. else: - raise NotImplementedError + raise YAMLPathException( + ("{} data has no child nodes in YAML Path").format(type(data)), + str(yaml_path)) From f4ac6afe80d3948767c3870d13b150902973d26f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 00:53:20 -0500 Subject: [PATCH 25/90] WIP: Test expansion --- tests/test_commands_yaml_set.py | 2 +- tests/test_common_keywordsearches.py | 35 ++++++++++++++++++++++++++++ tests/test_yamlpath.py | 10 ++++++++ yamlpath/common/keywordsearches.py | 11 +++++---- yamlpath/path/searchkeywordterms.py | 1 + 5 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 tests/test_common_keywordsearches.py diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index 3a485aad..2c9aafd3 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1293,7 +1293,7 @@ def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) result = script_runner.run( self.command, - "--change=/devices/*/[!has_child(os)]/os", + "--change=/devices/*[!has_child(os)]/os", "--value=generic", yaml_file ) diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py new file mode 100644 index 00000000..116a70e9 --- /dev/null +++ b/tests/test_common_keywordsearches.py @@ -0,0 +1,35 @@ +import pytest + +import ruamel.yaml as ry + +from yamlpath.enums import PathSearchKeywords +from yamlpath.path import SearchKeywordTerms +from yamlpath.common import KeywordSearches +from yamlpath.exceptions import YAMLPathException +from yamlpath import YAMLPath + +class Test_common_keywordsearches(): + """Tests for the KeywordSearches helper class.""" + + ### + # search_matches + ### + def test_unknown_search_keyword(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.search_matches( + SearchKeywordTerms(False, None, ""), + {}, + YAMLPath("/") + )) + + + ### + # has_child + ### + def test_has_child_invalid_param_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.search_matches( + SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, ""), + {}, + YAMLPath("/") + )) diff --git a/tests/test_yamlpath.py b/tests/test_yamlpath.py index 124d145a..8351f062 100644 --- a/tests/test_yamlpath.py +++ b/tests/test_yamlpath.py @@ -186,3 +186,13 @@ def test_parse_meaningless_traversal(self): with pytest.raises(YAMLPathException) as ex: str(YAMLPath("abc**")) assert -1 < str(ex.value).find("The ** traversal operator has no meaning when combined with other characters") + + def test_parse_bad_following_char(self): + with pytest.raises(YAMLPathException) as ex: + str(YAMLPath("abc[has_child(def)ghi]")) + assert -1 < str(ex.value).find("Invalid YAML Path at g, which must be ]") + + def test_parse_unknown_search_keyword(self): + with pytest.raises(YAMLPathException) as ex: + str(YAMLPath("abc[unknown_keyword()]")) + assert -1 < str(ex.value).find("Unknown search keyword, unknown_keyword") diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index ec08da28..5d76005c 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -32,7 +32,9 @@ def search_matches( nc_matches = KeywordSearches.has_child( haystack, invert, parameters, yaml_path, **kwargs) else: - raise NotImplementedError + raise YAMLPathException( + "Unsupported search keyword {} in".format(keyword), + str(yaml_path)) for nc_match in nc_matches: yield nc_match @@ -78,6 +80,10 @@ def has_child( if not traverse_lists: return + # Against an AoH, this will scan each element's immediate children, + # treating and yielding as if this search were performed directly + # against each map in the list. + child_present = match_key in data if ( (invert and not child_present) or @@ -87,9 +93,6 @@ def has_child( data, parent, parentref, translated_path) - # Against an AoH, this will scan each element's immediate children, - # treating and yielding as if this search were performed directly - # against each map in the list. else: raise YAMLPathException( ("{} data has no child nodes in YAML Path").format(type(data)), diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index 8e7f780e..829368a3 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -72,6 +72,7 @@ def parameters(self) -> List[str]: params: List[str] = [] escape_next: bool = False demarc_stack: List[str] = [] + demarc_count: int = 0 # pylint: disable=locally-disabled,too-many-nested-blocks for char in self._parameters: From 6dde7333e1f2968e6551ef44f9b42c83cbf56e88 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 11:31:08 -0500 Subject: [PATCH 26/90] Enable AoH pass-through in has_child --- yamlpath/common/keywordsearches.py | 19 +++++++++++++++---- yamlpath/common/nodes.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 5d76005c..b0274ab9 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -13,6 +13,7 @@ from yamlpath.exceptions import YAMLPathException from yamlpath.wrappers import NodeCoords from yamlpath import YAMLPath +import yamlpath.common class KeywordSearches: """Helper methods for common data searching operations.""" @@ -45,10 +46,10 @@ def has_child( **kwargs: Any ) -> Generator[NodeCoords, None, None]: """Indicate whether data has a named child.""" - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - traverse_lists = kwargs.pop("traverse_lists", True) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + traverse_lists: bool = kwargs.pop("traverse_lists", True) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) # There must be exactly one parameter param_count = len(parameters) @@ -83,6 +84,16 @@ def has_child( # Against an AoH, this will scan each element's immediate children, # treating and yielding as if this search were performed directly # against each map in the list. + if yamlpath.common.Nodes.node_is_aoh(data): + for idx, ele in enumerate(data): + next_path = translated_path.append("[{}]".format(str(idx))) + for aoh_match in KeywordSearches.has_child( + ele, invert, parameters, yaml_path, + parent=data, parentref=idx, translated_path=next_path, + traverse_lists=traverse_lists + ): + yield aoh_match + return child_present = match_key in data if ( diff --git a/yamlpath/common/nodes.py b/yamlpath/common/nodes.py index 5e653bfb..12914cf3 100644 --- a/yamlpath/common/nodes.py +++ b/yamlpath/common/nodes.py @@ -396,6 +396,21 @@ def node_is_leaf(node: Any) -> bool: """Indicate whether a node is a leaf (Scalar data).""" return not isinstance(node, (dict, list, set)) + @staticmethod + def node_is_aoh(node: Any) -> bool: + """Indicate whether a node is an Array-of-Hashes (List of Dicts).""" + if node is None: + return False + + if not isinstance(node, (list, set)): + return False + + for ele in node: + if not isinstance(ele, dict): + return False + + return True + @staticmethod def tagless_elements(data: list) -> list: """Get a copy of a list with all elements stripped of YAML Tags.""" From cea39363cf3402c923bdea399e953afdc4e5218f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 12:21:47 -0500 Subject: [PATCH 27/90] Add code coverage --- tests/test_commands_yaml_get.py | 79 ++++++++++++++++++++++++++++++ tests/test_commands_yaml_set.py | 2 +- tests/test_common_nodes.py | 16 ++++++ yamlpath/common/keywordsearches.py | 1 + 4 files changed, 97 insertions(+), 1 deletion(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index c5da59da..7c320886 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -174,3 +174,82 @@ def test_get_every_data_type(self, script_runner, tmp_path_factory): for line in result.stdout.splitlines(): assert line == results[match_index] match_index += 1 + + def test_get_only_aoh_nodes_without_named_child(self, script_runner, tmp_path_factory): + content = """--- +items: + - - alpha + - bravo + - charlie + - - alpha + - charlie + - delta + - - alpha + - bravo + - delta + - - bravo + - charlie + - delta +""" + results = [ + "alpha", + "charlie", + "delta" + ] + + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run(self.command, "--query=/items/*[!has_child(bravo)]*", yaml_file) + assert result.success, result.stderr + + match_index = 0 + for line in result.stdout.splitlines(): + assert line == results[match_index] + match_index += 1 + + def test_get_only_aoh_nodes_with_named_child(self, script_runner, tmp_path_factory): + content = """--- +products: + - name: something + price: 0.99 + weight: 0.75 + recalled: false + - name: other + price: 9.99 + weight: 2.25 + dimensions: + width: 1 + height: 1 + depth: 1 + - name: moar + weight: 100 + dimensions: + width: 100 + height: 100 + depth: 100 + - name: less + price: 5 + dimensions: + width: 5 + height: 5 + - name: bad + price: 0 + weight: 4 + dimensions: + width: 13 + height: 4 + depth: 7 + recalled: true +""" + results = [ + "something", + "bad", + ] + + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run(self.command, "--query=/products[has_child(recalled)]/name", yaml_file) + assert result.success, result.stderr + + match_index = 0 + for line in result.stdout.splitlines(): + assert line == results[match_index] + match_index += 1 diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index 2c9aafd3..1f51f313 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1244,7 +1244,7 @@ def test_change_null(self, script_runner, tmp_path_factory): assert filedat == yamlout def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): - # Inspiration: https://github.com/wwkimball/yamlpath/issues/107 + # Contributed By: https://github.com/dwapstra yamlin = """--- devices: R1: diff --git a/tests/test_common_nodes.py b/tests/test_common_nodes.py index b0ecfa6b..9e532ed2 100644 --- a/tests/test_common_nodes.py +++ b/tests/test_common_nodes.py @@ -57,3 +57,19 @@ def test_delete_tag(self): ### def test_tagless_value_syntax_error(self): assert "[abc" == Nodes.tagless_value("[abc") + + + ### + # node_is_aoh + ### + def test_aoh_node_is_none(self): + assert False == Nodes.node_is_aoh(None) + + def test_aoh_node_is_not_list(self): + assert False == Nodes.node_is_aoh({"key": "value"}) + + def test_aoh_is_inconsistent(self): + assert False == Nodes.node_is_aoh([ + {"key": "value"}, + None + ]) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index b0274ab9..c6336833 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -41,6 +41,7 @@ def search_matches( yield nc_match @staticmethod + # pylint: disable=locally-disabled,too-many-locals def has_child( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any From 8f8a0d1d9142389ddf473ca4c776ab99b7dff510 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 12:48:53 -0500 Subject: [PATCH 28/90] 100% code coverage --- tests/test_common_keywordsearches.py | 9 +++++++++ yamlpath/common/keywordsearches.py | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py index 116a70e9..d7cf3dc6 100644 --- a/tests/test_common_keywordsearches.py +++ b/tests/test_common_keywordsearches.py @@ -33,3 +33,12 @@ def test_has_child_invalid_param_count(self): {}, YAMLPath("/") )) + + def test_has_child_invalid_node(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.has_child( + "abc: xyz", + False, + ["wwk"], + YAMLPath("") + )) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index c6336833..8472df3c 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -79,9 +79,6 @@ def has_child( # parameters and any list elements. When inverted, every # non-matching element is yielded. elif isinstance(data, list): - if not traverse_lists: - return - # Against an AoH, this will scan each element's immediate children, # treating and yielding as if this search were performed directly # against each map in the list. From 221c6638584d52a2021afc182fd44609e96f3afc Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 14:55:20 -0500 Subject: [PATCH 29/90] Enable suffix removal from YAMLPath --- yamlpath/yamlpath.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index ee57f9b9..213295b4 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -842,7 +842,6 @@ def strip_path_prefix(path: "YAMLPath", prefix: "YAMLPath") -> "YAMLPath": if str(prefix) == "/": return path - prefix.seperator = PathSeperators.FSLASH path.seperator = PathSeperators.FSLASH prefix_str = str(prefix) path_str = str(path) @@ -852,6 +851,33 @@ def strip_path_prefix(path: "YAMLPath", prefix: "YAMLPath") -> "YAMLPath": return path + @staticmethod + def strip_path_suffix(path: "YAMLPath", suffix: "YAMLPath") -> "YAMLPath": + """ + Remove a suffix from a YAML Path. + + Parameters: + 1. path (YAMLPath) The path from which to remove the suffix. + 2. suffix (YAMLPath) The suffix to remove. + + Returns: (YAMLPath) The trimmed YAML Path. + """ + if suffix is None: + return path + + suffix.seperator = PathSeperators.FSLASH + if str(suffix) == "/": + return path + + path.seperator = PathSeperators.FSLASH + suffix_str = str(suffix) + path_str = str(path) + if path_str.endswith(suffix_str): + path_str = path_str[:len(path_str) - len(suffix_str)] + return YAMLPath(path_str) + + return path + @staticmethod def ensure_escaped(value: str, *symbols: str) -> str: r""" From 1c7492de91ffaf4133a43b435b9d821e80f69dda Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 19:39:09 -0500 Subject: [PATCH 30/90] WIP: Experimental [parent()] search keyword --- yamlpath/common/keywordsearches.py | 71 +++++++++- yamlpath/enums/pathsearchkeywords.py | 5 + yamlpath/processor.py | 204 +++++++++++++++++---------- yamlpath/wrappers/nodecoords.py | 8 +- 4 files changed, 212 insertions(+), 76 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 8472df3c..008d8121 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -32,6 +32,9 @@ def search_matches( if keyword is PathSearchKeywords.HAS_CHILD: nc_matches = KeywordSearches.has_child( haystack, invert, parameters, yaml_path, **kwargs) + elif keyword is PathSearchKeywords.PARENT: + nc_matches = KeywordSearches.parent( + haystack, invert, parameters, yaml_path, **kwargs) else: raise YAMLPathException( "Unsupported search keyword {} in".format(keyword), @@ -49,7 +52,6 @@ def has_child( """Indicate whether data has a named child.""" parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) - traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) # There must be exactly one parameter @@ -87,8 +89,7 @@ def has_child( next_path = translated_path.append("[{}]".format(str(idx))) for aoh_match in KeywordSearches.has_child( ele, invert, parameters, yaml_path, - parent=data, parentref=idx, translated_path=next_path, - traverse_lists=traverse_lists + parent=data, parentref=idx, translated_path=next_path ): yield aoh_match return @@ -106,3 +107,67 @@ def has_child( raise YAMLPathException( ("{} data has no child nodes in YAML Path").format(type(data)), str(yaml_path)) + + @staticmethod + def parent( + data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """Climb back up N parent levels in the data hierarchy.""" + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) + + # There may be 0 or 1 parameters + param_count = len(parameters) + if param_count > 1: + raise YAMLPathException(( + "Invalid parameter count to {}; up to {} permitted, got {} in" + " YAML Path" + ).format(PathSearchKeywords.PARENT, 1, param_count), + str(yaml_path)) + + if invert: + raise YAMLPathException(( + "Inversion is meaningless to {}" + ).format(PathSearchKeywords.PARENT), + str(yaml_path)) + + parent_levels: int = 1 + try: + parent_levels = int(parameters[0]) + except ValueError as ex: + raise YAMLPathException(( + "Invalid parameter passed to {}, {}; must be unset or an" + " integer number indicating how may parent levels to climb in" + ).format(PathSearchKeywords.PARENT, parameters[0]), + str(yaml_path)) from ex + + if parent_levels > len(ancestry): + raise YAMLPathException(( + "Invalid parent levels passed to {}; only {} are available in" + ).format(PathSearchKeywords.PARENT, len(ancestry)), + str(yaml_path)) from ex + + if parent_levels < 1: + # parent(0) is the present node + yield NodeCoords( + data, parent, parentref, + translated_path) + else: + ancestor: tuple = ancestry[-parent_levels - 1] + + if parent_levels == len(ancestry): + parent = None + parentref = None + else: + predecessor = ancestry[-parent_levels - 2] + parent = predecessor[0] + parentref = predecessor[1] + + parent_path = translated_path + for _ in range(parent_levels): + parent_path = YAMLPath.strip_path_suffix( + parent_path, YAMLPath(parent_path.unescaped[-1][1])) + yield NodeCoords(ancestor[0], parent, parentref, parent_path) diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index 3da8e2e7..56850499 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -15,15 +15,20 @@ class PathSearchKeywords(Enum): `HAS_CHILD` Matches when the node has a direct child with a given name. + `PARENT` + Access the parent of the present node. """ HAS_CHILD = auto() + PARENT = auto() def __str__(self) -> str: """Get a String representation of an employed value of this enum.""" keyword = '' if self is PathSearchKeywords.HAS_CHILD: keyword = 'has_child' + elif self is PathSearchKeywords.PARENT: + keyword = 'parent' return keyword diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 48d6ae81..cdc135c8 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -554,6 +554,7 @@ def _get_nodes_by_path_segment(self, data: Any, * parentref (Any) The Index or Key of data within parent * traverse_lists (Boolean) Indicate whether KEY searches against lists are permitted to automatically traverse into the list; Default=True + * ancestry (List[tuple]) Returns: (Generator[Any, None, None]) Each node coordinate or list of node coordinates as they are matched. You must check with isinstance() @@ -564,10 +565,11 @@ def _get_nodes_by_path_segment(self, data: Any, - `NotImplementedError` when the segment indicates an unknown PathSegmentTypes value. """ - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - traverse_lists = kwargs.pop("traverse_lists", True) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + traverse_lists: bool = kwargs.pop("traverse_lists", True) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) if data is None: self.logger.debug( "Bailing out on None data at parentref, {}, of parent:" @@ -602,15 +604,15 @@ def _get_nodes_by_path_segment(self, data: Any, if segment_type == PathSegmentTypes.KEY: node_coords = self._get_nodes_by_key( data, yaml_path, segment_index, traverse_lists=traverse_lists, - translated_path=translated_path) + translated_path=translated_path, ancestry=ancestry) elif segment_type == PathSegmentTypes.INDEX: node_coords = self._get_nodes_by_index( data, yaml_path, segment_index, - translated_path=translated_path) + translated_path=translated_path, ancestry=ancestry) elif segment_type == PathSegmentTypes.ANCHOR: node_coords = self._get_nodes_by_anchor( data, yaml_path, segment_index, - translated_path=translated_path) + translated_path=translated_path, ancestry=ancestry) elif ( segment_type == PathSegmentTypes.KEYWORD_SEARCH and isinstance(stripped_attrs, SearchKeywordTerms) @@ -618,25 +620,28 @@ def _get_nodes_by_path_segment(self, data: Any, node_coords = self._get_nodes_by_keyword_search( data, yaml_path, stripped_attrs, parent=parent, parentref=parentref, traverse_lists=traverse_lists, - translated_path=translated_path) + translated_path=translated_path, ancestry=ancestry) elif ( segment_type == PathSegmentTypes.SEARCH and isinstance(stripped_attrs, SearchTerms) ): node_coords = self._get_nodes_by_search( data, stripped_attrs, parent=parent, parentref=parentref, - traverse_lists=traverse_lists, translated_path=translated_path) + traverse_lists=traverse_lists, translated_path=translated_path, + ancestry=ancestry) elif ( unesc_type == PathSegmentTypes.COLLECTOR and isinstance(unesc_attrs, CollectorTerms) ): node_coords = self._get_nodes_by_collector( data, yaml_path, segment_index, unesc_attrs, parent=parent, - parentref=parentref, translated_path=translated_path) + parentref=parentref, translated_path=translated_path, + ancestry=ancestry) elif segment_type == PathSegmentTypes.TRAVERSE: node_coords = self._get_nodes_by_traversal( data, yaml_path, segment_index, parent=parent, - parentref=parentref, translated_path=translated_path) + parentref=parentref, translated_path=translated_path, + ancestry=ancestry) else: raise NotImplementedError @@ -667,8 +672,9 @@ def _get_nodes_by_key( Raises: N/A """ - traverse_lists = kwargs.pop("traverse_lists", True) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + traverse_lists: bool = kwargs.pop("traverse_lists", True) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) (_, stripped_attrs) = yaml_path.escaped[segment_index] str_stripped = str(stripped_attrs) @@ -681,6 +687,7 @@ def _get_nodes_by_key( next_translated_path = (translated_path + YAMLPath.escape_path_section( str_stripped, translated_path.seperator)) + next_ancestry = ancestry + [(data, stripped_attrs)] if stripped_attrs in data: self.logger.debug( "Processor::_get_nodes_by_key: FOUND key node by name at" @@ -688,14 +695,15 @@ def _get_nodes_by_key( .format(str_stripped)) yield NodeCoords( data[stripped_attrs], data, stripped_attrs, - next_translated_path) + next_translated_path, next_ancestry) else: # Check for a string/int type mismatch try: intkey = int(str_stripped) if intkey in data: yield NodeCoords( - data[intkey], data, intkey, next_translated_path) + data[intkey], data, intkey, next_translated_path, + ancestry + [(data, intkey)]) except ValueError: pass elif isinstance(data, list): @@ -709,7 +717,8 @@ def _get_nodes_by_key( .format(str_stripped)) yield NodeCoords( data[idx], data, idx, - translated_path + "[{}]".format(idx)) + translated_path + "[{}]".format(idx), + ancestry + [(data, idx)]) except ValueError: # Pass-through search against possible Array-of-Hashes, if # allowed. @@ -722,10 +731,12 @@ def _get_nodes_by_key( for eleidx, element in enumerate(data): next_translated_path = translated_path + "[{}]".format( eleidx) + next_ancestry = ancestry + [(data, stripped_attrs)] for node_coord in self._get_nodes_by_path_segment( element, yaml_path, segment_index, parent=data, parentref=eleidx, traverse_lists=traverse_lists, - translated_path=next_translated_path): + translated_path=next_translated_path, + ancestry=next_ancestry): self.logger.debug( "Processor::_get_nodes_by_key: FOUND key node " " via pass-through Array-of-Hashes search at {}." @@ -756,7 +767,8 @@ def _get_nodes_by_index( (_, stripped_attrs) = yaml_path.escaped[segment_index] (_, unstripped_attrs) = yaml_path.unescaped[segment_index] str_stripped = str(stripped_attrs) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) self.logger.debug( "Processor::_get_nodes_by_index: Seeking INDEX node at {}." @@ -782,16 +794,19 @@ def _get_nodes_by_index( if intmin == intmax and len(data) > intmin: yield NodeCoords( [data[intmin]], data, intmin, - translated_path + "[{}]".format(intmin)) + translated_path + "[{}]".format(intmin), + ancestry + [(data, intmin)]) else: sliced_elements = [] for slice_index in range(intmin, intmax): sliced_elements.append(NodeCoords( data[slice_index], data, intmin, - translated_path + "[{}]".format(slice_index))) + translated_path + "[{}]".format(slice_index), + ancestry + [(data, slice_index)])) yield NodeCoords( sliced_elements, data, intmin, - translated_path + "[{}:{}]".format(intmin, intmax)) + translated_path + "[{}:{}]".format(intmin, intmax), + ancestry + [(data, intmin)]) elif isinstance(data, dict): for key, val in data.items(): @@ -799,7 +814,8 @@ def _get_nodes_by_index( yield NodeCoords( val, data, key, translated_path + YAMLPath.escape_path_section( - key, translated_path.seperator)) + key, translated_path.seperator), + ancestry + [(data, key)]) else: try: idx: int = int(str_stripped) @@ -813,7 +829,8 @@ def _get_nodes_by_index( if isinstance(data, list) and len(data) > idx: yield NodeCoords( - data[idx], data, idx, translated_path + "[{}]".format(idx)) + data[idx], data, idx, translated_path + "[{}]".format(idx), + ancestry + [(data, idx)]) def _get_nodes_by_anchor( self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs @@ -835,7 +852,8 @@ def _get_nodes_by_anchor( Raises: N/A """ (_, stripped_attrs) = yaml_path.escaped[segment_index] - translated_path = kwargs.pop("translated_path", YAMLPath("")) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) next_translated_path = translated_path + "[&{}]".format( YAMLPath.escape_path_section( str(stripped_attrs), translated_path.seperator)) @@ -848,15 +866,21 @@ def _get_nodes_by_anchor( for lstidx, ele in enumerate(data): if (hasattr(ele, "anchor") and stripped_attrs == ele.anchor.value): - yield NodeCoords(ele, data, lstidx, next_translated_path) + yield NodeCoords(ele, data, lstidx, next_translated_path, + ancestry + [(data, lstidx)]) elif isinstance(data, dict): for key, val in data.items(): + next_ancestry = ancestry + [(data, key)] if (hasattr(key, "anchor") and stripped_attrs == key.anchor.value): - yield NodeCoords(val, data, key, next_translated_path) + yield NodeCoords( + val, data, key, next_translated_path, + next_ancestry) elif (hasattr(val, "anchor") and stripped_attrs == val.anchor.value): - yield NodeCoords(val, data, key, next_translated_path) + yield NodeCoords( + val, data, key, next_translated_path, + next_ancestry) def _get_nodes_by_keyword_search( self, data: Any, yaml_path: YAMLPath, terms: SearchKeywordTerms, @@ -923,10 +947,11 @@ def _get_nodes_by_search( data=data, prefix="Processor::_get_nodes_by_search: ") - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - traverse_lists = kwargs.pop("traverse_lists", True) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + traverse_lists: bool = kwargs.pop("traverse_lists", True) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) invert = terms.inverted method = terms.method attr = terms.attribute @@ -950,8 +975,11 @@ def _get_nodes_by_search( # Attempt a descendant search next_translated_path = translated_path + "[{}]".format( lstidx) + next_ancestry = ancestry + [(data, lstidx)] for desc_node in self._get_required_nodes( - ele, desc_path, 0, translated_path=next_translated_path + ele, desc_path, 0, + translated_path=next_translated_path, + ancestry=next_ancestry ): matches = Searches.search_matches( method, term, desc_node.node) @@ -965,7 +993,8 @@ def _get_nodes_by_search( prefix="Processor::_get_nodes_by_search: ") yield NodeCoords( ele, data, lstidx, - translated_path + "[{}]".format(lstidx)) + translated_path + "[{}]".format(lstidx), + ancestry + [(data, lstidx)]) elif isinstance(data, dict): # Allow . to mean "each key's name" @@ -982,7 +1011,8 @@ def _get_nodes_by_search( yield NodeCoords( val, data, key, translated_path + YAMLPath.escape_path_section( - key, translated_path.seperator)) + key, translated_path.seperator), + ancestry + [(data, key)]) elif attr in data: value = data[attr] @@ -997,13 +1027,14 @@ def _get_nodes_by_search( yield NodeCoords( value, data, attr, translated_path + YAMLPath.escape_path_section( - attr, translated_path.seperator)) + attr, translated_path.seperator), + ancestry + [(data, attr)]) else: # Attempt a descendant search for desc_node in self._get_required_nodes( data, desc_path, 0, parent=parent, parentref=parentref, - translated_path=translated_path + translated_path=translated_path, ancestry=ancestry ): matches = Searches.search_matches( method, term, desc_node.node) @@ -1016,7 +1047,8 @@ def _get_nodes_by_search( .format(attr), data=data, prefix="Processor::_get_nodes_by_search: ") - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) else: # Check the passed data itself for a match @@ -1026,7 +1058,8 @@ def _get_nodes_by_search( self.logger.debug( "Yielding the queried data itself because it matches.", prefix="Processor::_get_nodes_by_search: ") - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) self.logger.debug( "Finished seeking SEARCH nodes matching {} in data with {}:" @@ -1066,16 +1099,18 @@ def _get_nodes_by_collector( yield data return - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) node_coords = [] # A list of NodeCoords self.logger.debug( "Processor::_get_nodes_by_collector: Getting required nodes" " matching search expression: {}".format(terms.expression)) for node_coord in self._get_required_nodes( data, YAMLPath(terms.expression), 0, parent=parent, - parentref=parentref, translated_path=translated_path): + parentref=parentref, translated_path=translated_path, + ancestry=ancestry): node_coords.append(node_coord) # This may end up being a bad idea for some cases, but this method will @@ -1096,7 +1131,7 @@ def _get_nodes_by_collector( flat_nodes.append( NodeCoords( flatten_node, node_coord.parent, flatten_idx, - node_coord.path)) + node_coord.path, node_coord.ancestry)) node_coords = flat_nodes # As long as each next segment is an ADDITION or SUBTRACTION @@ -1116,7 +1151,8 @@ def _get_nodes_by_collector( for node_coord in self._get_required_nodes( data, peek_path, 0, parent=parent, parentref=parentref, - translated_path=translated_path): + translated_path=translated_path, + ancestry=ancestry): if (isinstance(node_coord, NodeCoords) and isinstance(node_coord.node, list)): for coord_idx, coord in enumerate(node_coord.node): @@ -1126,9 +1162,12 @@ def _get_nodes_by_collector( next_translated_path = ( next_translated_path + "[{}]".format(coord_idx)) + next_ancestry = ancestry + [( + node_coord.data, coord_idx)] coord = NodeCoords( coord, node_coord.node, coord_idx, - next_translated_path) + next_translated_path, + next_ancestry) node_coords.append(coord) else: node_coords.append(node_coord) @@ -1137,7 +1176,8 @@ def _get_nodes_by_collector( for node_coord in self._get_required_nodes( data, peek_path, 0, parent=parent, parentref=parentref, - translated_path=translated_path): + translated_path=translated_path, + ancestry=ancestry): unwrapped_data = NodeCoords.unwrap_node_coords( node_coord) if isinstance(unwrapped_data, list): @@ -1185,9 +1225,10 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, Returns: (Generator[Any, None, None]) Each node coordinate as they are matched. """ - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) self.logger.debug( "TRAVERSING the tree at parentref:", @@ -1208,10 +1249,12 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, next_translated_path = ( translated_path + YAMLPath.escape_path_section( key, translated_path.seperator)) + next_ancestry = ancestry + [(data, key)] for node_coord in self._get_nodes_by_traversal( val, yaml_path, segment_index, parent=data, parentref=key, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): self.logger.debug( "Yielding unfiltered Hash value:", @@ -1235,7 +1278,8 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, self.logger.debug( "Yielding unfiltered Scalar value:", prefix="Processor::_get_nodes_by_traversal: ", data=data) - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) else: # There is a filter in the next segment; recurse data, comparing # every child against the following segment until there are no more @@ -1251,14 +1295,15 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, for node_coord in self._get_nodes_by_path_segment( data, yaml_path, segment_index + 1, parent=parent, parentref=parentref, traverse_lists=False, - translated_path=translated_path + translated_path=translated_path, ancestry=ancestry ): self.logger.debug( "Yielding filtered DIRECT node at parentref {} of coord:" .format(parentref), prefix="Processor::_get_nodes_by_traversal: ", data=node_coord) - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) # Then, recurse into each child to perform the same test. if isinstance(data, dict): @@ -1270,10 +1315,12 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, next_translated_path = ( translated_path + YAMLPath.escape_path_section( key, translated_path.seperator)) + next_ancestry = ancestry + [(data, key)] for node_coord in self._get_nodes_by_traversal( val, yaml_path, segment_index, parent=data, parentref=key, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): self.logger.debug( "Yielding filtered indirect Hash value from KEY" @@ -1288,10 +1335,12 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, " INDEX '{}' at ref '{}' for next-segment matches..." .format(idx, parentref)) next_translated_path = translated_path + "[{}]".format(idx) + next_ancestry = ancestry + [(data, idx)] for node_coord in self._get_nodes_by_traversal( ele, yaml_path, segment_index, parent=data, parentref=idx, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): self.logger.debug( "Yielding filtered indirect Array value from INDEX" @@ -1319,9 +1368,10 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, Raises: N/A """ - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) if data is None: self.logger.debug( @@ -1343,7 +1393,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, for segment_node_coords in self._get_nodes_by_path_segment( data, yaml_path, depth, parent=parent, parentref=parentref, - translated_path=translated_path + translated_path=translated_path, ancestry=ancestry ): self.logger.debug( "Got data of type {} at <{}>{} in the data." @@ -1374,7 +1424,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, data=segment_node_coords) for subnode_coord in self._get_required_nodes( segment_node_coords, yaml_path, depth + 1, - translated_path=translated_path): + translated_path=translated_path, + ancestry=ancestry): yield subnode_coord else: self.logger.debug( @@ -1384,7 +1435,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, segment_node_coords.node, yaml_path, depth + 1, parent=segment_node_coords.parent, parentref=segment_node_coords.parentref, - translated_path=segment_node_coords.path): + translated_path=segment_node_coords.path, + ancestry=segment_node_coords.ancestry): self.logger.debug( "Finally returning segment data of type {} at" " parentref {}:" @@ -1399,7 +1451,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, .format(type(data), parentref), prefix="Processor::_get_required_nodes: ", data=data, footer=" ") - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) # pylint: disable=locally-disabled,too-many-statements def _get_optional_nodes( @@ -1431,9 +1484,10 @@ def _get_optional_nodes( an element that does not exist in data and this code isn't yet prepared to add it. """ - parent = kwargs.pop("parent", None) - parentref = kwargs.pop("parentref", None) - translated_path = kwargs.pop("translated_path", YAMLPath("")) + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) segments = yaml_path.escaped # pylint: disable=locally-disabled,too-many-nested-blocks @@ -1457,7 +1511,7 @@ def _get_optional_nodes( matched_nodes = 0 for next_coord in self._get_nodes_by_path_segment( data, yaml_path, depth, parent=parent, parentref=parentref, - translated_path=translated_path + translated_path=translated_path, ancestry=ancestry ): matched_nodes += 1 self.logger.debug( @@ -1469,7 +1523,8 @@ def _get_optional_nodes( next_coord.node, yaml_path, value, depth + 1, parent=next_coord.parent, parentref=next_coord.parentref, - translated_path=next_coord.path + translated_path=next_coord.path, + ancestry=next_coord.ancestry ): yield node_coord @@ -1503,10 +1558,12 @@ def _get_optional_nodes( new_idx = len(data) - 1 next_translated_path = translated_path + "[{}]".format( new_idx) + next_ancestry = ancestry + [(data, new_idx)] for node_coord in self._get_optional_nodes( new_ele, yaml_path, value, depth + 1, parent=data, parentref=new_idx, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): matched_nodes += 1 yield node_coord @@ -1535,10 +1592,12 @@ def _get_optional_nodes( Nodes.append_list_element(data, next_node) next_translated_path = translated_path + "[{}]".format( newidx) + next_ancestry = ancestry + [(data, newidx)] for node_coord in self._get_optional_nodes( data[newidx], yaml_path, value, depth + 1, parent=data, parentref=newidx, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): matched_nodes += 1 yield node_coord @@ -1568,11 +1627,13 @@ def _get_optional_nodes( translated_path + YAMLPath.escape_path_section( str(stripped_attrs), translated_path.seperator)) + next_ancestry = ancestry + [(data, stripped_attrs)] for node_coord in self._get_optional_nodes( data[stripped_attrs], yaml_path, value, depth + 1, parent=data, parentref=stripped_attrs, - translated_path=next_translated_path + translated_path=next_translated_path, + ancestry=next_ancestry ): matched_nodes += 1 yield node_coord @@ -1606,7 +1667,8 @@ def _get_optional_nodes( "Finally returning data of type {}:" .format(type(data)), prefix="Processor::_get_optional_nodes: ", data=data) - yield NodeCoords(data, parent, parentref, translated_path) + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) # pylint: disable=too-many-arguments def _update_node( diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index 2a049edf..01f51f68 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -1,5 +1,5 @@ """Wrap a node along with its relative coordinates within its DOM.""" -from typing import Any +from typing import Any, List from yamlpath import YAMLPath @@ -14,7 +14,8 @@ class NodeCoords: """ def __init__( - self, node: Any, parent: Any, parentref: Any, path: YAMLPath = None + self, node: Any, parent: Any, parentref: Any, path: YAMLPath = None, + ancestry: List[tuple] = None ) -> None: """ Initialize a new NodeCoords. @@ -26,6 +27,8 @@ def __init__( within `parent` the `node` is located 4. path (YAMLPath) The YAML Path for this node, as reported by its creator process + 5. ancestry (List[tuple]) Tuples in (parent,parentref) form tracking + the hierarchical ancestry of this node through its parent document Returns: N/A @@ -35,6 +38,7 @@ def __init__( self.parent = parent self.parentref = parentref self.path = path + self.ancestry: List[tuple] = [] if ancestry is None else ancestry def __str__(self) -> str: """Get a String representation of this object.""" From 03049bf3f93bc293309bb4abdf3b8e6d247f9548 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 18 Apr 2021 23:19:48 -0500 Subject: [PATCH 31/90] WIP: Working out parent crawls --- yamlpath/common/keywordsearches.py | 64 ++++++++++++++---------------- yamlpath/yamlpath.py | 53 +++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 37 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 008d8121..23a0786f 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -53,6 +53,7 @@ def has_child( parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) # There must be exactly one parameter param_count = len(parameters) @@ -74,8 +75,7 @@ def has_child( (child_present and not invert) ): yield NodeCoords( - data, parent, parentref, - translated_path) + data, parent, parentref, translated_path, ancestry) # Against a list, this will merely require an exact match between # parameters and any list elements. When inverted, every @@ -100,8 +100,7 @@ def has_child( (child_present and not invert) ): yield NodeCoords( - data, parent, parentref, - translated_path) + data, parent, parentref, translated_path, ancestry) else: raise YAMLPathException( @@ -109,6 +108,7 @@ def has_child( str(yaml_path)) @staticmethod + # pylint: disable=locally-disabled,too-many-locals def parent( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any @@ -135,39 +135,35 @@ def parent( str(yaml_path)) parent_levels: int = 1 - try: - parent_levels = int(parameters[0]) - except ValueError as ex: - raise YAMLPathException(( - "Invalid parameter passed to {}, {}; must be unset or an" - " integer number indicating how may parent levels to climb in" - ).format(PathSearchKeywords.PARENT, parameters[0]), - str(yaml_path)) from ex - - if parent_levels > len(ancestry): + ancestry_len: int = len(ancestry) + steps_max = ancestry_len - 1 + if param_count > 0: + try: + parent_levels = int(parameters[0]) + except ValueError as ex: + raise YAMLPathException(( + "Invalid parameter passed to {}([STEPS]), {}; must be" + " unset or an integer number indicating how may parent" + " STEPS to climb in" + ).format(PathSearchKeywords.PARENT, parameters[0]), + str(yaml_path)) from ex + + if parent_levels > steps_max: raise YAMLPathException(( - "Invalid parent levels passed to {}; only {} are available in" - ).format(PathSearchKeywords.PARENT, len(ancestry)), - str(yaml_path)) from ex + "Too many STEPS passed to {}([STEPS]) keyword search; only {}" + " available in" + ).format(PathSearchKeywords.PARENT, steps_max), + str(yaml_path)) if parent_levels < 1: # parent(0) is the present node yield NodeCoords( - data, parent, parentref, - translated_path) + data, parent, parentref, translated_path, ancestry) else: - ancestor: tuple = ancestry[-parent_levels - 1] - - if parent_levels == len(ancestry): - parent = None - parentref = None - else: - predecessor = ancestry[-parent_levels - 2] - parent = predecessor[0] - parentref = predecessor[1] - - parent_path = translated_path - for _ in range(parent_levels): - parent_path = YAMLPath.strip_path_suffix( - parent_path, YAMLPath(parent_path.unescaped[-1][1])) - yield NodeCoords(ancestor[0], parent, parentref, parent_path) + for _ in range(parent_levels + 1): + translated_path.pop() + (parent, parentref) = ancestry.pop() + data = parent[parentref] + + yield NodeCoords( + data, parent, parentref, translated_path, ancestry) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 213295b4..d1dcdc3b 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -127,6 +127,51 @@ def append(self, segment: str) -> "YAMLPath": self.original += "{}{}".format(seperator, segment) return self + def pop(self) -> PathSegment: + """ + Pop the last segment off this YAML Path. + + This mutates the YAML Path and returns the removed segment tuple. + + Returns: (PathSegment) The removed segment + """ + segments: Deque[PathSegment] = self.unescaped + if len(segments) < 1: + raise YAMLPathException( + "Cannot pop when there are no segments to pop from", + str(self)) + + popped_queue = deque() + popped_segment: PathSegment = segments.pop() + popped_queue.append(popped_segment) + removable_segment = YAMLPath._stringify_yamlpath_segments( + popped_queue, self.seperator) + prefixed_segment = "{}{}".format(self.seperator, removable_segment) + path_now = self.original + + if path_now.endswith(prefixed_segment): + self.original = path_now[0:len(path_now) - len(prefixed_segment)] + elif path_now.endswith(removable_segment): + self.original = path_now[0:len(path_now) - len(removable_segment)] + elif ( + self.seperator == PathSeperators.FSLASH + and path_now.endswith(prefixed_segment[1:]) + ): + self.original = path_now[ + 0:len(path_now) - len(prefixed_segment) + 1] + elif ( + self.seperator == PathSeperators.FSLASH + and path_now.endswith(removable_segment[1:]) + ): + self.original = path_now[ + 0:len(path_now) - len(removable_segment) + 1] + else: + raise YAMLPathException( + "Unable to pop unmatchable segment, {}" + .format(removable_segment), str(self)) + + return popped_segment + @property def original(self) -> str: """ @@ -152,11 +197,13 @@ def original(self, value: str) -> None: Raises: N/A """ + str_val = str(value) + # Check for empty paths - if not str(value).strip(): - value = "" + if not str_val.strip(): + str_val = "" - self._original = value + self._original = str_val self._seperator = PathSeperators.AUTO self._unescaped = deque() self._escaped = deque() From b3298f88e80e2bdffb844bdca3ec208e6552bc53 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 13:25:52 -0500 Subject: [PATCH 32/90] parent([STEPS]) is fully working --- yamlpath/common/keywordsearches.py | 30 +++++++++++++++++------------ yamlpath/processor.py | 4 ++++ yamlpath/wrappers/consoleprinter.py | 6 ++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 23a0786f..17384ea9 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -114,6 +114,7 @@ def parent( **kwargs: Any ) -> Generator[NodeCoords, None, None]: """Climb back up N parent levels in the data hierarchy.""" + parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) @@ -123,20 +124,20 @@ def parent( param_count = len(parameters) if param_count > 1: raise YAMLPathException(( - "Invalid parameter count to {}; up to {} permitted, got {} in" - " YAML Path" + "Invalid parameter count to {}([STEPS]); up to {} permitted, " + " got {} in YAML Path" ).format(PathSearchKeywords.PARENT, 1, param_count), str(yaml_path)) if invert: raise YAMLPathException(( - "Inversion is meaningless to {}" + "Inversion is meaningless to {}([STEPS])" ).format(PathSearchKeywords.PARENT), str(yaml_path)) parent_levels: int = 1 ancestry_len: int = len(ancestry) - steps_max = ancestry_len - 1 + steps_max = ancestry_len if param_count > 0: try: parent_levels = int(parameters[0]) @@ -144,15 +145,15 @@ def parent( raise YAMLPathException(( "Invalid parameter passed to {}([STEPS]), {}; must be" " unset or an integer number indicating how may parent" - " STEPS to climb in" + " STEPS to climb in YAML Path" ).format(PathSearchKeywords.PARENT, parameters[0]), str(yaml_path)) from ex if parent_levels > steps_max: raise YAMLPathException(( - "Too many STEPS passed to {}([STEPS]) keyword search; only {}" - " available in" - ).format(PathSearchKeywords.PARENT, steps_max), + "Cannot {}([STEPS]) higher than the document root. {} steps" + " requested when {} available in YAML Path" + ).format(PathSearchKeywords.PARENT, parent_levels, steps_max), str(yaml_path)) if parent_levels < 1: @@ -160,10 +161,15 @@ def parent( yield NodeCoords( data, parent, parentref, translated_path, ancestry) else: - for _ in range(parent_levels + 1): + for _ in range(parent_levels): translated_path.pop() - (parent, parentref) = ancestry.pop() - data = parent[parentref] + (data, _) = ancestry.pop() + ancestry_len -= 1 - yield NodeCoords( + parentref = ancestry[-1][1] if ancestry_len > 1 else None + parent = ancestry[-1][0] if ancestry_len > 1 else None + + parent_nc = NodeCoords( data, parent, parentref, translated_path, ancestry) + + yield parent_nc diff --git a/yamlpath/processor.py b/yamlpath/processor.py index cdc135c8..d9713aea 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -914,6 +914,10 @@ def _get_nodes_by_keyword_search( for res_nc in KeywordSearches.search_matches( terms, data, yaml_path, **kwargs ): + self.logger.debug( + "Yielding keyword search match:", + data=res_nc, + prefix="Processor::_get_nodes_by_keyword_search: ") yield res_nc # pylint: disable=too-many-statements diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 01239e07..93ed4f8f 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -291,6 +291,7 @@ def _debug_node_coord( node_prefix = "{}(node)".format(prefix) parent_prefix = "{}(parent)".format(prefix) parentref_prefix = "{}(parentref)".format(prefix) + ancestry_prefix = "{}(ancestry)".format(prefix) for line in ConsolePrinter._debug_dump(data.path, prefix=path_prefix): yield line @@ -308,6 +309,11 @@ def _debug_node_coord( ): yield line + for line in ConsolePrinter._debug_dump( + data.ancestry, prefix=ancestry_prefix + ): + yield line + @staticmethod def _debug_list( data: Union[List[Any], Set[Any], Tuple[Any, ...], Deque[Any]], **kwargs From 86cae7397cb758ff65008959aeca890f47663833 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 13:38:34 -0500 Subject: [PATCH 33/90] BUGFIX: Some YAML dates slipped past JSONification --- yamlpath/common/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yamlpath/common/parsers.py b/yamlpath/common/parsers.py index e45b8fdc..81bfcdaa 100644 --- a/yamlpath/common/parsers.py +++ b/yamlpath/common/parsers.py @@ -307,7 +307,7 @@ def jsonify_yaml_data(data: Any) -> Any: support for certain YAML extensions -- like tags -- and some otherwise native data-types, like dates. """ - if isinstance(data, CommentedMap): + if isinstance(data, (dict, CommentedMap)): for i, k in [ (idx, key) for idx, key in enumerate(data.keys()) if isinstance(key, TaggedScalar) @@ -317,7 +317,7 @@ def jsonify_yaml_data(data: Any) -> Any: for key, val in data.items(): data[key] = Parsers.jsonify_yaml_data(val) - elif isinstance(data, CommentedSeq): + elif isinstance(data, (list, CommentedSeq)): for idx, ele in enumerate(data): data[idx] = Parsers.jsonify_yaml_data(ele) elif isinstance(data, TaggedScalar): From 22c379127bf525a8ebcb51cc321c38135b15fd1b Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 20:27:04 -0500 Subject: [PATCH 34/90] nc.data -> nc.node --- yamlpath/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index d9713aea..d8e9f325 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1167,7 +1167,7 @@ def _get_nodes_by_collector( next_translated_path + "[{}]".format(coord_idx)) next_ancestry = ancestry + [( - node_coord.data, coord_idx)] + node_coord.node, coord_idx)] coord = NodeCoords( coord, node_coord.node, coord_idx, next_translated_path, From 31333f3f10587bbfc098a3677d8ac000492349b7 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 20:27:28 -0500 Subject: [PATCH 35/90] Specify Deque data-type --- yamlpath/yamlpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index d1dcdc3b..06075b33 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -141,7 +141,7 @@ def pop(self) -> PathSegment: "Cannot pop when there are no segments to pop from", str(self)) - popped_queue = deque() + popped_queue: Deque = deque() popped_segment: PathSegment = segments.pop() popped_queue.append(popped_segment) removable_segment = YAMLPath._stringify_yamlpath_segments( From 9ea8b6afc2d3e3c6f996916454a451fca8d2557b Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 20:27:42 -0500 Subject: [PATCH 36/90] Remove unnecessary blank line --- yamlpath/common/keywordsearches.py | 1 - 1 file changed, 1 deletion(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 17384ea9..7fce0427 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -114,7 +114,6 @@ def parent( **kwargs: Any ) -> Generator[NodeCoords, None, None]: """Climb back up N parent levels in the data hierarchy.""" - parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) From 8234bfea90e8bd9d7e00f0bc2a8700935434c645 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 20:28:03 -0500 Subject: [PATCH 37/90] dict doesn't (yet) support insert --- yamlpath/common/parsers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/yamlpath/common/parsers.py b/yamlpath/common/parsers.py index 81bfcdaa..260abacd 100644 --- a/yamlpath/common/parsers.py +++ b/yamlpath/common/parsers.py @@ -307,7 +307,7 @@ def jsonify_yaml_data(data: Any) -> Any: support for certain YAML extensions -- like tags -- and some otherwise native data-types, like dates. """ - if isinstance(data, (dict, CommentedMap)): + if isinstance(data, CommentedMap): for i, k in [ (idx, key) for idx, key in enumerate(data.keys()) if isinstance(key, TaggedScalar) @@ -315,6 +315,16 @@ def jsonify_yaml_data(data: Any) -> Any: unwrapped_key = Parsers.jsonify_yaml_data(k) data.insert(i, unwrapped_key, data.pop(k)) + for key, val in data.items(): + data[key] = Parsers.jsonify_yaml_data(val) + elif isinstance(data, dict): + for i, k in [ + (idx, key) for idx, key in enumerate(data.keys()) + if isinstance(key, TaggedScalar) + ]: + unwrapped_key = Parsers.jsonify_yaml_data(k) + data[unwrapped_key] = data.pop(k) + for key, val in data.items(): data[key] = Parsers.jsonify_yaml_data(val) elif isinstance(data, (list, CommentedSeq)): From e71a94743158f5aec3f8677e5802a34cad89fdae Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Mon, 19 Apr 2021 20:28:16 -0500 Subject: [PATCH 38/90] appease pylint --- yamlpath/wrappers/nodecoords.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index 01f51f68..7057ca3c 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -13,6 +13,7 @@ class NodeCoords: 3. Index-or-Key-of-the-Node-Within-Its-Immediate-Parent """ + # pylint: disable=locally-disabled,too-many-arguments def __init__( self, node: Any, parent: Any, parentref: Any, path: YAMLPath = None, ancestry: List[tuple] = None From 16713a6cbfa573be4256a9891a0178f64dc1e070 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 20 Apr 2021 10:39:19 -0500 Subject: [PATCH 39/90] Remove unused strip_path_suffix(); use pop() --- yamlpath/yamlpath.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 06075b33..69057502 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -898,33 +898,6 @@ def strip_path_prefix(path: "YAMLPath", prefix: "YAMLPath") -> "YAMLPath": return path - @staticmethod - def strip_path_suffix(path: "YAMLPath", suffix: "YAMLPath") -> "YAMLPath": - """ - Remove a suffix from a YAML Path. - - Parameters: - 1. path (YAMLPath) The path from which to remove the suffix. - 2. suffix (YAMLPath) The suffix to remove. - - Returns: (YAMLPath) The trimmed YAML Path. - """ - if suffix is None: - return path - - suffix.seperator = PathSeperators.FSLASH - if str(suffix) == "/": - return path - - path.seperator = PathSeperators.FSLASH - suffix_str = str(suffix) - path_str = str(path) - if path_str.endswith(suffix_str): - path_str = path_str[:len(path_str) - len(suffix_str)] - return YAMLPath(path_str) - - return path - @staticmethod def ensure_escaped(value: str, *symbols: str) -> str: r""" From e3f3b2419cb5f090a221c1922ae72bcc3c36e320 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 20 Apr 2021 18:09:26 -0500 Subject: [PATCH 40/90] 100% pytest coverage --- tests/test_commands_yaml_get.py | 31 ++++++++++++++++++ tests/test_common_keywordsearches.py | 46 ++++++++++++++++++++++++++- tests/test_common_parsers.py | 13 +++++++- tests/test_wrappers_consoleprinter.py | 13 ++++++-- tests/test_yamlpath.py | 22 +++++++++++++ yamlpath/common/parsers.py | 7 ---- yamlpath/yamlpath.py | 16 ++++------ 7 files changed, 127 insertions(+), 21 deletions(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index 7c320886..46120982 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -253,3 +253,34 @@ def test_get_only_aoh_nodes_with_named_child(self, script_runner, tmp_path_facto for line in result.stdout.splitlines(): assert line == results[match_index] match_index += 1 + + @pytest.mark.parametrize("query,output", [ + ("/items/*[!has_child(bravo)][2][parent(0)]", ['delta']), + ("/items/*[!has_child(bravo)][2][parent()]", ['["alpha", "charlie", "delta"]']), + ("/items/*[!has_child(bravo)][2][parent(2)]", ['[["alpha", "bravo", "charlie"], ["alpha", "charlie", "delta"], ["alpha", "bravo", "delta"], ["bravo", "charlie", "delta"]]']), + ]) + def test_get_parent_nodes(self, script_runner, tmp_path_factory, query, output): + content = """--- +items: + - - alpha + - bravo + - charlie + - - alpha + - charlie + - delta + - - alpha + - bravo + - delta + - - bravo + - charlie + - delta +""" + + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run(self.command, "--query={}".format(query), yaml_file) + assert result.success, result.stderr + + match_index = 0 + for line in result.stdout.splitlines(): + assert line == output[match_index] + match_index += 1 diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py index d7cf3dc6..e87a455c 100644 --- a/tests/test_common_keywordsearches.py +++ b/tests/test_common_keywordsearches.py @@ -29,7 +29,7 @@ def test_unknown_search_keyword(self): def test_has_child_invalid_param_count(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.search_matches( - SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, ""), + SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, []), {}, YAMLPath("/") )) @@ -42,3 +42,47 @@ def test_has_child_invalid_node(self): ["wwk"], YAMLPath("") )) + + + ### + # parent + ### + def test_parent_invalid_param_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.parent( + {}, + False, + ["1", "2"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Invalid parameter count to ") + + def test_parent_invalid_inversion(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.parent( + {}, + True, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Inversion is meaningless to ") + + def test_parent_invalid_parameter(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.parent( + {}, + False, + ["abc"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Invalid parameter passed to ") + + def test_parent_invalid_step_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.parent( + {}, + False, + ["5"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("higher than the document root") diff --git a/tests/test_common_parsers.py b/tests/test_common_parsers.py index a4776abe..7cdb59c0 100644 --- a/tests/test_common_parsers.py +++ b/tests/test_common_parsers.py @@ -78,7 +78,7 @@ def test_stringify_complex_data_with_dates(self): ### # jsonify_yaml_data ### - def test_jsonify_complex_data(self): + def test_jsonify_complex_ruamel_data(self): tagged_tag = "!tagged" tagged_value = "tagged value" tagged_scalar = ry.scalarstring.PlainScalarString(tagged_value) @@ -101,3 +101,14 @@ def test_jsonify_complex_data(self): assert jdata["null"] == null_value assert jdata["dates"][0] == "2020-10-31" assert jdata["dates"][1] == "2020-11-03" + + def test_jsonify_complex_python_data(self): + cdata = { + "dates": [ + dt.date(2020, 10, 31), + dt.date(2020, 11, 3) + ] + } + jdata = Parsers.jsonify_yaml_data(cdata) + assert jdata["dates"][0] == "2020-10-31" + assert jdata["dates"][1] == "2020-11-03" diff --git a/tests/test_wrappers_consoleprinter.py b/tests/test_wrappers_consoleprinter.py index e6590ffc..88c20bac 100644 --- a/tests/test_wrappers_consoleprinter.py +++ b/tests/test_wrappers_consoleprinter.py @@ -162,16 +162,25 @@ def test_debug_noisy(self, capsys): "DEBUG: [tagged_array][1]b", ]) - nc = NodeCoords("value", dict(key="value"), "key", YAMLPath("key")) + nc = NodeCoords( + "value", dict(key="value"), "key", YAMLPath("doc_root.key"), + [ + (dict(doc_root=dict(key="value")), "doc_root"), + (dict(key="value"), "key"), + ]) logger.debug( "A node coordinate:", prefix="test_debug_noisy: ", data=nc) console = capsys.readouterr() assert "\n".join([ "DEBUG: test_debug_noisy: A node coordinate:", - "DEBUG: test_debug_noisy: (path)key", + "DEBUG: test_debug_noisy: (path)doc_root.key", "DEBUG: test_debug_noisy: (node)value", "DEBUG: test_debug_noisy: (parent)[key]value", "DEBUG: test_debug_noisy: (parentref)key", + "DEBUG: test_debug_noisy: (ancestry)[0][0][doc_root][key]value", + "DEBUG: test_debug_noisy: (ancestry)[0][1]doc_root", + "DEBUG: test_debug_noisy: (ancestry)[1][0][key]value", + "DEBUG: test_debug_noisy: (ancestry)[1][1]key", ]) + "\n" == console.out logger.debug(foldedval) diff --git a/tests/test_yamlpath.py b/tests/test_yamlpath.py index 8351f062..ecae5b87 100644 --- a/tests/test_yamlpath.py +++ b/tests/test_yamlpath.py @@ -196,3 +196,25 @@ def test_parse_unknown_search_keyword(self): with pytest.raises(YAMLPathException) as ex: str(YAMLPath("abc[unknown_keyword()]")) assert -1 < str(ex.value).find("Unknown search keyword, unknown_keyword") + + @pytest.mark.parametrize("path,pops,results", [ + ("/abc", 1, [(PathSegmentTypes.KEY, "abc")]), + ("abc", 1, [(PathSegmentTypes.KEY, "abc")]), + ("/abc/def", 2, [(PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ("abc.def", 2, [(PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ("/abc/def[3]", 3, [(PathSegmentTypes.INDEX, 3), (PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ("abc.def[3]", 3, [(PathSegmentTypes.INDEX, 3), (PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ("/abc/def[3][1]", 4, [(PathSegmentTypes.INDEX, 1), (PathSegmentTypes.INDEX, 3), (PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ("abc.def[3][1]", 4, [(PathSegmentTypes.INDEX, 1), (PathSegmentTypes.INDEX, 3), (PathSegmentTypes.KEY, "def"), (PathSegmentTypes.KEY, "abc")]), + ]) + def test_pop_segments(self, path, pops, results): + yp = YAMLPath(path) + for pop in range(pops): + assert results[pop] == yp.pop() + + def test_pop_too_many(self): + yp = YAMLPath("abc.def") + with pytest.raises(YAMLPathException) as ex: + for _ in range(5): + yp.pop() + assert -1 < str(ex.value).find("Cannot pop when") diff --git a/yamlpath/common/parsers.py b/yamlpath/common/parsers.py index 260abacd..a6c37799 100644 --- a/yamlpath/common/parsers.py +++ b/yamlpath/common/parsers.py @@ -318,13 +318,6 @@ def jsonify_yaml_data(data: Any) -> Any: for key, val in data.items(): data[key] = Parsers.jsonify_yaml_data(val) elif isinstance(data, dict): - for i, k in [ - (idx, key) for idx, key in enumerate(data.keys()) - if isinstance(key, TaggedScalar) - ]: - unwrapped_key = Parsers.jsonify_yaml_data(k) - data[unwrapped_key] = data.pop(k) - for key, val in data.items(): data[key] = Parsers.jsonify_yaml_data(val) elif isinstance(data, (list, CommentedSeq)): diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 69057502..16ee8ada 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -153,22 +153,18 @@ def pop(self) -> PathSegment: self.original = path_now[0:len(path_now) - len(prefixed_segment)] elif path_now.endswith(removable_segment): self.original = path_now[0:len(path_now) - len(removable_segment)] - elif ( - self.seperator == PathSeperators.FSLASH - and path_now.endswith(prefixed_segment[1:]) - ): - self.original = path_now[ - 0:len(path_now) - len(prefixed_segment) + 1] elif ( self.seperator == PathSeperators.FSLASH and path_now.endswith(removable_segment[1:]) ): self.original = path_now[ 0:len(path_now) - len(removable_segment) + 1] - else: - raise YAMLPathException( - "Unable to pop unmatchable segment, {}" - .format(removable_segment), str(self)) + + # I cannot come up with a test that would trigger this Exception: + # else: + # raise YAMLPathException( + # "Unable to pop unmatchable segment, {}" + # .format(removable_segment), str(self)) return popped_segment From 7ffeb686c2c58d84c228e4e2894e46b6f79e5e45 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 20 Apr 2021 18:43:22 -0500 Subject: [PATCH 41/90] Added immediate objectives for 3.5.0 --- CHANGES | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGES b/CHANGES index bb4d5bb2..6462a306 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,22 @@ +3.5.0: +Bug Fixes: +* Some Python-generated complex data types were escaping JSONification, + leading to unexpected stack-dumps when writing out JSON data for data types + like date and datetime. + +Enhancements: +* An entirely new segment type has been added to YAML Path and is now supported + by the reference implementation command-line tools: Keyword Searches. + Similar to programming language keywords, these reserved Keywords work much + like functions, accepting parameters and performing algorythmic operations or + returning data not otherwise accessible to other YAML Path segment types. + These new capabilities -- explored on the project Wiki -- include: + * [has_child(NAME)] + * [name()] + * [max(NAME)] + * [min(NAME)] + * [parent([STEPS])] + 3.4.1: Bug Fixes: * yaml-set (and the underlying Processor class) were unable to change nodes From 21511392f0d77535e5e7576ef60adbfb4b9ed228 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Tue, 20 Apr 2021 19:04:05 -0500 Subject: [PATCH 42/90] Initial version of kw name() --- yamlpath/common/keywordsearches.py | 36 ++++++++++++++++++++++++++++ yamlpath/enums/pathsearchkeywords.py | 9 ++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 7fce0427..6362c6ef 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -32,6 +32,9 @@ def search_matches( if keyword is PathSearchKeywords.HAS_CHILD: nc_matches = KeywordSearches.has_child( haystack, invert, parameters, yaml_path, **kwargs) + elif keyword is PathSearchKeywords.NAME: + nc_matches = KeywordSearches.name( + haystack, invert, parameters, yaml_path, **kwargs) elif keyword is PathSearchKeywords.PARENT: nc_matches = KeywordSearches.parent( haystack, invert, parameters, yaml_path, **kwargs) @@ -107,6 +110,39 @@ def has_child( ("{} data has no child nodes in YAML Path").format(type(data)), str(yaml_path)) + @staticmethod + # pylint: disable=locally-disabled,too-many-locals + def name( + data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """Match only the key-name of the present node.""" + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) + + # There are no parameters + param_count = len(parameters) + if param_count > 1: + raise YAMLPathException(( + "Invalid parameter count to {}(); {} are permitted, " + " got {} in YAML Path" + ).format(PathSearchKeywords.NAME, 0, param_count), + str(yaml_path)) + + if invert: + raise YAMLPathException(( + "Inversion is meaningless to {}()" + ).format(PathSearchKeywords.NAME), + str(yaml_path)) + + # parent_node = KeywordSearches.parent( + # data, False, [], yaml_path, **kwargs) + + yield NodeCoords( + parentref, parent, parentref, translated_path, ancestry) + @staticmethod # pylint: disable=locally-disabled,too-many-locals def parent( diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index 56850499..d085d28e 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -15,11 +15,16 @@ class PathSearchKeywords(Enum): `HAS_CHILD` Matches when the node has a direct child with a given name. + `NAME` + Matches only the key-name of the present node, discarding any and all + child node data. Can be used to rename the matched key as long as the + new name is unique within the parent. `PARENT` - Access the parent of the present node. + Access the parent(s) of the present node. """ HAS_CHILD = auto() + NAME = () PARENT = auto() def __str__(self) -> str: @@ -27,6 +32,8 @@ def __str__(self) -> str: keyword = '' if self is PathSearchKeywords.HAS_CHILD: keyword = 'has_child' + elif self is PathSearchKeywords.NAME: + keyword = 'name' elif self is PathSearchKeywords.PARENT: keyword = 'parent' From 6a42483d154043d972d833a459946b33e9591248 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 15:04:27 -0500 Subject: [PATCH 43/90] Descendent searches match ANY, not FIRST --- yamlpath/processor.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index d8e9f325..581f33e8 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1003,6 +1003,9 @@ def _get_nodes_by_search( elif isinstance(data, dict): # Allow . to mean "each key's name" if attr == '.': + self.logger.debug( + "Scanning every key's name...", + prefix="Processor::_get_nodes_by_search: ") for key, val in data.items(): matches = Searches.search_matches(method, term, key) if (matches and not invert) or (invert and not matches): @@ -1021,6 +1024,10 @@ def _get_nodes_by_search( elif attr in data: value = data[attr] matches = Searches.search_matches(method, term, value) + self.logger.debug( + "Scanning for an attribute match against {}, which {}." + .format(attr, "matches" if matches else "does not match"), + prefix="Processor::_get_nodes_by_search: ") if (matches and not invert) or (invert and not matches): debug_matched = "one dictionary attribute match yielded" self.logger.debug( @@ -1035,14 +1042,31 @@ def _get_nodes_by_search( ancestry + [(data, attr)]) else: - # Attempt a descendant search + # Attempt a descendant search; return every node which has ANY + # descendent matching the search expression. + self.logger.debug(( + "Attempting a descendant search against data at" + " desc_path={}, translated_path={}:" + ).format(desc_path, translated_path), + prefix="Processor::_get_nodes_by_search: ", + data=data) for desc_node in self._get_required_nodes( data, desc_path, 0, parent=parent, parentref=parentref, translated_path=translated_path, ancestry=ancestry ): matches = Searches.search_matches( method, term, desc_node.node) - break + + if (matches and not invert) or (invert and not matches): + # Search no further because the parent node of this + # search has at least one matching descendent. + self.logger.debug(( + "BREAKING OUT of descendent search with matches={}" + " and invert={}").format( + "matching" if matches else "NOT matching", + "yes" if invert else "no"), + prefix="Processor::_get_nodes_by_search: ") + break if (matches and not invert) or (invert and not matches): debug_matched = "one descendant search match yielded" From 4ec77e685ac1d528dd9907bdb6eab589b879db42 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 15:32:07 -0500 Subject: [PATCH 44/90] Update 3.5.0 CHANGES to match work effort --- CHANGES | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES b/CHANGES index 6462a306..6c83f47d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,11 @@ 3.5.0: Bug Fixes: +* Search expressions against Boolean values, [key=True] and [key=False], were + impossible. Now, they are working and are not case-sensitive, so [key=True], + [key=true], [key=TRUE], and such all work as expected. +* Descendent searches were considering only the first child of the search + ancestor. Now, ANY matching descendent node will correctly yield the + ancestor. * Some Python-generated complex data types were escaping JSONification, leading to unexpected stack-dumps when writing out JSON data for data types like date and datetime. @@ -16,6 +22,10 @@ Enhancements: * [max(NAME)] * [min(NAME)] * [parent([STEPS])] +* When stringified, YAML Paths with a solitary * wildcard segment were printed + using their internal RegEx variant, [.=~/.*/]. They are now printed as they + are entered, using a solitary *. As a consequence, any deliberate RegEx of + [.=~/.*/] is also printed as its equivalent solitary *. 3.4.1: Bug Fixes: From 824af3ab6a50cd0776c69da179fed8bdc80e4859 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 15:35:35 -0500 Subject: [PATCH 45/90] Clarity about lib employment of Search Keywords --- CHANGES | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 6c83f47d..385d0eef 100644 --- a/CHANGES +++ b/CHANGES @@ -12,11 +12,12 @@ Bug Fixes: Enhancements: * An entirely new segment type has been added to YAML Path and is now supported - by the reference implementation command-line tools: Keyword Searches. - Similar to programming language keywords, these reserved Keywords work much - like functions, accepting parameters and performing algorythmic operations or - returning data not otherwise accessible to other YAML Path segment types. - These new capabilities -- explored on the project Wiki -- include: + by the library and reference implementation command-line tools: Keyword + Searches. Similar to programming language keywords, these reserved Keywords + work much like functions, accepting parameters and performing algorythmic + operations or returning data not otherwise accessible to other YAML Path + segment types. These new capabilities -- explored on the project Wiki -- + include: * [has_child(NAME)] * [name()] * [max(NAME)] From 2eb03c922f8cb1ddaed415efa3c77e1436129e31 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 16:04:58 -0500 Subject: [PATCH 46/90] Enable root node ancestry --- yamlpath/common/keywordsearches.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 6362c6ef..8644e858 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -137,9 +137,6 @@ def name( ).format(PathSearchKeywords.NAME), str(yaml_path)) - # parent_node = KeywordSearches.parent( - # data, False, [], yaml_path, **kwargs) - yield NodeCoords( parentref, parent, parentref, translated_path, ancestry) @@ -201,8 +198,8 @@ def parent( (data, _) = ancestry.pop() ancestry_len -= 1 - parentref = ancestry[-1][1] if ancestry_len > 1 else None - parent = ancestry[-1][0] if ancestry_len > 1 else None + parentref = ancestry[-1][1] if ancestry_len > 0 else None + parent = ancestry[-1][0] if ancestry_len > 0 else None parent_nc = NodeCoords( data, parent, parentref, translated_path, ancestry) From 445e15dc249ecd9c9e847ab44989720770ee55d1 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 16:20:07 -0500 Subject: [PATCH 47/90] name() doesn't need haystack --- yamlpath/common/keywordsearches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 8644e858..5529e4c7 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -34,7 +34,7 @@ def search_matches( haystack, invert, parameters, yaml_path, **kwargs) elif keyword is PathSearchKeywords.NAME: nc_matches = KeywordSearches.name( - haystack, invert, parameters, yaml_path, **kwargs) + invert, parameters, yaml_path, **kwargs) elif keyword is PathSearchKeywords.PARENT: nc_matches = KeywordSearches.parent( haystack, invert, parameters, yaml_path, **kwargs) @@ -113,7 +113,7 @@ def has_child( @staticmethod # pylint: disable=locally-disabled,too-many-locals def name( - data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: """Match only the key-name of the present node.""" From 75147a7eb59ef5aadd54a0165fb04a8b64b16c1b Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 16:37:11 -0500 Subject: [PATCH 48/90] Add exception tests for kw name() --- tests/test_common_keywordsearches.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py index e87a455c..12479649 100644 --- a/tests/test_common_keywordsearches.py +++ b/tests/test_common_keywordsearches.py @@ -21,6 +21,7 @@ def test_unknown_search_keyword(self): {}, YAMLPath("/") )) + assert -1 < str(ex.value).find("Unsupported search keyword") ### @@ -33,6 +34,7 @@ def test_has_child_invalid_param_count(self): {}, YAMLPath("/") )) + assert -1 < str(ex.value).find("Invalid parameter count to ") def test_has_child_invalid_node(self): with pytest.raises(YAMLPathException) as ex: @@ -42,6 +44,29 @@ def test_has_child_invalid_node(self): ["wwk"], YAMLPath("") )) + assert -1 < str(ex.value).find("has no child nodes") + + + ### + # name + ### + def test_name_invalid_param_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.name( + False, + ["1", "2"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Invalid parameter count to ") + + def test_name_invalid_inversion(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.name( + True, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Inversion is meaningless to ") ### From 863a2efd73f325bb2ead517e5f5f18b7b47f2632 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 16:48:15 -0500 Subject: [PATCH 49/90] Add tests for kw name() --- tests/test_commands_yaml_get.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index 46120982..aab58108 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -284,3 +284,35 @@ def test_get_parent_nodes(self, script_runner, tmp_path_factory, query, output): for line in result.stdout.splitlines(): assert line == output[match_index] match_index += 1 + + @pytest.mark.parametrize("query,output", [ + ("svcs.*[name()]", ['coolserver', 'logsender']), + ("svcs.*[enabled=false][parent()][name()]", ['logsender']), + ("svcs[name()]", ['svcs']), + ("indexes[.^Item][name()]", ['0', '1', '3']), + ]) + def test_get_node_names(self, script_runner, tmp_path_factory, query, output): + # Contributed by https://github.com/AndydeCleyre + content = """--- +svcs: + coolserver: + enabled: true + exec: ./coolserver.py + logsender: + enabled: false + exec: remote_syslog -D +indexes: + - Item 1 + - Item 2 + - Disabled 3 + - Item 4 +""" + + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run(self.command, "--query={}".format(query), yaml_file) + assert result.success, result.stderr + + match_index = 0 + for line in result.stdout.splitlines(): + assert line == output[match_index] + match_index += 1 From 7e5c476fbd149ca5295c1b0a83fe2845216e71e5 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 18:43:07 -0500 Subject: [PATCH 50/90] Enable [name()] to rename keys --- yamlpath/common/keywordsearches.py | 24 +++-- yamlpath/processor.py | 152 +++++++++++++++++++--------- yamlpath/wrappers/consoleprinter.py | 6 ++ yamlpath/wrappers/nodecoords.py | 12 ++- 4 files changed, 131 insertions(+), 63 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 5529e4c7..09a784dc 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -8,6 +8,7 @@ """ from typing import Any, Generator, List +from yamlpath.types import PathSegment from yamlpath.enums import PathSearchKeywords from yamlpath.path import SearchKeywordTerms from yamlpath.exceptions import YAMLPathException @@ -57,6 +58,7 @@ def has_child( parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There must be exactly one parameter param_count = len(parameters) @@ -78,7 +80,8 @@ def has_child( (child_present and not invert) ): yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + relay_segment) # Against a list, this will merely require an exact match between # parameters and any list elements. When inverted, every @@ -103,7 +106,8 @@ def has_child( (child_present and not invert) ): yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + relay_segment) else: raise YAMLPathException( @@ -121,6 +125,7 @@ def name( parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There are no parameters param_count = len(parameters) @@ -138,7 +143,8 @@ def name( str(yaml_path)) yield NodeCoords( - parentref, parent, parentref, translated_path, ancestry) + parentref, parent, parentref, translated_path, ancestry, + relay_segment) @staticmethod # pylint: disable=locally-disabled,too-many-locals @@ -151,6 +157,7 @@ def parent( parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There may be 0 or 1 parameters param_count = len(parameters) @@ -191,7 +198,8 @@ def parent( if parent_levels < 1: # parent(0) is the present node yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + relay_segment) else: for _ in range(parent_levels): translated_path.pop() @@ -200,8 +208,6 @@ def parent( parentref = ancestry[-1][1] if ancestry_len > 0 else None parent = ancestry[-1][0] if ancestry_len > 0 else None - - parent_nc = NodeCoords( - data, parent, parentref, translated_path, ancestry) - - yield parent_nc + yield NodeCoords( + data, parent, parentref, translated_path, ancestry, + relay_segment) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 581f33e8..14935904 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -6,6 +6,9 @@ """ from typing import Any, Dict, Generator, List, Union +from ruamel.yaml.comments import CommentedMap + +from yamlpath.types import PathSegment from yamlpath.common import Anchors, KeywordSearches, Nodes, Searches from yamlpath import YAMLPath from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms @@ -14,6 +17,7 @@ from yamlpath.enums import ( YAMLValueFormats, PathSegmentTypes, + PathSearchKeywords, CollectorOperators, PathSeperators, ) @@ -188,6 +192,31 @@ def set_value(self, yaml_path: Union[YAMLPath, str], "Setting its value with format {} to:".format(value_format) , data=value , prefix="Processor::set_value: ") + + last_segment = node_coord.path_segment + if last_segment is not None: + (_, segment_value) = last_segment + if ( + isinstance(segment_value, SearchKeywordTerms) + and segment_value.keyword is PathSearchKeywords.NAME + ): + # Rename a key + parent = node_coord.parent + parentref = node_coord.parentref + if isinstance(parent, CommentedMap): + for i, k in [ + (idx, key) for idx, key + in enumerate(parent.keys()) + if key == parentref + ]: + parent.insert(i, value, parent.pop(k)) + else: + raise YAMLPathException(( + "Keys can be renamed only in Hash/map/dict" + " data; got a {}, instead." + ).format(type(parent)), str(yaml_path)) + return + try: self._update_node( node_coord.parent, node_coord.parentref, value, @@ -587,8 +616,9 @@ def _get_nodes_by_path_segment(self, data: Any, data=segments) return + pathseg: PathSegment = yaml_path.unescaped[segment_index] + (unesc_type, unesc_attrs) = pathseg (segment_type, stripped_attrs) = segments[segment_index] - (unesc_type, unesc_attrs) = yaml_path.unescaped[segment_index] # Disallow traversal recursion (because it creates a denial-of-service) if segment_index > 0 and segment_type == PathSegmentTypes.TRAVERSE: @@ -620,7 +650,8 @@ def _get_nodes_by_path_segment(self, data: Any, node_coords = self._get_nodes_by_keyword_search( data, yaml_path, stripped_attrs, parent=parent, parentref=parentref, traverse_lists=traverse_lists, - translated_path=translated_path, ancestry=ancestry) + translated_path=translated_path, ancestry=ancestry, + relay_segment=pathseg) elif ( segment_type == PathSegmentTypes.SEARCH and isinstance(stripped_attrs, SearchTerms) @@ -676,7 +707,8 @@ def _get_nodes_by_key( translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) - (_, stripped_attrs) = yaml_path.escaped[segment_index] + pathseg: PathSegment = yaml_path.escaped[segment_index] + (_, stripped_attrs) = pathseg str_stripped = str(stripped_attrs) self.logger.debug( @@ -695,7 +727,7 @@ def _get_nodes_by_key( .format(str_stripped)) yield NodeCoords( data[stripped_attrs], data, stripped_attrs, - next_translated_path, next_ancestry) + next_translated_path, next_ancestry, pathseg) else: # Check for a string/int type mismatch try: @@ -703,7 +735,7 @@ def _get_nodes_by_key( if intkey in data: yield NodeCoords( data[intkey], data, intkey, next_translated_path, - ancestry + [(data, intkey)]) + ancestry + [(data, intkey)], pathseg) except ValueError: pass elif isinstance(data, list): @@ -718,7 +750,7 @@ def _get_nodes_by_key( yield NodeCoords( data[idx], data, idx, translated_path + "[{}]".format(idx), - ancestry + [(data, idx)]) + ancestry + [(data, idx)], pathseg) except ValueError: # Pass-through search against possible Array-of-Hashes, if # allowed. @@ -764,7 +796,8 @@ def _get_nodes_by_index( Raises: N/A """ - (_, stripped_attrs) = yaml_path.escaped[segment_index] + pathseg: PathSegment = yaml_path.escaped[segment_index] + (_, stripped_attrs) = pathseg (_, unstripped_attrs) = yaml_path.unescaped[segment_index] str_stripped = str(stripped_attrs) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) @@ -795,18 +828,18 @@ def _get_nodes_by_index( yield NodeCoords( [data[intmin]], data, intmin, translated_path + "[{}]".format(intmin), - ancestry + [(data, intmin)]) + ancestry + [(data, intmin)], pathseg) else: sliced_elements = [] for slice_index in range(intmin, intmax): sliced_elements.append(NodeCoords( data[slice_index], data, intmin, translated_path + "[{}]".format(slice_index), - ancestry + [(data, slice_index)])) + ancestry + [(data, slice_index)], pathseg)) yield NodeCoords( sliced_elements, data, intmin, translated_path + "[{}:{}]".format(intmin, intmax), - ancestry + [(data, intmin)]) + ancestry + [(data, intmin)], pathseg) elif isinstance(data, dict): for key, val in data.items(): @@ -815,7 +848,7 @@ def _get_nodes_by_index( val, data, key, translated_path + YAMLPath.escape_path_section( key, translated_path.seperator), - ancestry + [(data, key)]) + ancestry + [(data, key)], pathseg) else: try: idx: int = int(str_stripped) @@ -830,7 +863,7 @@ def _get_nodes_by_index( if isinstance(data, list) and len(data) > idx: yield NodeCoords( data[idx], data, idx, translated_path + "[{}]".format(idx), - ancestry + [(data, idx)]) + ancestry + [(data, idx)], pathseg) def _get_nodes_by_anchor( self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs @@ -851,7 +884,8 @@ def _get_nodes_by_anchor( Raises: N/A """ - (_, stripped_attrs) = yaml_path.escaped[segment_index] + pathseg: PathSegment = yaml_path.escaped[segment_index] + (_, stripped_attrs) = pathseg translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) next_translated_path = translated_path + "[&{}]".format( @@ -867,7 +901,7 @@ def _get_nodes_by_anchor( if (hasattr(ele, "anchor") and stripped_attrs == ele.anchor.value): yield NodeCoords(ele, data, lstidx, next_translated_path, - ancestry + [(data, lstidx)]) + ancestry + [(data, lstidx)], pathseg) elif isinstance(data, dict): for key, val in data.items(): next_ancestry = ancestry + [(data, key)] @@ -875,12 +909,12 @@ def _get_nodes_by_anchor( and stripped_attrs == key.anchor.value): yield NodeCoords( val, data, key, next_translated_path, - next_ancestry) + next_ancestry, pathseg) elif (hasattr(val, "anchor") and stripped_attrs == val.anchor.value): yield NodeCoords( val, data, key, next_translated_path, - next_ancestry) + next_ancestry, pathseg) def _get_nodes_by_keyword_search( self, data: Any, yaml_path: YAMLPath, terms: SearchKeywordTerms, @@ -955,6 +989,7 @@ def _get_nodes_by_search( parentref: Any = kwargs.pop("parentref", None) traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + pathseg: PathSegment = (PathSegmentTypes.SEARCH, terms) ancestry: List[tuple] = kwargs.pop("ancestry", []) invert = terms.inverted method = terms.method @@ -983,7 +1018,7 @@ def _get_nodes_by_search( for desc_node in self._get_required_nodes( ele, desc_path, 0, translated_path=next_translated_path, - ancestry=next_ancestry + ancestry=next_ancestry, relay_segment=pathseg ): matches = Searches.search_matches( method, term, desc_node.node) @@ -998,7 +1033,7 @@ def _get_nodes_by_search( yield NodeCoords( ele, data, lstidx, translated_path + "[{}]".format(lstidx), - ancestry + [(data, lstidx)]) + ancestry + [(data, lstidx)], pathseg) elif isinstance(data, dict): # Allow . to mean "each key's name" @@ -1019,7 +1054,7 @@ def _get_nodes_by_search( val, data, key, translated_path + YAMLPath.escape_path_section( key, translated_path.seperator), - ancestry + [(data, key)]) + ancestry + [(data, key)], pathseg) elif attr in data: value = data[attr] @@ -1039,7 +1074,7 @@ def _get_nodes_by_search( value, data, attr, translated_path + YAMLPath.escape_path_section( attr, translated_path.seperator), - ancestry + [(data, attr)]) + ancestry + [(data, attr)], pathseg) else: # Attempt a descendant search; return every node which has ANY @@ -1052,7 +1087,8 @@ def _get_nodes_by_search( data=data) for desc_node in self._get_required_nodes( data, desc_path, 0, parent=parent, parentref=parentref, - translated_path=translated_path, ancestry=ancestry + translated_path=translated_path, ancestry=ancestry, + relay_segment=pathseg ): matches = Searches.search_matches( method, term, desc_node.node) @@ -1076,7 +1112,8 @@ def _get_nodes_by_search( data=data, prefix="Processor::_get_nodes_by_search: ") yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + pathseg) else: # Check the passed data itself for a match @@ -1087,7 +1124,8 @@ def _get_nodes_by_search( "Yielding the queried data itself because it matches.", prefix="Processor::_get_nodes_by_search: ") yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + pathseg) self.logger.debug( "Finished seeking SEARCH nodes matching {} in data with {}:" @@ -1131,14 +1169,18 @@ def _get_nodes_by_collector( parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) - node_coords = [] # A list of NodeCoords + node_coords: List[NodeCoords] = [] + segments = yaml_path.escaped + next_segment_idx = segment_index + 1 + pathseg: PathSegment = segments[segment_index] + self.logger.debug( "Processor::_get_nodes_by_collector: Getting required nodes" " matching search expression: {}".format(terms.expression)) for node_coord in self._get_required_nodes( data, YAMLPath(terms.expression), 0, parent=parent, parentref=parentref, translated_path=translated_path, - ancestry=ancestry): + ancestry=ancestry, relay_segment=pathseg): node_coords.append(node_coord) # This may end up being a bad idea for some cases, but this method will @@ -1159,17 +1201,15 @@ def _get_nodes_by_collector( flat_nodes.append( NodeCoords( flatten_node, node_coord.parent, flatten_idx, - node_coord.path, node_coord.ancestry)) + node_coord.path, node_coord.ancestry, pathseg)) node_coords = flat_nodes # As long as each next segment is an ADDITION or SUBTRACTION # COLLECTOR, keep combining the results. - segments = yaml_path.escaped - next_segment_idx = segment_index + 1 - # pylint: disable=too-many-nested-blocks while next_segment_idx < len(segments): - (peek_type, peek_attrs) = segments[next_segment_idx] + peekseg: PathSegment = segments[next_segment_idx] + (peek_type, peek_attrs) = peekseg if ( peek_type is PathSegmentTypes.COLLECTOR and isinstance(peek_attrs, CollectorTerms) @@ -1180,7 +1220,7 @@ def _get_nodes_by_collector( data, peek_path, 0, parent=parent, parentref=parentref, translated_path=translated_path, - ancestry=ancestry): + ancestry=ancestry, relay_segment=peekseg): if (isinstance(node_coord, NodeCoords) and isinstance(node_coord.node, list)): for coord_idx, coord in enumerate(node_coord.node): @@ -1195,7 +1235,7 @@ def _get_nodes_by_collector( coord = NodeCoords( coord, node_coord.node, coord_idx, next_translated_path, - next_ancestry) + next_ancestry, peekseg) node_coords.append(coord) else: node_coords.append(node_coord) @@ -1205,7 +1245,7 @@ def _get_nodes_by_collector( data, peek_path, 0, parent=parent, parentref=parentref, translated_path=translated_path, - ancestry=ancestry): + ancestry=ancestry, relay_segment=peekseg): unwrapped_data = NodeCoords.unwrap_node_coords( node_coord) if isinstance(unwrapped_data, list): @@ -1257,6 +1297,9 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + segments = yaml_path.escaped + pathseg: PathSegment = segments[segment_index] + next_segment_idx: int = segment_index + 1 self.logger.debug( "TRAVERSING the tree at parentref:", @@ -1265,12 +1308,11 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, if data is None: self.logger.debug( "Processor::_get_nodes_by_traversal: Yielding a None node.") - yield NodeCoords(None, parent, parentref) + yield NodeCoords(None, parent, parentref, pathseg) return # Is there a next segment? - segments = yaml_path.escaped - if segment_index + 1 == len(segments): + if next_segment_idx == len(segments): # This traversal is gathering every leaf node if isinstance(data, dict): for key, val in data.items(): @@ -1307,7 +1349,8 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, "Yielding unfiltered Scalar value:", prefix="Processor::_get_nodes_by_traversal: ", data=data) yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + pathseg) else: # There is a filter in the next segment; recurse data, comparing # every child against the following segment until there are no more @@ -1320,8 +1363,10 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, self.logger.debug( "Processor::_get_nodes_by_traversal: Checking the DIRECT node" " for a next-segment match at {}...".format(parentref)) + + peekseg: PathSegment = segments[next_segment_idx] for node_coord in self._get_nodes_by_path_segment( - data, yaml_path, segment_index + 1, parent=parent, + data, yaml_path, next_segment_idx, parent=parent, parentref=parentref, traverse_lists=False, translated_path=translated_path, ancestry=ancestry ): @@ -1331,7 +1376,8 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, prefix="Processor::_get_nodes_by_traversal: ", data=node_coord) yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + peekseg) # Then, recurse into each child to perform the same test. if isinstance(data, dict): @@ -1400,6 +1446,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) if data is None: self.logger.debug( @@ -1411,7 +1458,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, segments = yaml_path.escaped if segments and len(segments) > depth: - (segment_type, unstripped_attrs) = yaml_path.unescaped[depth] + pathseg: PathSegment = yaml_path.unescaped[depth] + (segment_type, unstripped_attrs) = pathseg except_segment = str(unstripped_attrs) self.logger.debug( "Seeking segment <{}>{} in data of type {}:" @@ -1453,7 +1501,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, for subnode_coord in self._get_required_nodes( segment_node_coords, yaml_path, depth + 1, translated_path=translated_path, - ancestry=ancestry): + ancestry=ancestry, relay_segment=pathseg): yield subnode_coord else: self.logger.debug( @@ -1464,7 +1512,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, parent=segment_node_coords.parent, parentref=segment_node_coords.parentref, translated_path=segment_node_coords.path, - ancestry=segment_node_coords.ancestry): + ancestry=segment_node_coords.ancestry, + relay_segment=pathseg): self.logger.debug( "Finally returning segment data of type {} at" " parentref {}:" @@ -1480,7 +1529,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, prefix="Processor::_get_required_nodes: ", data=data, footer=" ") yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + relay_segment) # pylint: disable=locally-disabled,too-many-statements def _get_optional_nodes( @@ -1516,11 +1566,13 @@ def _get_optional_nodes( parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) segments = yaml_path.escaped # pylint: disable=locally-disabled,too-many-nested-blocks if segments and len(segments) > depth: - (segment_type, unstripped_attrs) = yaml_path.unescaped[depth] + pathseg: PathSegment = yaml_path.unescaped[depth] + (segment_type, unstripped_attrs) = pathseg stripped_attrs: Union[ str, int, @@ -1552,7 +1604,8 @@ def _get_optional_nodes( parent=next_coord.parent, parentref=next_coord.parentref, translated_path=next_coord.path, - ancestry=next_coord.ancestry + ancestry=next_coord.ancestry, + relay_segment=pathseg ): yield node_coord @@ -1591,7 +1644,7 @@ def _get_optional_nodes( new_ele, yaml_path, value, depth + 1, parent=data, parentref=new_idx, translated_path=next_translated_path, - ancestry=next_ancestry + ancestry=next_ancestry, relay_segment=pathseg ): matched_nodes += 1 yield node_coord @@ -1625,7 +1678,7 @@ def _get_optional_nodes( data[newidx], yaml_path, value, depth + 1, parent=data, parentref=newidx, translated_path=next_translated_path, - ancestry=next_ancestry + ancestry=next_ancestry, relay_segment=pathseg ): matched_nodes += 1 yield node_coord @@ -1661,7 +1714,7 @@ def _get_optional_nodes( depth + 1, parent=data, parentref=stripped_attrs, translated_path=next_translated_path, - ancestry=next_ancestry + ancestry=next_ancestry, relay_segment=pathseg ): matched_nodes += 1 yield node_coord @@ -1696,7 +1749,8 @@ def _get_optional_nodes( .format(type(data)), prefix="Processor::_get_optional_nodes: ", data=data) yield NodeCoords( - data, parent, parentref, translated_path, ancestry) + data, parent, parentref, translated_path, ancestry, + relay_segment) # pylint: disable=too-many-arguments def _update_node( diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 93ed4f8f..b21283ae 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -288,6 +288,7 @@ def _debug_node_coord( """Helper method for debug.""" prefix = kwargs.pop("prefix", "") path_prefix = "{}(path)".format(prefix) + segment_prefix = "{}(segment)".format(prefix) node_prefix = "{}(node)".format(prefix) parent_prefix = "{}(parent)".format(prefix) parentref_prefix = "{}(parentref)".format(prefix) @@ -296,6 +297,11 @@ def _debug_node_coord( for line in ConsolePrinter._debug_dump(data.path, prefix=path_prefix): yield line + for line in ConsolePrinter._debug_dump( + data.path_segment, prefix=segment_prefix + ): + yield line + for line in ConsolePrinter._debug_dump(data.node, prefix=node_prefix): yield line diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index 7057ca3c..da2c7027 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -1,6 +1,7 @@ """Wrap a node along with its relative coordinates within its DOM.""" from typing import Any, List +from yamlpath.types import PathSegment from yamlpath import YAMLPath class NodeCoords: @@ -16,7 +17,7 @@ class NodeCoords: # pylint: disable=locally-disabled,too-many-arguments def __init__( self, node: Any, parent: Any, parentref: Any, path: YAMLPath = None, - ancestry: List[tuple] = None + ancestry: List[tuple] = None, path_segment: PathSegment = None ) -> None: """ Initialize a new NodeCoords. @@ -35,11 +36,12 @@ def __init__( Raises: N/A """ - self.node = node - self.parent = parent - self.parentref = parentref - self.path = path + self.node: Any = node + self.parent: Any = parent + self.parentref: Any = parentref + self.path: YAMLPath = path self.ancestry: List[tuple] = [] if ancestry is None else ancestry + self.path_segment: PathSegment = path_segment def __str__(self) -> str: """Get a String representation of this object.""" From 85f6afe39634acca3656cbe8466661c5883a673d Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 21:20:27 -0500 Subject: [PATCH 51/90] mypy fixes --- yamlpath/processor.py | 98 +++++++++++++++++++-------------- yamlpath/wrappers/nodecoords.py | 6 +- 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 14935904..4874887f 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -184,48 +184,63 @@ def set_value(self, yaml_path: Union[YAMLPath, str], for node_coord in self._get_optional_nodes( self.data, yaml_path, value ): - self.logger.debug( - "Matched optional node coordinate:" - , data=node_coord - , prefix="Processor::set_value: ") - self.logger.debug( - "Setting its value with format {} to:".format(value_format) - , data=value - , prefix="Processor::set_value: ") + self._apply_change(yaml_path, node_coord, value, + value_format=value_format, tag=tag) + + def _apply_change( + self, yaml_path: YAMLPath, node_coord: NodeCoords, value: Any, + **kwargs: Any + ): + """Helper for set_value.""" + value_format: YAMLValueFormats = kwargs.pop("value_format", + YAMLValueFormats.DEFAULT) + tag: str = kwargs.pop("tag", None) - last_segment = node_coord.path_segment - if last_segment is not None: - (_, segment_value) = last_segment - if ( - isinstance(segment_value, SearchKeywordTerms) - and segment_value.keyword is PathSearchKeywords.NAME - ): - # Rename a key - parent = node_coord.parent - parentref = node_coord.parentref - if isinstance(parent, CommentedMap): - for i, k in [ - (idx, key) for idx, key - in enumerate(parent.keys()) - if key == parentref - ]: - parent.insert(i, value, parent.pop(k)) - else: - raise YAMLPathException(( - "Keys can be renamed only in Hash/map/dict" - " data; got a {}, instead." - ).format(type(parent)), str(yaml_path)) - return + self.logger.debug( + "Matched optional node coordinate:" + , data=node_coord + , prefix="Processor::_apply_change: ") + self.logger.debug( + "Setting its value with format {} to:".format(value_format) + , data=value + , prefix="Processor::_apply_change: ") - try: - self._update_node( - node_coord.parent, node_coord.parentref, value, - value_format, tag) - except ValueError as vex: - raise YAMLPathException( - "Impossible to write '{}' as {}. The error was: {}" - .format(value, value_format, str(vex)) - , str(yaml_path)) from vex + last_segment = node_coord.path_segment + if last_segment is not None: + (_, segment_value) = last_segment + if ( + isinstance(segment_value, SearchKeywordTerms) + and segment_value.keyword is PathSearchKeywords.NAME + ): + # Rename a key + parent = node_coord.parent + parentref = node_coord.parentref + if isinstance(parent, CommentedMap): + for i, k in [ + (idx, key) for idx, key + in enumerate(parent.keys()) + if key == parentref + ]: + parent.insert(i, value, parent.pop(k)) + elif isinstance(parent, dict): + parent[value] = parent[parentref] + del parent[parentref] + else: + raise YAMLPathException(( + "Keys can be renamed only in Hash/map/dict" + " data; got a {}, instead." + ).format(type(parent)), str(yaml_path)) + return + + try: + self._update_node( + node_coord.parent, node_coord.parentref, value, + value_format, tag) + except ValueError as vex: + raise YAMLPathException( + "Impossible to write '{}' as {}. The error was: {}" + .format(value, value_format, str(vex)) + , str(yaml_path)) from vex def _get_anchor_node( self, anchor_path: Union[YAMLPath, str], **kwargs: Any @@ -1308,7 +1323,8 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, if data is None: self.logger.debug( "Processor::_get_nodes_by_traversal: Yielding a None node.") - yield NodeCoords(None, parent, parentref, pathseg) + yield NodeCoords(None, parent, parentref, translated_path, + ancestry, pathseg) return # Is there a next segment? diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index da2c7027..ed32e573 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -1,5 +1,5 @@ """Wrap a node along with its relative coordinates within its DOM.""" -from typing import Any, List +from typing import Any, List, Optional from yamlpath.types import PathSegment from yamlpath import YAMLPath @@ -39,9 +39,9 @@ def __init__( self.node: Any = node self.parent: Any = parent self.parentref: Any = parentref - self.path: YAMLPath = path + self.path: Optional[YAMLPath] = path self.ancestry: List[tuple] = [] if ancestry is None else ancestry - self.path_segment: PathSegment = path_segment + self.path_segment: Optional[PathSegment] = path_segment def __str__(self) -> str: """Get a String representation of this object.""" From 144f3b8aec10b1f25de1ffd4b9bb3dc5ede6bf00 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 22:42:16 -0500 Subject: [PATCH 52/90] 100% test coverage --- tests/test_commands_yaml_set.py | 37 +++++++++++++++++++++++++++ tests/test_processor.py | 7 +++++ tests/test_wrappers_consoleprinter.py | 19 +++++++++----- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index 1f51f313..14ee357e 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1302,3 +1302,40 @@ def test_assign_to_nonexistent_nodes(self, script_runner, tmp_path_factory): with open(yaml_file, 'r') as fhnd: filedat = fhnd.read() assert filedat == yamlout + + def test_change_key_name_good(self, script_runner, tmp_path_factory): + yamlin = """--- +key: value +""" + yamlout = """--- +renamed_key: value +""" + yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) + result = script_runner.run( + self.command, + "--change=/key[name()]", + "--value=renamed_key", + yaml_file + ) + assert result.success, result.stderr + + with open(yaml_file, 'r') as fhnd: + filedat = fhnd.read() + assert filedat == yamlout + + def test_change_key_name_bad(self, script_runner, tmp_path_factory): + yamlin = """--- +items: + - one + - two +""" + + yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) + result = script_runner.run( + self.command, + "--change=/items[0][name()]", + "--value=2", + yaml_file + ) + assert not result.success, result.stdout + assert "Keys can be renamed only in Hash/map/dict" in result.stderr diff --git a/tests/test_processor.py b/tests/test_processor.py index 2607c73d..265fb2b8 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -910,3 +910,10 @@ def test_tag_nodes(self, quiet_logger, yaml_path, tag, pathseperator): assert isinstance(data['key'], TaggedScalar) assert data['key'].tag.value == tag + + @pytest.mark.parametrize("yaml_path,value,old_data,new_data", [ + (YAMLPath("/key[name()]"), "renamed_key", {'key': 'value'}, {'renamed_key': 'value'}), + ]) + def test_rename_dict_key(self, quiet_logger, yaml_path, value, old_data, new_data): + processor = Processor(quiet_logger, old_data) + processor.set_value(yaml_path, value) diff --git a/tests/test_wrappers_consoleprinter.py b/tests/test_wrappers_consoleprinter.py index 88c20bac..0d94225d 100644 --- a/tests/test_wrappers_consoleprinter.py +++ b/tests/test_wrappers_consoleprinter.py @@ -5,8 +5,8 @@ from ruamel.yaml.comments import CommentedMap, CommentedSeq, TaggedScalar from ruamel.yaml.scalarstring import PlainScalarString, FoldedScalarString -from yamlpath.wrappers import NodeCoords -from yamlpath.wrappers import ConsolePrinter +from yamlpath.enums import PathSegmentTypes +from yamlpath.wrappers import NodeCoords, ConsolePrinter from yamlpath import YAMLPath class Test_wrappers_ConsolePrinter(): @@ -163,17 +163,22 @@ def test_debug_noisy(self, capsys): ]) nc = NodeCoords( - "value", dict(key="value"), "key", YAMLPath("doc_root.key"), - [ - (dict(doc_root=dict(key="value")), "doc_root"), - (dict(key="value"), "key"), - ]) + "value", + dict(key="value"), + "key", + YAMLPath("doc_root.key"), + [ (dict(doc_root=dict(key="value")), "doc_root"), + (dict(key="value"), "key")], + (PathSegmentTypes.KEY, "key") + ) logger.debug( "A node coordinate:", prefix="test_debug_noisy: ", data=nc) console = capsys.readouterr() assert "\n".join([ "DEBUG: test_debug_noisy: A node coordinate:", "DEBUG: test_debug_noisy: (path)doc_root.key", + "DEBUG: test_debug_noisy: (segment)[0]PathSegmentTypes.KEY", + "DEBUG: test_debug_noisy: (segment)[1]key", "DEBUG: test_debug_noisy: (node)value", "DEBUG: test_debug_noisy: (parent)[key]value", "DEBUG: test_debug_noisy: (parentref)key", From 7e724c24933b520cb85a740e5345de7238e673d5 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Wed, 21 Apr 2021 23:47:24 -0500 Subject: [PATCH 53/90] Let mustexist paths also rename Hash keys --- yamlpath/processor.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 4874887f..b616ba35 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -161,15 +161,8 @@ def set_value(self, yaml_path: Union[YAMLPath, str], found_nodes: int = 0 for req_node in self._get_required_nodes(self.data, yaml_path): found_nodes += 1 - try: - self._update_node( - req_node.parent, req_node.parentref, value, - value_format, tag) - except ValueError as vex: - raise YAMLPathException( - "Impossible to write '{}' as {}. The error was: {}" - .format(value, value_format, str(vex)) - , str(yaml_path)) from vex + self._apply_change(yaml_path, req_node, value, + value_format=value_format, tag=tag) if found_nodes < 1: raise YAMLPathException( From 1d97e0e4f46d524907d08f54c124487d479471a2 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:14:02 -0500 Subject: [PATCH 54/90] Do not recurse into None (null) nodes --- yamlpath/processor.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index b616ba35..8a14326c 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1603,11 +1603,23 @@ def _get_optional_nodes( translated_path=translated_path, ancestry=ancestry ): matched_nodes += 1 - self.logger.debug( - ("Processor::_get_optional_nodes: Found element <{}>{} in" - + " the data; recursing into it..." - ).format(segment_type, except_segment) + if next_coord.node is None: + self.logger.debug(( + "Relaying a None element <{}>{} from the data." + ).format(segment_type, except_segment), + prefix="Processor::_get_optional_nodes: ", + data=next_coord + ) + yield next_coord + continue + + self.logger.debug(( + "Found element <{}>{} in the data; recursing into it..." + ).format(segment_type, except_segment), + prefix="Processor::_get_optional_nodes: ", + data=next_coord ) + for node_coord in self._get_optional_nodes( next_coord.node, yaml_path, value, depth + 1, parent=next_coord.parent, From 3c746e6bfac6a2086201f393de37f19389494132 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:23:21 -0500 Subject: [PATCH 55/90] Don't yield null traversal nodes when filtering --- yamlpath/processor.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 8a14326c..c68abf11 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1313,16 +1313,17 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, "TRAVERSING the tree at parentref:", prefix="Processor::_get_nodes_by_traversal: ", data=parentref) - if data is None: - self.logger.debug( - "Processor::_get_nodes_by_traversal: Yielding a None node.") - yield NodeCoords(None, parent, parentref, translated_path, - ancestry, pathseg) - return - # Is there a next segment? if next_segment_idx == len(segments): # This traversal is gathering every leaf node + if data is None: + self.logger.debug(( + "Yielding a None node."), + prefix="Processor::_get_nodes_by_traversal: ") + yield NodeCoords(None, parent, parentref, translated_path, + ancestry, pathseg) + return + if isinstance(data, dict): for key, val in data.items(): next_translated_path = ( From 354fd192fb061fd8a6effd169f85b7c9019fb40c Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:33:36 -0500 Subject: [PATCH 56/90] Test for null node traversals --- tests/test_processor.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_processor.py b/tests/test_processor.py index 265fb2b8..e9a7f7a4 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -917,3 +917,29 @@ def test_tag_nodes(self, quiet_logger, yaml_path, tag, pathseperator): def test_rename_dict_key(self, quiet_logger, yaml_path, value, old_data, new_data): processor = Processor(quiet_logger, old_data) processor.set_value(yaml_path, value) + + def test_traverse_with_null(self, quiet_logger): + # Contributed by https://github.com/rbordelo + yamldata = """--- +Things: + - name: first thing + rank: 42 + - name: second thing + rank: 5 + - name: third thing + rank: null + - name: fourth thing + rank: 1 +""" + + results = ["first thing", "second thing", "third thing", "fourth thing"] + + yaml = YAML() + data = yaml.load(yamldata) + processor = Processor(quiet_logger, data) + yamlpath = YAMLPath("/**/name") + + match_index = 0 + for node in processor.get_nodes(yamlpath): + assert unwrap_node_coords(node) == results[match_index] + match_index += 1 From 356cd20666c9b83a58f92e3fee1a038413c637af Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:42:55 -0500 Subject: [PATCH 57/90] Update CHANGES for #125 --- CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 385d0eef..0f12cea1 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,10 @@ Bug Fixes: * Search expressions against Boolean values, [key=True] and [key=False], were impossible. Now, they are working and are not case-sensitive, so [key=True], [key=true], [key=TRUE], and such all work as expected. +* When null values were present, Deep Traversal (**) segments would always + return every node with a null value even when they would not match filter + conditions after the ** segment. When mustexist=False, this would also cause + a YAMLPathException. * Descendent searches were considering only the first child of the search ancestor. Now, ANY matching descendent node will correctly yield the ancestor. From f463ac428b44530eda0d1a925af49b508906225a Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:47:43 -0500 Subject: [PATCH 58/90] Mention expansion of NodeCoords --- CHANGES | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES b/CHANGES index 0f12cea1..c437960e 100644 --- a/CHANGES +++ b/CHANGES @@ -31,6 +31,12 @@ Enhancements: using their internal RegEx variant, [.=~/.*/]. They are now printed as they are entered, using a solitary *. As a consequence, any deliberate RegEx of [.=~/.*/] is also printed as its equivalent solitary *. +* [API] The NodeCoords class now tracks ancestry and the last YAML Path segment + responsible for triggering its generation. The ancestry stack was necessary + to support the [parent()] Search Keyword. The responsible YAML Path segment + tracking was necessary to enable Hash/map/dict key renaming via the [name()] + Search Keyword. These must be set when the NodeCoords is generated; it is + not automatic. 3.4.1: Bug Fixes: From bb25d41ed52dd1f407cf0f31f66b9f720eed0b33 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 09:50:42 -0500 Subject: [PATCH 59/90] Mention YAMLPath.pop() --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index c437960e..661cf8c8 100644 --- a/CHANGES +++ b/CHANGES @@ -37,6 +37,8 @@ Enhancements: tracking was necessary to enable Hash/map/dict key renaming via the [name()] Search Keyword. These must be set when the NodeCoords is generated; it is not automatic. +* [API] YAMLPath instances now have a pop() method. This mutates the YAMLPath + by popping off its last segment, returning that segment. 3.4.1: Bug Fixes: From 42f61d7a2461c710b4c0d17b51313fec7ff36503 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 12:42:30 -0500 Subject: [PATCH 60/90] First pass at max() --- yamlpath/common/keywordsearches.py | 142 +++++++++++++++++++++++++++ yamlpath/enums/pathsearchkeywords.py | 14 ++- 2 files changed, 153 insertions(+), 3 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 09a784dc..dd126469 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -36,6 +36,9 @@ def search_matches( elif keyword is PathSearchKeywords.NAME: nc_matches = KeywordSearches.name( invert, parameters, yaml_path, **kwargs) + elif keyword is PathSearchKeywords.MAX: + nc_matches = KeywordSearches.max( + haystack, invert, parameters, yaml_path, **kwargs) elif keyword is PathSearchKeywords.PARENT: nc_matches = KeywordSearches.parent( haystack, invert, parameters, yaml_path, **kwargs) @@ -146,6 +149,145 @@ def name( parentref, parent, parentref, translated_path, ancestry, relay_segment) + @staticmethod + # pylint: disable=locally-disabled,too-many-locals,too-many-branches + def max( + data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """Find whichever nodes/elements have a maximum value.""" + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) + + # There may be 0 or 1 parameters + param_count = len(parameters) + if param_count > 1: + raise YAMLPathException(( + "Invalid parameter count to {}([NAME]); up to {} permitted, " + " got {} in YAML Path" + ).format(PathSearchKeywords.MAX, 1, param_count), + str(yaml_path)) + + scan_node = parameters[0] if param_count > 0 else None + match_value: Any = None + match_nodes: List[NodeCoords] = [] + discard_nodes: List[NodeCoords] = [] + if yamlpath.common.Nodes.node_is_aoh(data): + # A named child node is mandatory + if scan_node is None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword requires a key name to scan" + " when evaluating an Array-of-Hashes." + ).format(PathSearchKeywords.MAX), + str(yaml_path)) + + for idx, ele in enumerate(data): + next_path = translated_path + "[{}]".format(idx) + next_ancestry = ancestry + [(data, idx)] + if hasattr(ele, scan_node): + eval_val = ele[scan_node] + if match_value is None or eval_val > match_value: + match_value = eval_val + match_nodes = [ + NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment) + ] + continue + + if eval_val == match_value: + match_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + continue + + discard_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + + elif isinstance(data, dict): + # A named child node is mandatory + if scan_node is None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword requires a key name to scan" + " when comparing Hash/map/dict children." + ).format(PathSearchKeywords.MAX), + str(yaml_path)) + + for key, val in data.items(): + if hasattr(val, scan_node): + eval_val = val[scan_node] + next_path = ( + translated_path + YAMLPath.escape_path_section( + key, translated_path.seperator)) + next_ancestry = ancestry + [(data, key)] + if match_value is None or eval_val > match_value: + match_value = eval_val + match_nodes = [ + NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment) + ] + continue + + if eval_val == match_value: + match_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) + continue + + discard_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) + + elif isinstance(data, list): + # A named child node is useless + if scan_node is not None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword cannot utilize a key name" + " when comparing Array/sequence/list elements to one" + " another." + ).format(PathSearchKeywords.MAX), + str(yaml_path)) + + for idx, ele in enumerate(data): + next_path = translated_path + "[{}]".format(idx) + next_ancestry = ancestry + [(data, idx)] + if match_value is None or ele > match_value: + match_value = ele + match_nodes = [ + NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment) + ] + continue + + if ele == match_value: + match_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + continue + + discard_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + + else: + # Non-complex data is always its own maximum and does not invert + match_value = data + match_nodes = [ + NodeCoords( + data, parent, parentref, translated_path, ancestry, + relay_segment) + ] + + yield_nodes = discard_nodes if invert else match_nodes + for node_coord in yield_nodes: + yield node_coord + @staticmethod # pylint: disable=locally-disabled,too-many-locals def parent( diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index d085d28e..1edc5313 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -16,15 +16,21 @@ class PathSearchKeywords(Enum): `HAS_CHILD` Matches when the node has a direct child with a given name. `NAME` - Matches only the key-name of the present node, discarding any and all - child node data. Can be used to rename the matched key as long as the - new name is unique within the parent. + Matches only the key-name or element-index of the present node, + discarding any and all child node data. Can be used to rename the + matched key as long as the new name is unique within the parent, lest + the preexisting node be overwritten. Cannot be used to reassign an + Array/sequence/list element to another position. + `MAX` + Matches whichever node(s) has/have the maximum value for a named child + key or the maximum value within an Array/sequence/list. `PARENT` Access the parent(s) of the present node. """ HAS_CHILD = auto() NAME = () + MAX = () PARENT = auto() def __str__(self) -> str: @@ -34,6 +40,8 @@ def __str__(self) -> str: keyword = 'has_child' elif self is PathSearchKeywords.NAME: keyword = 'name' + elif self is PathSearchKeywords.MAX: + keyword = 'max' elif self is PathSearchKeywords.PARENT: keyword = 'parent' From 003bdbd5bc489759c6947664b15b9261658f8dd1 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 12:58:54 -0500 Subject: [PATCH 61/90] Allow max() inversion --- yamlpath/common/keywordsearches.py | 11 +++++++---- yamlpath/enums/pathsearchkeywords.py | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index dd126469..862eb104 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -150,7 +150,7 @@ def name( relay_segment) @staticmethod - # pylint: disable=locally-disabled,too-many-locals,too-many-branches + # pylint: disable=locally-disabled,too-many-locals,too-many-branches,too-many-statements def max( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any @@ -180,7 +180,7 @@ def max( if scan_node is None: raise YAMLPathException(( "The {}([NAME]) Search Keyword requires a key name to scan" - " when evaluating an Array-of-Hashes." + " when evaluating an Array-of-Hashes in YAML Path" ).format(PathSearchKeywords.MAX), str(yaml_path)) @@ -191,6 +191,7 @@ def max( eval_val = ele[scan_node] if match_value is None or eval_val > match_value: match_value = eval_val + discard_nodes.extend(match_nodes) match_nodes = [ NodeCoords( ele, data, idx, next_path, next_ancestry, @@ -213,7 +214,7 @@ def max( if scan_node is None: raise YAMLPathException(( "The {}([NAME]) Search Keyword requires a key name to scan" - " when comparing Hash/map/dict children." + " when comparing Hash/map/dict children in YAML Path" ).format(PathSearchKeywords.MAX), str(yaml_path)) @@ -226,6 +227,7 @@ def max( next_ancestry = ancestry + [(data, key)] if match_value is None or eval_val > match_value: match_value = eval_val + discard_nodes.extend(match_nodes) match_nodes = [ NodeCoords( val, data, key, next_path, next_ancestry, @@ -249,7 +251,7 @@ def max( raise YAMLPathException(( "The {}([NAME]) Search Keyword cannot utilize a key name" " when comparing Array/sequence/list elements to one" - " another." + " another in YAML Path" ).format(PathSearchKeywords.MAX), str(yaml_path)) @@ -258,6 +260,7 @@ def max( next_ancestry = ancestry + [(data, idx)] if match_value is None or ele > match_value: match_value = ele + discard_nodes.extend(match_nodes) match_nodes = [ NodeCoords( ele, data, idx, next_path, next_ancestry, diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index 1edc5313..a4cd36d2 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -29,8 +29,8 @@ class PathSearchKeywords(Enum): """ HAS_CHILD = auto() - NAME = () - MAX = () + NAME = auto() + MAX = auto() PARENT = auto() def __str__(self) -> str: From 312a29e9d249da629029767450acf1e1f29abd6f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 13:00:43 -0500 Subject: [PATCH 62/90] Better unknown search keyword error grammar --- yamlpath/yamlpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 16ee8ada..6ad3e397 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -448,7 +448,7 @@ def _parse_path(self, " Encountered in YAML Path") .format( segment_id, - ','.join(PathSearchKeywords.get_keywords()) + ', '.join(PathSearchKeywords.get_keywords()) ) , yaml_path ) From 0d794950804e21d904803ba817acaa4f66aca771 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 14:54:08 -0500 Subject: [PATCH 63/90] Let more null data through processing --- yamlpath/common/keywordsearches.py | 13 ++++++++++--- yamlpath/processor.py | 26 +------------------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 862eb104..75e9c81c 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -77,7 +77,7 @@ def has_child( # child key exactly named as per parameters. When inverted, only # parents with no such key are yielded. if isinstance(data, dict): - child_present = match_key in data + child_present = data is not None and match_key in data if ( (invert and not child_present) or (child_present and not invert) @@ -112,6 +112,12 @@ def has_child( data, parent, parentref, translated_path, ancestry, relay_segment) + elif data is None: + if invert: + yield NodeCoords( + data, parent, parentref, translated_path, ancestry, + relay_segment) + else: raise YAMLPathException( ("{} data has no child nodes in YAML Path").format(type(data)), @@ -187,7 +193,7 @@ def max( for idx, ele in enumerate(data): next_path = translated_path + "[{}]".format(idx) next_ancestry = ancestry + [(data, idx)] - if hasattr(ele, scan_node): + if scan_node in ele: eval_val = ele[scan_node] if match_value is None or eval_val > match_value: match_value = eval_val @@ -219,7 +225,8 @@ def max( str(yaml_path)) for key, val in data.items(): - if hasattr(val, scan_node): + print("Looking at key[{}] and val[{}]".format(key, val)) + if val is not None and scan_node in val: eval_val = val[scan_node] next_path = ( translated_path + YAMLPath.escape_path_section( diff --git a/yamlpath/processor.py b/yamlpath/processor.py index c68abf11..6c5a5a26 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -607,14 +607,6 @@ def _get_nodes_by_path_segment(self, data: Any, traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) - if data is None: - self.logger.debug( - "Bailing out on None data at parentref, {}, of parent:" - .format(parentref), - prefix="Processor::_get_nodes_by_path_segment: ", - data=parent) - return - segments = yaml_path.escaped if not (segments and len(segments) > segment_index): self.logger.debug( @@ -1457,15 +1449,6 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) ancestry: List[tuple] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) - - if data is None: - self.logger.debug( - "Bailing out on None data at parentref, {}, of parent:" - .format(parentref), - prefix="Processor::_get_required_nodes: ", - data=parent) - return - segments = yaml_path.escaped if segments and len(segments) > depth: pathseg: PathSegment = yaml_path.unescaped[depth] @@ -1492,14 +1475,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, prefix="Processor::_get_required_nodes: ", data=segment_node_coords) - if (segment_node_coords is None - or (hasattr(segment_node_coords, "node") - and segment_node_coords.node is None) - ): - self.logger.debug( - "Processor::_get_required_nodes: Yielding null.") - yield segment_node_coords - elif isinstance(segment_node_coords, list): + if isinstance(segment_node_coords, list): # Most likely the output of a Collector, this list will be # of NodeCoords rather than an actual DOM reference. As # such, it must be treated as a virtual DOM element that From 230d17a5cf2524dfcf7f3dc4c4ab1f326bfbbda9 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Thu, 22 Apr 2021 17:05:31 -0500 Subject: [PATCH 64/90] Just about done with kw-max --- tests/test_commands_yaml_get.py | 68 ++++++++++++++++++++++++++++ tests/test_common_keywordsearches.py | 53 ++++++++++++++++++++++ tests/test_wrappers_nodecoords.py | 20 ++++++++ yamlpath/common/keywordsearches.py | 63 ++++++++++++++++---------- yamlpath/wrappers/nodecoords.py | 12 +++++ 5 files changed, 191 insertions(+), 25 deletions(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index aab58108..7d2f399d 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -258,6 +258,8 @@ def test_get_only_aoh_nodes_with_named_child(self, script_runner, tmp_path_facto ("/items/*[!has_child(bravo)][2][parent(0)]", ['delta']), ("/items/*[!has_child(bravo)][2][parent()]", ['["alpha", "charlie", "delta"]']), ("/items/*[!has_child(bravo)][2][parent(2)]", ['[["alpha", "bravo", "charlie"], ["alpha", "charlie", "delta"], ["alpha", "bravo", "delta"], ["bravo", "charlie", "delta"]]']), + ("/prices_hash/*[has_child(price)][name()]", ['doohickey', 'whatchamacallit', 'widget']), + ("/prices_hash/*[!has_child(price)][name()]", ['unknown']), ]) def test_get_parent_nodes(self, script_runner, tmp_path_factory, query, output): content = """--- @@ -274,6 +276,15 @@ def test_get_parent_nodes(self, script_runner, tmp_path_factory, query, output): - - bravo - charlie - delta + +prices_hash: + doohickey: + price: 4.99 + whatchamacallit: + price: 9.95 + widget: + price: 0.98 + unknown: """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) @@ -316,3 +327,60 @@ def test_get_node_names(self, script_runner, tmp_path_factory, query, output): for line in result.stdout.splitlines(): assert line == output[match_index] match_index += 1 + + @pytest.mark.parametrize("query,output", [ + ("prices_aoh[max(price)].product", ["whatchamacallit"]), + ("prices_aoh[!max(price)].price", ["4.99", "4.99", "0.98"]), + ("/prices_hash[max(price)][name()]", ["whatchamacallit"]), + ("/prices_hash[!max(price)][name()]", ["doohickey", "fob", "widget", "unknown"]), + ("(/prices_hash/*/price)[max()]", ["9.95"]), + ("(/prices_hash/*/price)[!max()]", ["4.99", "4.99", "0.98"]), + ("/prices_array[max()]", ["9.95"]), + ("/prices_array[!max()]", ["4.99", "4.99", "0.98", "\x00"]), + ("/bare[max()]", ["value"]), + ]) + def test_get_max_nodes(self, script_runner, tmp_path_factory, query, output): + content = """--- +# Consistent Data Types +prices_aoh: + - product: doohickey + price: 4.99 + - product: fob + price: 4.99 + - product: whatchamacallit + price: 9.95 + - product: widget + price: 0.98 + - product: unknown + +prices_hash: + doohickey: + price: 4.99 + fob: + price: 4.99 + whatchamacallit: + price: 9.95 + widget: + price: 0.98 + unknown: + +prices_array: + - 4.99 + - 4.99 + - 9.95 + - 0.98 + - null + +bare: value + +# TODO: Inconsistent Data Types +""" + + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run(self.command, "--query={}".format(query), yaml_file) + assert result.success, result.stderr + + match_index = 0 + for line in result.stdout.splitlines(): + assert line == output[match_index] + match_index += 1 diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py index 12479649..d45ac610 100644 --- a/tests/test_common_keywordsearches.py +++ b/tests/test_common_keywordsearches.py @@ -69,6 +69,59 @@ def test_name_invalid_inversion(self): assert -1 < str(ex.value).find("Inversion is meaningless to ") + ### + # max + ### + def test_max_invalid_param_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.max( + {}, + False, + ["1", "2"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Invalid parameter count to ") + + def test_max_missing_aoh_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.max( + [{'a': 1},{'a': 2}], + False, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when evaluating an Array-of-Hashes") + + def test_max_missing_hash_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.max( + {'a': {'b': 1}, 'c': {'d': 2}}, + False, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when comparing Hash/map/dict children") + + def test_max_invalid_array_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.max( + [1, 2, 3], + False, + ['3'], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when comparing Array/sequence/list elements to one another") + + def test_max_incorrect_node(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.max( + {'b': 2}, + False, + ['b'], + YAMLPath("/*[max(b)]") + )) + assert -1 < str(ex.value).find("operates against collections of data") + ### # parent ### diff --git a/tests/test_wrappers_nodecoords.py b/tests/test_wrappers_nodecoords.py index 82d2c78d..f851f55c 100644 --- a/tests/test_wrappers_nodecoords.py +++ b/tests/test_wrappers_nodecoords.py @@ -15,3 +15,23 @@ def test_repr(self): def test_str(self): node_coord = NodeCoords([], None, None) assert str(node_coord) == "[]" + + def test_gt(self): + lhs_nc = NodeCoords(5, None, None) + rhs_nc = NodeCoords(3, None, None) + assert lhs_nc > rhs_nc + + def test_null_gt(self): + lhs_nc = NodeCoords(5, None, None) + rhs_nc = NodeCoords(None, None, None) + assert not lhs_nc > rhs_nc + + def test_lt(self): + lhs_nc = NodeCoords(5, None, None) + rhs_nc = NodeCoords(7, None, None) + assert lhs_nc < rhs_nc + + def test_null_lt(self): + lhs_nc = NodeCoords(5, None, None) + rhs_nc = NodeCoords(None, None, None) + assert not lhs_nc < rhs_nc diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 75e9c81c..9a52e049 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -225,32 +225,43 @@ def max( str(yaml_path)) for key, val in data.items(): - print("Looking at key[{}] and val[{}]".format(key, val)) - if val is not None and scan_node in val: - eval_val = val[scan_node] - next_path = ( - translated_path + YAMLPath.escape_path_section( - key, translated_path.seperator)) - next_ancestry = ancestry + [(data, key)] - if match_value is None or eval_val > match_value: - match_value = eval_val - discard_nodes.extend(match_nodes) - match_nodes = [ - NodeCoords( + if isinstance(val, dict): + if val is not None and scan_node in val: + eval_val = val[scan_node] + next_path = ( + translated_path + YAMLPath.escape_path_section( + key, translated_path.seperator)) + next_ancestry = ancestry + [(data, key)] + if match_value is None or eval_val > match_value: + match_value = eval_val + discard_nodes.extend(match_nodes) + match_nodes = [ + NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment) + ] + continue + + if eval_val == match_value: + match_nodes.append(NodeCoords( val, data, key, next_path, next_ancestry, - relay_segment) - ] - continue + relay_segment)) + continue - if eval_val == match_value: - match_nodes.append(NodeCoords( - val, data, key, next_path, next_ancestry, - relay_segment)) - continue + discard_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) - discard_nodes.append(NodeCoords( - val, data, key, next_path, next_ancestry, - relay_segment)) + elif scan_node in data: + # The user probably meant to operate against the parent + raise YAMLPathException(( + "The {}([NAME]) Search Keyword operates against" + " collections of data which share a common attribute" + " yet there is only a single node to consider. Did" + " you mean to evaluate the parent of the selected" + " node? Please review your YAML Path" + ).format(PathSearchKeywords.MAX), + str(yaml_path)) elif isinstance(data, list): # A named child node is useless @@ -265,7 +276,9 @@ def max( for idx, ele in enumerate(data): next_path = translated_path + "[{}]".format(idx) next_ancestry = ancestry + [(data, idx)] - if match_value is None or ele > match_value: + if (ele is not None + and (match_value is None or ele > match_value) + ): match_value = ele discard_nodes.extend(match_nodes) match_nodes = [ @@ -275,7 +288,7 @@ def max( ] continue - if ele == match_value: + if ele is not None and ele == match_value: match_nodes.append(NodeCoords( ele, data, idx, next_path, next_ancestry, relay_segment)) diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index ed32e573..51c3f7ad 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -57,6 +57,18 @@ def __repr__(self) -> str: self.__class__.__name__, self.node, self.parent, self.parentref)) + def __gt__(self, rhs: "NodeCoords") -> Any: + """Indicate whether this node's data is greater-than another's.""" + if self.node is None or rhs.node is None: + return False + return self.node > rhs.node + + def __lt__(self, rhs: "NodeCoords") -> Any: + """Indicate whether this node's data is less-than another's.""" + if self.node is None or rhs.node is None: + return False + return self.node < rhs.node + @staticmethod def unwrap_node_coords(data: Any) -> Any: """ From 365f9a9d92125b2cc80a07aca15a72a5046e019c Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 11:39:38 -0500 Subject: [PATCH 65/90] Fallback to string comparisons for non-numerics --- yamlpath/common/searches.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yamlpath/common/searches.py b/yamlpath/common/searches.py index 0abfbfde..a4ebb012 100644 --- a/yamlpath/common/searches.py +++ b/yamlpath/common/searches.py @@ -65,7 +65,7 @@ def search_matches( except ValueError: matches = False else: - matches = haystack > needle + matches = str(haystack) > str(needle) elif method is PathSearchMethods.LESS_THAN: if isinstance(haystack, int): try: @@ -78,7 +78,7 @@ def search_matches( except ValueError: matches = False else: - matches = haystack < needle + matches = str(haystack) < str(needle) elif method is PathSearchMethods.GREATER_THAN_OR_EQUAL: if isinstance(haystack, int): try: @@ -91,7 +91,7 @@ def search_matches( except ValueError: matches = False else: - matches = haystack >= needle + matches = str(haystack) >= str(needle) elif method is PathSearchMethods.LESS_THAN_OR_EQUAL: if isinstance(haystack, int): try: @@ -104,7 +104,7 @@ def search_matches( except ValueError: matches = False else: - matches = haystack <= needle + matches = str(haystack) <= str(needle) elif method == PathSearchMethods.REGEX: matcher = re.compile(needle) matches = matcher.search(str(haystack)) is not None From a74acce2f258be4eb0977d7bf2d3d63cddc613e4 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 12:14:07 -0500 Subject: [PATCH 66/90] Defer all search comparisons to Searches.* --- yamlpath/common/keywordsearches.py | 48 ++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 9a52e049..97547fab 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -9,7 +9,7 @@ from typing import Any, Generator, List from yamlpath.types import PathSegment -from yamlpath.enums import PathSearchKeywords +from yamlpath.enums import PathSearchKeywords, PathSearchMethods from yamlpath.path import SearchKeywordTerms from yamlpath.exceptions import YAMLPathException from yamlpath.wrappers import NodeCoords @@ -195,7 +195,11 @@ def max( next_ancestry = ancestry + [(data, idx)] if scan_node in ele: eval_val = ele[scan_node] - if match_value is None or eval_val > match_value: + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.GREATER_THAN, match_value, + eval_val) + ): match_value = eval_val discard_nodes.extend(match_nodes) match_nodes = [ @@ -205,7 +209,11 @@ def max( ] continue - if eval_val == match_value: + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + eval_val) + ): match_nodes.append(NodeCoords( ele, data, idx, next_path, next_ancestry, relay_segment)) @@ -232,7 +240,11 @@ def max( translated_path + YAMLPath.escape_path_section( key, translated_path.seperator)) next_ancestry = ancestry + [(data, key)] - if match_value is None or eval_val > match_value: + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.GREATER_THAN, match_value, + eval_val) + ): match_value = eval_val discard_nodes.extend(match_nodes) match_nodes = [ @@ -242,16 +254,16 @@ def max( ] continue - if eval_val == match_value: + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + eval_val) + ): match_nodes.append(NodeCoords( val, data, key, next_path, next_ancestry, relay_segment)) continue - discard_nodes.append(NodeCoords( - val, data, key, next_path, next_ancestry, - relay_segment)) - elif scan_node in data: # The user probably meant to operate against the parent raise YAMLPathException(( @@ -263,6 +275,10 @@ def max( ).format(PathSearchKeywords.MAX), str(yaml_path)) + discard_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) + elif isinstance(data, list): # A named child node is useless if scan_node is not None: @@ -277,8 +293,12 @@ def max( next_path = translated_path + "[{}]".format(idx) next_ancestry = ancestry + [(data, idx)] if (ele is not None - and (match_value is None or ele > match_value) - ): + and ( + match_value is None or + yamlpath.common.Searches.search_matches( + PathSearchMethods.GREATER_THAN, match_value, + ele) + )): match_value = ele discard_nodes.extend(match_nodes) match_nodes = [ @@ -288,7 +308,11 @@ def max( ] continue - if ele is not None and ele == match_value: + if (ele is not None + and yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + ele) + ): match_nodes.append(NodeCoords( ele, data, idx, next_path, next_ancestry, relay_segment)) From 40a57dd42d2b649ee212abbf2ede4552b3ee964a Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 12:35:03 -0500 Subject: [PATCH 67/90] Added bad-case tests for max() --- tests/test_commands_yaml_get.py | 44 ++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index 7d2f399d..0af41143 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -333,11 +333,19 @@ def test_get_node_names(self, script_runner, tmp_path_factory, query, output): ("prices_aoh[!max(price)].price", ["4.99", "4.99", "0.98"]), ("/prices_hash[max(price)][name()]", ["whatchamacallit"]), ("/prices_hash[!max(price)][name()]", ["doohickey", "fob", "widget", "unknown"]), - ("(/prices_hash/*/price)[max()]", ["9.95"]), - ("(/prices_hash/*/price)[!max()]", ["4.99", "4.99", "0.98"]), + ("(prices_hash.*.price)[max()]", ["9.95"]), + ("(prices_hash.*.price)[!max()]", ["4.99", "4.99", "0.98"]), ("/prices_array[max()]", ["9.95"]), ("/prices_array[!max()]", ["4.99", "4.99", "0.98", "\x00"]), - ("/bare[max()]", ["value"]), + ("bare[max()]", ["value"]), + ("/bad_prices_aoh[max(price)]/product", ["fob"]), + ("/bad_prices_aoh[!max(price)]/price", ["4.99", "9.95", "True"]), + ("bad_prices_hash[max(price)][name()]", ["fob"]), + ("bad_prices_hash[!max(price)][name()]", ["doohickey", "whatchamacallit", "widget", "unknown"]), + ("(/bad_prices_hash/*/price)[max()]", ["not set"]), + ("(/bad_prices_hash/*/price)[!max()]", ["4.99", "9.95", "True"]), + ("bad_prices_array[max()]", ["not set"]), + ("bad_prices_array[!max()]", ["4.99", "9.95", "0.98", "\x00"]), ]) def test_get_max_nodes(self, script_runner, tmp_path_factory, query, output): content = """--- @@ -371,9 +379,37 @@ def test_get_max_nodes(self, script_runner, tmp_path_factory, query, output): - 0.98 - null +# TODO: Inconsistent Data Types bare: value -# TODO: Inconsistent Data Types +bad_prices_aoh: + - product: doohickey + price: 4.99 + - product: fob + price: not set + - product: whatchamacallit + price: 9.95 + - product: widget + price: true + - product: unknown + +bad_prices_hash: + doohickey: + price: 4.99 + fob: + price: not set + whatchamacallit: + price: 9.95 + widget: + price: true + unknown: + +bad_prices_array: + - 4.99 + - not set + - 9.95 + - 0.98 + - null """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) From 27814845a475901a0434116bed4e4ba97358cccc Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 13:17:25 -0500 Subject: [PATCH 68/90] Set goal for this fix --- yamlpath/processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 6c5a5a26..50ddf6b5 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -190,7 +190,7 @@ def _apply_change( tag: str = kwargs.pop("tag", None) self.logger.debug( - "Matched optional node coordinate:" + "Matched node coordinate:" , data=node_coord , prefix="Processor::_apply_change: ") self.logger.debug( @@ -205,7 +205,8 @@ def _apply_change( isinstance(segment_value, SearchKeywordTerms) and segment_value.keyword is PathSearchKeywords.NAME ): - # Rename a key + # Rename a key; the new name must not already exist in its + # parent. parent = node_coord.parent parentref = node_coord.parentref if isinstance(parent, CommentedMap): From 1ba28500e15fd4545f3c868672222cef16971e1e Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 13:50:01 -0500 Subject: [PATCH 69/90] Bump version to 3.5.0 --- yamlpath/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/__init__.py b/yamlpath/__init__.py index 5962188e..8b0a6021 100644 --- a/yamlpath/__init__.py +++ b/yamlpath/__init__.py @@ -1,6 +1,6 @@ """Core YAML Path classes.""" # Establish the version number common to all components -__version__ = "3.4.1" +__version__ = "3.5.0" from yamlpath.yamlpath import YAMLPath from yamlpath.processor import Processor From 87c924233122191f02c40a3f44af1aeee0d006c3 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 14:50:12 -0500 Subject: [PATCH 70/90] Protect against data loss during bad key renames --- tests/test_commands_yaml_get.py | 2 +- tests/test_commands_yaml_set.py | 18 +++++++++++++++++- tests/test_processor.py | 10 ++++++++++ yamlpath/processor.py | 13 +++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index 0af41143..f89821a0 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -379,7 +379,7 @@ def test_get_max_nodes(self, script_runner, tmp_path_factory, query, output): - 0.98 - null -# TODO: Inconsistent Data Types +# Inconsistent Data Types bare: value bad_prices_aoh: diff --git a/tests/test_commands_yaml_set.py b/tests/test_commands_yaml_set.py index 14ee357e..ba28005b 100644 --- a/tests/test_commands_yaml_set.py +++ b/tests/test_commands_yaml_set.py @@ -1323,7 +1323,7 @@ def test_change_key_name_good(self, script_runner, tmp_path_factory): filedat = fhnd.read() assert filedat == yamlout - def test_change_key_name_bad(self, script_runner, tmp_path_factory): + def test_change_key_name_maps_only(self, script_runner, tmp_path_factory): yamlin = """--- items: - one @@ -1339,3 +1339,19 @@ def test_change_key_name_bad(self, script_runner, tmp_path_factory): ) assert not result.success, result.stdout assert "Keys can be renamed only in Hash/map/dict" in result.stderr + + def test_change_key_name_unique_only(self, script_runner, tmp_path_factory): + yamlin = """--- +key: value +another_key: value +""" + + yaml_file = create_temp_yaml_file(tmp_path_factory, yamlin) + result = script_runner.run( + self.command, + "--change=another_key[name()]", + "--value=key", + yaml_file + ) + assert not result.success, result.stdout + assert "already exists at the same document level" in result.stderr diff --git a/tests/test_processor.py b/tests/test_processor.py index e9a7f7a4..a637c772 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -917,6 +917,16 @@ def test_tag_nodes(self, quiet_logger, yaml_path, tag, pathseperator): def test_rename_dict_key(self, quiet_logger, yaml_path, value, old_data, new_data): processor = Processor(quiet_logger, old_data) processor.set_value(yaml_path, value) + assert new_data == old_data + + @pytest.mark.parametrize("yaml_path,value,old_data", [ + (YAMLPath("/key[name()]"), "renamed_key", {'key': 'value', 'renamed_key': 'value'}), + ]) + def test_rename_dict_key_cannot_overwrite(self, quiet_logger, yaml_path, value, old_data): + processor = Processor(quiet_logger, old_data) + with pytest.raises(YAMLPathException) as ex: + processor.set_value(yaml_path, value) + assert -1 < str(ex.value).find("already exists at the same document level") def test_traverse_with_null(self, quiet_logger): # Contributed by https://github.com/rbordelo diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 50ddf6b5..c84b8721 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -210,13 +210,26 @@ def _apply_change( parent = node_coord.parent parentref = node_coord.parentref if isinstance(parent, CommentedMap): + if value in parent: + raise YAMLPathException(( + "Key, {}, already exists at the same document" + " level in YAML Path" + ).format(value), str(yaml_path)) + for i, k in [ (idx, key) for idx, key in enumerate(parent.keys()) if key == parentref ]: parent.insert(i, value, parent.pop(k)) + break elif isinstance(parent, dict): + if value in parent: + raise YAMLPathException(( + "Key, {}, already exists at the same document" + " level in YAML Path" + ).format(value), str(yaml_path)) + parent[value] = parent[parentref] del parent[parentref] else: From d6db28b270671c5008b9f9b917d4e4036671d527 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 15:31:21 -0500 Subject: [PATCH 71/90] Add search keyword min([NAME]) --- tests/test_commands_yaml_get.py | 20 ++- tests/test_common_keywordsearches.py | 55 ++++++++ yamlpath/common/keywordsearches.py | 184 +++++++++++++++++++++++++++ yamlpath/enums/pathsearchkeywords.py | 10 +- 4 files changed, 267 insertions(+), 2 deletions(-) diff --git a/tests/test_commands_yaml_get.py b/tests/test_commands_yaml_get.py index f89821a0..31e5f2b0 100644 --- a/tests/test_commands_yaml_get.py +++ b/tests/test_commands_yaml_get.py @@ -346,8 +346,26 @@ def test_get_node_names(self, script_runner, tmp_path_factory, query, output): ("(/bad_prices_hash/*/price)[!max()]", ["4.99", "9.95", "True"]), ("bad_prices_array[max()]", ["not set"]), ("bad_prices_array[!max()]", ["4.99", "9.95", "0.98", "\x00"]), + + ("prices_aoh[min(price)].product", ["widget"]), + ("prices_aoh[!min(price)].price", ["9.95", "4.99", "4.99"]), + ("/prices_hash[min(price)][name()]", ["widget"]), + ("/prices_hash[!min(price)][name()]", ["whatchamacallit", "doohickey", "fob", "unknown"]), + ("(prices_hash.*.price)[min()]", ["0.98"]), + ("(prices_hash.*.price)[!min()]", ["9.95", "4.99", "4.99"]), + ("/prices_array[min()]", ["0.98"]), + ("/prices_array[!min()]", ["9.95", "4.99", "4.99", "\x00"]), + ("bare[min()]", ["value"]), + ("/bad_prices_aoh[min(price)]/product", ["widget"]), + ("/bad_prices_aoh[!min(price)]/price", ["not set", "9.95", "4.99"]), + ("bad_prices_hash[min(price)][name()]", ["widget"]), + ("bad_prices_hash[!min(price)][name()]", ["fob", "whatchamacallit", "doohickey", "unknown"]), + ("(/bad_prices_hash/*/price)[min()]", ["4.99"]), + ("(/bad_prices_hash/*/price)[!min()]", ["not set", "9.95", "True"]), + ("bad_prices_array[min()]", ["0.98"]), + ("bad_prices_array[!min()]", ["not set", "9.95", "4.99", "\x00"]), ]) - def test_get_max_nodes(self, script_runner, tmp_path_factory, query, output): + def test_get_min_max_nodes(self, script_runner, tmp_path_factory, query, output): content = """--- # Consistent Data Types prices_aoh: diff --git a/tests/test_common_keywordsearches.py b/tests/test_common_keywordsearches.py index d45ac610..34e9bea8 100644 --- a/tests/test_common_keywordsearches.py +++ b/tests/test_common_keywordsearches.py @@ -122,6 +122,61 @@ def test_max_incorrect_node(self): )) assert -1 < str(ex.value).find("operates against collections of data") + + ### + # min + ### + def test_min_invalid_param_count(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.min( + {}, + False, + ["1", "2"], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("Invalid parameter count to ") + + def test_min_missing_aoh_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.min( + [{'a': 1},{'a': 2}], + False, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when evaluating an Array-of-Hashes") + + def test_min_missing_hash_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.min( + {'a': {'b': 1}, 'c': {'d': 2}}, + False, + [], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when comparing Hash/map/dict children") + + def test_min_invalid_array_param(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.min( + [1, 2, 3], + False, + ['3'], + YAMLPath("/") + )) + assert -1 < str(ex.value).find("when comparing Array/sequence/list elements to one another") + + def test_min_incorrect_node(self): + with pytest.raises(YAMLPathException) as ex: + nodes = list(KeywordSearches.min( + {'b': 2}, + False, + ['b'], + YAMLPath("/*[max(b)]") + )) + assert -1 < str(ex.value).find("operates against collections of data") + + ### # parent ### diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 97547fab..56649260 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -39,6 +39,9 @@ def search_matches( elif keyword is PathSearchKeywords.MAX: nc_matches = KeywordSearches.max( haystack, invert, parameters, yaml_path, **kwargs) + elif keyword is PathSearchKeywords.MIN: + nc_matches = KeywordSearches.min( + haystack, invert, parameters, yaml_path, **kwargs) elif keyword is PathSearchKeywords.PARENT: nc_matches = KeywordSearches.parent( haystack, invert, parameters, yaml_path, **kwargs) @@ -335,6 +338,187 @@ def max( for node_coord in yield_nodes: yield node_coord + + @staticmethod + # pylint: disable=locally-disabled,too-many-locals,too-many-branches,too-many-statements + def min( + data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, + **kwargs: Any + ) -> Generator[NodeCoords, None, None]: + """Find whichever nodes/elements have a minimum value.""" + parent: Any = kwargs.pop("parent", None) + parentref: Any = kwargs.pop("parentref", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[tuple] = kwargs.pop("ancestry", []) + relay_segment: PathSegment = kwargs.pop("relay_segment", None) + + # There may be 0 or 1 parameters + param_count = len(parameters) + if param_count > 1: + raise YAMLPathException(( + "Invalid parameter count to {}([NAME]); up to {} permitted, " + " got {} in YAML Path" + ).format(PathSearchKeywords.MIN, 1, param_count), + str(yaml_path)) + + scan_node = parameters[0] if param_count > 0 else None + match_value: Any = None + match_nodes: List[NodeCoords] = [] + discard_nodes: List[NodeCoords] = [] + if yamlpath.common.Nodes.node_is_aoh(data): + # A named child node is mandatory + if scan_node is None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword requires a key name to scan" + " when evaluating an Array-of-Hashes in YAML Path" + ).format(PathSearchKeywords.MIN), + str(yaml_path)) + + for idx, ele in enumerate(data): + next_path = translated_path + "[{}]".format(idx) + next_ancestry = ancestry + [(data, idx)] + if scan_node in ele: + eval_val = ele[scan_node] + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.LESS_THAN, match_value, + eval_val) + ): + match_value = eval_val + discard_nodes.extend(match_nodes) + match_nodes = [ + NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment) + ] + continue + + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + eval_val) + ): + match_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + continue + + discard_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + + elif isinstance(data, dict): + # A named child node is mandatory + if scan_node is None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword requires a key name to scan" + " when comparing Hash/map/dict children in YAML Path" + ).format(PathSearchKeywords.MIN), + str(yaml_path)) + + for key, val in data.items(): + if isinstance(val, dict): + if val is not None and scan_node in val: + eval_val = val[scan_node] + next_path = ( + translated_path + YAMLPath.escape_path_section( + key, translated_path.seperator)) + next_ancestry = ancestry + [(data, key)] + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.LESS_THAN, match_value, + eval_val) + ): + match_value = eval_val + discard_nodes.extend(match_nodes) + match_nodes = [ + NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment) + ] + continue + + if (match_value is None + or yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + eval_val) + ): + match_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) + continue + + elif scan_node in data: + # The user probably meant to operate against the parent + raise YAMLPathException(( + "The {}([NAME]) Search Keyword operates against" + " collections of data which share a common attribute" + " yet there is only a single node to consider. Did" + " you mean to evaluate the parent of the selected" + " node? Please review your YAML Path" + ).format(PathSearchKeywords.MIN), + str(yaml_path)) + + discard_nodes.append(NodeCoords( + val, data, key, next_path, next_ancestry, + relay_segment)) + + elif isinstance(data, list): + # A named child node is useless + if scan_node is not None: + raise YAMLPathException(( + "The {}([NAME]) Search Keyword cannot utilize a key name" + " when comparing Array/sequence/list elements to one" + " another in YAML Path" + ).format(PathSearchKeywords.MIN), + str(yaml_path)) + + for idx, ele in enumerate(data): + next_path = translated_path + "[{}]".format(idx) + next_ancestry = ancestry + [(data, idx)] + if (ele is not None + and ( + match_value is None or + yamlpath.common.Searches.search_matches( + PathSearchMethods.LESS_THAN, match_value, + ele) + )): + match_value = ele + discard_nodes.extend(match_nodes) + match_nodes = [ + NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment) + ] + continue + + if (ele is not None + and yamlpath.common.Searches.search_matches( + PathSearchMethods.EQUALS, match_value, + ele) + ): + match_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + continue + + discard_nodes.append(NodeCoords( + ele, data, idx, next_path, next_ancestry, + relay_segment)) + + else: + # Non-complex data is always its own maximum and does not invert + match_value = data + match_nodes = [ + NodeCoords( + data, parent, parentref, translated_path, ancestry, + relay_segment) + ] + + yield_nodes = discard_nodes if invert else match_nodes + for node_coord in yield_nodes: + yield node_coord + @staticmethod # pylint: disable=locally-disabled,too-many-locals def parent( diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index a4cd36d2..42d7fe9f 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -23,7 +23,12 @@ class PathSearchKeywords(Enum): Array/sequence/list element to another position. `MAX` Matches whichever node(s) has/have the maximum value for a named child - key or the maximum value within an Array/sequence/list. + key or the maximum value within an Array/sequence/list. When used + against a scalar value, that value is always its own maximum. + `MIN` + Matches whichever node(s) has/have the minimum value for a named child + key or the minimum value within an Array/sequence/list. When used + against a scalar value, that value is always its own minimum. `PARENT` Access the parent(s) of the present node. """ @@ -31,6 +36,7 @@ class PathSearchKeywords(Enum): HAS_CHILD = auto() NAME = auto() MAX = auto() + MIN = auto() PARENT = auto() def __str__(self) -> str: @@ -42,6 +48,8 @@ def __str__(self) -> str: keyword = 'name' elif self is PathSearchKeywords.MAX: keyword = 'max' + elif self is PathSearchKeywords.MIN: + keyword = 'min' elif self is PathSearchKeywords.PARENT: keyword = 'parent' From 9a5fcb84b10143f349de4ac6fa400abec0a33136 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 16:05:15 -0500 Subject: [PATCH 72/90] BUGFIX: null list elements caused unwanted changes --- yamlpath/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index c84b8721..244da6cf 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -1815,7 +1815,7 @@ def recurse(data, parent, parentref, reference_node, replacement_node): replacement_node) elif isinstance(data, list): for idx, item in enumerate(data): - if item is reference_node: + if data is parent and item is reference_node: data[idx] = replacement_node else: recurse(item, parent, parentref, reference_node, From ca0e3628be79646b3ca864ee4b040a902f0e3092 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 16:36:45 -0500 Subject: [PATCH 73/90] Collected optional nodes can now be set --- yamlpath/processor.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 244da6cf..d31f1605 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -180,6 +180,7 @@ def set_value(self, yaml_path: Union[YAMLPath, str], self._apply_change(yaml_path, node_coord, value, value_format=value_format, tag=tag) + # pylint: disable=locally-disabled,too-many-locals def _apply_change( self, yaml_path: YAMLPath, node_coord: NodeCoords, value: Any, **kwargs: Any @@ -198,6 +199,14 @@ def _apply_change( , data=value , prefix="Processor::_apply_change: ") + if isinstance(node_coord, list): + if len(node_coord) < 1: + return + + for collector_node in node_coord: + self._apply_change(yaml_path, collector_node, value, **kwargs) + return + last_segment = node_coord.path_segment if last_segment is not None: (_, segment_value) = last_segment @@ -1594,6 +1603,10 @@ def _get_optional_nodes( translated_path=translated_path, ancestry=ancestry ): matched_nodes += 1 + if not isinstance(next_coord, NodeCoords): + yield next_coord + continue + if next_coord.node is None: self.logger.debug(( "Relaying a None element <{}>{} from the data." From 82baa4f43d382a16b3255613679ff9f82fa0975e Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:31:36 -0500 Subject: [PATCH 74/90] Allow all collected nodes to be changed --- yamlpath/processor.py | 44 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index d31f1605..9b4721ab 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -191,7 +191,7 @@ def _apply_change( tag: str = kwargs.pop("tag", None) self.logger.debug( - "Matched node coordinate:" + "Matched node coordinate of type {}:".format(type(node_coord)) , data=node_coord , prefix="Processor::_apply_change: ") self.logger.debug( @@ -204,6 +204,22 @@ def _apply_change( return for collector_node in node_coord: + self.logger.debug( + "Expanded Collector results to apply change:" + , data=collector_node + , prefix="Processor::_apply_change: ") + self._apply_change(yaml_path, collector_node, value, **kwargs) + return + + if (isinstance(node_coord.node, list) + and len(node_coord.node) > 0 + and isinstance(node_coord.node[0], NodeCoords) + ): + for collector_node in node_coord.node: + self.logger.debug( + "Expanded collected Collector results to apply change:" + , data=collector_node + , prefix="Processor::_apply_change: ") self._apply_change(yaml_path, collector_node, value, **kwargs) return @@ -1294,6 +1310,10 @@ def _get_nodes_by_collector( # yield only when there are results if node_coords: + self.logger.debug(( + "Yielding collected node list:"), + prefix="Processor::_get_nodes_by_collector: ", + data=node_coords) yield node_coords # pylint: disable=locally-disabled,too-many-branches @@ -1509,6 +1529,7 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, data=segment_node_coords) for subnode_coord in self._get_required_nodes( segment_node_coords, yaml_path, depth + 1, + parent=parent, parentref=parentref, translated_path=translated_path, ancestry=ancestry, relay_segment=pathseg): yield subnode_coord @@ -1603,8 +1624,25 @@ def _get_optional_nodes( translated_path=translated_path, ancestry=ancestry ): matched_nodes += 1 - if not isinstance(next_coord, NodeCoords): - yield next_coord + if isinstance(next_coord, list): + # Drill into Collector results + for node_coord in self._get_optional_nodes( + next_coord, yaml_path, value, depth + 1, + parent=parent, parentref=parentref, + translated_path=translated_path, + ancestry=ancestry, + relay_segment=pathseg + ): + self.logger.debug(( + "Relaying a drilled-into Collector node:"), + prefix="Processor::_get_optional_nodes: ", + data={ + "node": node_coord, + "parent": parent, + "parentref": parentref + } + ) + yield node_coord continue if next_coord.node is None: From b6f9d8ecca33ba859d5bdb5fff9f740d450fc6fc Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:35:24 -0500 Subject: [PATCH 75/90] pylint happiness --- yamlpath/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 9b4721ab..1c8b7322 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -180,7 +180,7 @@ def set_value(self, yaml_path: Union[YAMLPath, str], self._apply_change(yaml_path, node_coord, value, value_format=value_format, tag=tag) - # pylint: disable=locally-disabled,too-many-locals + # pylint: disable=locally-disabled,too-many-locals,too-many-branches def _apply_change( self, yaml_path: YAMLPath, node_coord: NodeCoords, value: Any, **kwargs: Any From 998a1ee6c0839bc33bd88ea976850733c860ad29 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 19:34:08 -0500 Subject: [PATCH 76/90] 100% test coverage --- tests/test_processor.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_processor.py b/tests/test_processor.py index a637c772..3ac32846 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -315,6 +315,8 @@ def test_set_value_in_none_data(self, capsys, quiet_logger): ("/top_hash/positive_float", -2.71828, 1, True, YAMLValueFormats.FLOAT, PathSeperators.FSLASH), ("/top_hash/negative_float", 5283.4, 1, True, YAMLValueFormats.FLOAT, PathSeperators.FSLASH), ("/null_value", "No longer null", 1, True, YAMLValueFormats.DEFAULT, PathSeperators.FSLASH), + ("(top_array[0])+(top_hash.negative_float)+(/null_value)", "REPLACEMENT", 3, True, YAMLValueFormats.DEFAULT, PathSeperators.FSLASH), + ("(((top_array[0])+(top_hash.negative_float))+(/null_value))", "REPLACEMENT", 3, False, YAMLValueFormats.DEFAULT, PathSeperators.FSLASH), ]) def test_set_value(self, quiet_logger, yamlpath, value, tally, mustexist, vformat, pathsep): yamldata = """--- @@ -338,7 +340,13 @@ def test_set_value(self, quiet_logger, yamlpath, value, tally, mustexist, vforma processor.set_value(yamlpath, value, mustexist=mustexist, value_format=vformat, pathsep=pathsep) matchtally = 0 for node in processor.get_nodes(yamlpath, mustexist=mustexist): - assert unwrap_node_coords(node) == value + changed_value = unwrap_node_coords(node) + if isinstance(changed_value, list): + for result in changed_value: + assert result == value + matchtally += 1 + continue + assert changed_value == value matchtally += 1 assert matchtally == tally From 9e83b9bcbb96ae5d4eab09c6f7bf3a1910a4ee60 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 19:34:29 -0500 Subject: [PATCH 77/90] Cover edge-cases for Collector node changes --- yamlpath/processor.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 1c8b7322..3477d464 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -190,26 +190,21 @@ def _apply_change( YAMLValueFormats.DEFAULT) tag: str = kwargs.pop("tag", None) - self.logger.debug( - "Matched node coordinate of type {}:".format(type(node_coord)) - , data=node_coord - , prefix="Processor::_apply_change: ") - self.logger.debug( - "Setting its value with format {} to:".format(value_format) - , data=value - , prefix="Processor::_apply_change: ") - - if isinstance(node_coord, list): - if len(node_coord) < 1: - return - - for collector_node in node_coord: - self.logger.debug( - "Expanded Collector results to apply change:" - , data=collector_node - , prefix="Processor::_apply_change: ") - self._apply_change(yaml_path, collector_node, value, **kwargs) - return + self.logger.debug(( + "Attempting to change a node coordinate of type {} to value with" + " format <{}>:" + ).format(type(node_coord), value_format), + data={ + "value": value, + "node_coord": node_coord + }, prefix="Processor::_apply_change: ") + + if isinstance(node_coord.node, NodeCoords): + self.logger.debug( + "Unpacked Collector results to apply change:" + , data=node_coord.node + , prefix="Processor::_apply_change: ") + self._apply_change(yaml_path, node_coord.node, value, **kwargs) if (isinstance(node_coord.node, list) and len(node_coord.node) > 0 From 5f00b1a7de95fb93ab0cb256b9aa94a5fd897724 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 22:15:41 -0500 Subject: [PATCH 78/90] Allow yaml-paths to print unescaped paths --- yamlpath/commands/yaml_paths.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/yamlpath/commands/yaml_paths.py b/yamlpath/commands/yaml_paths.py index 8e4040c5..0ef4159a 100644 --- a/yamlpath/commands/yaml_paths.py +++ b/yamlpath/commands/yaml_paths.py @@ -92,6 +92,13 @@ def processcli(): help="omit YAML Paths from the output (useful with --values or to\ indicate whether a file has any matches without printing them\ all, perhaps especially with --noexpression)") + valdump_group.add_argument( + "-n", "--noescape", + action="store_true", + help="omit escape characters from special characters in printed YAML\ + Paths; this is unsafe for feeding the resulting YAML Paths into\ + other YAML Path commands because the symbols that would be\ + escaped have special meaning to YAML Path processors") parser.add_argument( "-t", "--pathsep", @@ -688,7 +695,11 @@ def print_results( resline += buffers[0] if print_yaml_path: - resline += "{}".format(result) + if args.noescape: + for (_, segment) in result.escaped: + resline += "{}".format(segment) + else: + resline += "{}".format(result) resline += buffers[1] if print_value: From 626ea72e2b34d6f2f70a9afddc5e1ac965642b91 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 22:32:13 -0500 Subject: [PATCH 79/90] Test unescaped yaml-paths output --- tests/test_commands_yaml_paths.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_commands_yaml_paths.py b/tests/test_commands_yaml_paths.py index 82b5c939..ffc67214 100644 --- a/tests/test_commands_yaml_paths.py +++ b/tests/test_commands_yaml_paths.py @@ -989,3 +989,35 @@ def test_too_many_stdins(self, script_runner): result = script_runner.run(self.command, "--search", "=nothing", "-", "-") assert not result.success, result.stderr assert "Only one YAML_FILE may be the - pseudo-file" in result.stderr + + def test_unescaped_paths(self, script_runner, tmp_path_factory): + # Credit: https://stackoverflow.com/questions/62155284/trying-to-get-all-paths-in-a-yaml-file + content = """--- +# sample set of lines +foo: + x: 12 + y: hello world + ip_range['initial']: 1.2.3.4 + ip_range[]: tba + array['first']: Cluster1 + +array2[]: bar +""" + yaml_file = create_temp_yaml_file(tmp_path_factory, content) + result = script_runner.run( + self.command, + "--nostdin", "--nofile", + "--expand", "--noescape", + "--keynames", "--values", + "--search", "=~/.*/", + yaml_file + ) + assert result.success, result.stderr + assert "\n".join([ + 'foox: 12', + 'fooy: hello world', + "fooip_range['initial']: 1.2.3.4", + 'fooip_range[]: tba', + "fooarray['first']: Cluster1", + 'array2[]: bar', + ]) + "\n" == result.stdout From 422fef5998c362af9e2feb2e83a8e1a8ff8ef762 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 22:43:33 -0500 Subject: [PATCH 80/90] Mention yaml-paths --noescape --- CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES b/CHANGES index 661cf8c8..c53dbd59 100644 --- a/CHANGES +++ b/CHANGES @@ -31,6 +31,10 @@ Enhancements: using their internal RegEx variant, [.=~/.*/]. They are now printed as they are entered, using a solitary *. As a consequence, any deliberate RegEx of [.=~/.*/] is also printed as its equivalent solitary *. +* The yaml-paths command now allows printing YAML Paths without protective + escape symbols via a new --noescape option. While this makes the output more + human-friendly, the unescaped paths will not be suitable for use as YAML Path + input to other YAML Path processors where special symbols require escaping. * [API] The NodeCoords class now tracks ancestry and the last YAML Path segment responsible for triggering its generation. The ancestry stack was necessary to support the [parent()] Search Keyword. The responsible YAML Path segment From 2201ffb3f7f3ff258538cb272d2eaac0b9ff9353 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 23:24:54 -0500 Subject: [PATCH 81/90] Add selectable separators to yaml-paths --noescape --- tests/test_commands_yaml_paths.py | 39 ++++++++++++++++++++++++------- yamlpath/commands/yaml_paths.py | 7 +++++- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/test_commands_yaml_paths.py b/tests/test_commands_yaml_paths.py index ffc67214..81361a6f 100644 --- a/tests/test_commands_yaml_paths.py +++ b/tests/test_commands_yaml_paths.py @@ -2,6 +2,7 @@ from tests.conftest import create_temp_yaml_file +from yamlpath.enums import PathSeperators class Test_yaml_paths(): """Tests the yaml-paths command-line interface.""" @@ -990,7 +991,33 @@ def test_too_many_stdins(self, script_runner): assert not result.success, result.stderr assert "Only one YAML_FILE may be the - pseudo-file" in result.stderr - def test_unescaped_paths(self, script_runner, tmp_path_factory): + @pytest.mark.parametrize("pathsep,output", [ + (PathSeperators.AUTO, [ + 'foo.x: 12', + 'foo.y: hello world', + "foo.ip_range['initial']: 1.2.3.4", + 'foo.ip_range[]: tba', + "foo.array['first']: Cluster1", + 'array2[]: bar', + ]), + (PathSeperators.DOT, [ + 'foo.x: 12', + 'foo.y: hello world', + "foo.ip_range['initial']: 1.2.3.4", + 'foo.ip_range[]: tba', + "foo.array['first']: Cluster1", + 'array2[]: bar', + ]), + (PathSeperators.FSLASH, [ + '/foo/x: 12', + '/foo/y: hello world', + "/foo/ip_range['initial']: 1.2.3.4", + '/foo/ip_range[]: tba', + "/foo/array['first']: Cluster1", + '/array2[]: bar', + ]), + ]) + def test_unescaped_paths(self, script_runner, tmp_path_factory, pathsep, output): # Credit: https://stackoverflow.com/questions/62155284/trying-to-get-all-paths-in-a-yaml-file content = """--- # sample set of lines @@ -1010,14 +1037,8 @@ def test_unescaped_paths(self, script_runner, tmp_path_factory): "--expand", "--noescape", "--keynames", "--values", "--search", "=~/.*/", + "--pathsep", str(pathsep), yaml_file ) assert result.success, result.stderr - assert "\n".join([ - 'foox: 12', - 'fooy: hello world', - "fooip_range['initial']: 1.2.3.4", - 'fooip_range[]: tba', - "fooarray['first']: Cluster1", - 'array2[]: bar', - ]) + "\n" == result.stdout + assert "\n".join(output) + "\n" == result.stdout diff --git a/yamlpath/commands/yaml_paths.py b/yamlpath/commands/yaml_paths.py index 0ef4159a..c4b0afe4 100644 --- a/yamlpath/commands/yaml_paths.py +++ b/yamlpath/commands/yaml_paths.py @@ -696,8 +696,13 @@ def print_results( resline += buffers[0] if print_yaml_path: if args.noescape: + use_flash = args.pathsep is PathSeperators.FSLASH + seglines = [] + join_mark = "/" if use_flash else "." + path_prefix = "/" if use_flash else "" for (_, segment) in result.escaped: - resline += "{}".format(segment) + seglines.append(str(segment)) + resline += "{}{}".format(path_prefix, join_mark.join(seglines)) else: resline += "{}".format(result) From a2ee2072ab04d2beb5ea66b5f3587a46bcfb804f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Fri, 23 Apr 2021 23:37:42 -0500 Subject: [PATCH 82/90] Promote yamlpath as having CLI tools --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 9415f315..ea8f7de6 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,8 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Operating System :: OS Independent", + "Environment :: Console", + "Topic :: Utilities", "Topic :: Software Development :: Libraries :: Python Modules", ], url="https://github.com/wwkimball/yamlpath", From fea7a34fccd57374638cde801728744d1a3b13c3 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 24 Apr 2021 18:23:47 -0500 Subject: [PATCH 83/90] Commit the RC version --- .github/workflows/python-publish-to-test-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish-to-test-pypi.yml b/.github/workflows/python-publish-to-test-pypi.yml index fa08467b..98699ddf 100644 --- a/.github/workflows/python-publish-to-test-pypi.yml +++ b/.github/workflows/python-publish-to-test-pypi.yml @@ -33,7 +33,7 @@ jobs: python -m pip install --upgrade setuptools wheel - name: Build Artifacts run: | - sed -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py + sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py python setup.py sdist bdist_wheel - name: Publish Artifacts uses: pypa/gh-action-pypi-publish@v1.4.2 From d079101ff0dc3242245c1977742c28cc4259d4fa Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 24 Apr 2021 18:36:15 -0500 Subject: [PATCH 84/90] Publish only when builds succeed --- .github/workflows/python-publish-to-prod-pypi.yml | 6 ++++++ .github/workflows/python-publish-to-test-pypi.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/python-publish-to-prod-pypi.yml b/.github/workflows/python-publish-to-prod-pypi.yml index bf1ed5fe..8ae1cb8c 100644 --- a/.github/workflows/python-publish-to-prod-pypi.yml +++ b/.github/workflows/python-publish-to-prod-pypi.yml @@ -2,6 +2,11 @@ name: Upload PRODUCTION Python Package on: + workflow_run: + workflows: + - build + types: + - completed push: branches: - master @@ -11,6 +16,7 @@ on: jobs: publish: name: Publish to Production PyPI + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest environment: 'PyPI: Production' diff --git a/.github/workflows/python-publish-to-test-pypi.yml b/.github/workflows/python-publish-to-test-pypi.yml index 98699ddf..8f347bc4 100644 --- a/.github/workflows/python-publish-to-test-pypi.yml +++ b/.github/workflows/python-publish-to-test-pypi.yml @@ -3,6 +3,11 @@ name: Upload Python TEST Package on: workflow_dispatch: + workflow_run: + workflows: + - build + types: + - completed push: branches: - development @@ -18,6 +23,7 @@ on: jobs: test-publish: name: Publish to TEST PyPI + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest environment: 'PyPI: Test' From ab61efa9944bec507e71048e01107d6957f5fc2f Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 24 Apr 2021 18:44:15 -0500 Subject: [PATCH 85/90] Publication depends on Validation --- .../workflows/python-publish-to-test-pypi.yml | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/python-publish-to-test-pypi.yml b/.github/workflows/python-publish-to-test-pypi.yml index 8f347bc4..b7180aa4 100644 --- a/.github/workflows/python-publish-to-test-pypi.yml +++ b/.github/workflows/python-publish-to-test-pypi.yml @@ -3,11 +3,6 @@ name: Upload Python TEST Package on: workflow_dispatch: - workflow_run: - workflows: - - build - types: - - completed push: branches: - development @@ -21,29 +16,34 @@ on: - release/* jobs: - test-publish: + validate: + name: Code Quality Assessment + runs-on: ubuntu-latest + uses: wwkimball/build@development + + publish: name: Publish to TEST PyPI - if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest environment: 'PyPI: Test' + needs: validate steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Build Tools - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel - - name: Build Artifacts - run: | - sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py - python setup.py sdist bdist_wheel - - name: Publish Artifacts - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install Build Tools + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools wheel + - name: Build Artifacts + run: | + sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py + python setup.py sdist bdist_wheel + - name: Publish Artifacts + uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ From bffb79abb9c00b7794b3a2b4dcdd2122a9a26870 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 24 Apr 2021 19:32:10 -0500 Subject: [PATCH 86/90] Try to force validation before publication... --- .../workflows/python-publish-to-prod-pypi.yml | 43 +++++++++-- .../workflows/python-publish-to-test-pypi.yml | 72 +++++++++++++------ 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/.github/workflows/python-publish-to-prod-pypi.yml b/.github/workflows/python-publish-to-prod-pypi.yml index 8ae1cb8c..22950e5b 100644 --- a/.github/workflows/python-publish-to-prod-pypi.yml +++ b/.github/workflows/python-publish-to-prod-pypi.yml @@ -2,11 +2,6 @@ name: Upload PRODUCTION Python Package on: - workflow_run: - workflows: - - build - types: - - completed push: branches: - master @@ -14,11 +9,49 @@ on: - v* jobs: + validate: + name: Code Quality Assessment + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set Environment Variables + run: | + echo "${HOME}/.gem/ruby/2.7.0/bin" >> $GITHUB_PATH + - name: Install dependencies + run: | + gem install --user-install hiera-eyaml -v 2.1.0 + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install --upgrade wheel + python -m pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257 + python -m pip install --editable . + - name: Validate Compliance with PEP257 + run: | + pep257 yamlpath + - name: Validate Compliance with MyPY + run: | + mypy yamlpath + - name: Lint with pylint + run: | + pylint yamlpath + - name: Unit Test with pytest + run: | + pytest --verbose --cov=yamlpath --cov-report=term-missing --cov-fail-under=100 --script-launch-mode=subprocess tests + publish: name: Publish to Production PyPI if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest environment: 'PyPI: Production' + needs: validate steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-publish-to-test-pypi.yml b/.github/workflows/python-publish-to-test-pypi.yml index b7180aa4..afd9b2e8 100644 --- a/.github/workflows/python-publish-to-test-pypi.yml +++ b/.github/workflows/python-publish-to-test-pypi.yml @@ -19,7 +19,39 @@ jobs: validate: name: Code Quality Assessment runs-on: ubuntu-latest - uses: wwkimball/build@development + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set Environment Variables + run: | + echo "${HOME}/.gem/ruby/2.7.0/bin" >> $GITHUB_PATH + - name: Install dependencies + run: | + gem install --user-install hiera-eyaml -v 2.1.0 + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install --upgrade wheel + python -m pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257 + python -m pip install --editable . + - name: Validate Compliance with PEP257 + run: | + pep257 yamlpath + - name: Validate Compliance with MyPY + run: | + mypy yamlpath + - name: Lint with pylint + run: | + pylint yamlpath + - name: Unit Test with pytest + run: | + pytest --verbose --cov=yamlpath --cov-report=term-missing --cov-fail-under=100 --script-launch-mode=subprocess tests publish: name: Publish to TEST PyPI @@ -28,22 +60,22 @@ jobs: needs: validate steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Build Tools - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel - - name: Build Artifacts - run: | - sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py - python setup.py sdist bdist_wheel - - name: Publish Artifacts - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install Build Tools + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools wheel + - name: Build Artifacts + run: | + sed -i -r -e "s/(^__version__[[:space:]]*=[[:space:]]*)("'"'"[[:digit:]](\.[[:digit:]])+)"'"'"/\1\2.RC$(date "+%Y%m%d%H%M%S")"'"'"/" yamlpath/__init__.py + python setup.py sdist bdist_wheel + - name: Publish Artifacts + uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ From bff12fbf9c775049544c79054c3caeff61b2d348 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sat, 24 Apr 2021 20:14:45 -0500 Subject: [PATCH 87/90] Clarify pip/setuptools dependencies --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0dab0349..4b4f50f3 100644 --- a/README.md +++ b/README.md @@ -266,37 +266,71 @@ versions of `pip` or its own dependency, *setuptools*. ### Using pip -Each published version of this project and its dependencies can be installed -from [PyPI](https://pypi.org/) using `pip`. Note that on systems with more than -one version of Python, you will probably need to use `pip3`, or equivalent -(e.g.: Cygwin users may need to use `pip3.6`, `pip3.9`, or such). +Like most others, this Python project is published to [PyPI](https://pypi.org/) +so that it can be easily installed via Python's `pip` command (or its +version-specific `pip3`, `pip3.7`, and such depending on how your Python was +installed). + +Python's `pip` command is ever-changing. It is updated very frequently. This +command further depends on other libraries to do its job, namely *setuptools*. +It so happens that *setuptools* is also updated very frequently. Both of these +are separate from Python itself, despite versions of them being pre-installed +with Python. It is your responsibility to keep `pip` and *setuptools* +up-to-date. When `pip` or *setuptools* become outdated, _you will experience +errors_ when trying to install newer Python packages like *yamlpath* **unless +you preinstall such packages' dependencies**. In the case of *yamlpath*, this +means you'd need to preinstall *ruamel.yaml* if you cannot or choose not to +upgrade `pip` and/or *setuptools*. + +As long as your `pip` and *setuptools* are up-to-date, installing *yamlpath* is +as simple as a single command (the "3.7" suffix to the `pip` command is +optional, depending on how your Python 3 was installed): ```shell -pip3 install yamlpath +pip3.7 install yamlpath ``` #### Very Old Versions of pip or its setuptools Dependency Very old versions of Python 3 ship with seriously outdated versions of `pip` and its *setuptools* dependency. When using versions of `pip` older than **18.1** -or *setuptools* older than version **46.4.0**, you may not be able to install -yamlpath with a single command. In this case, you have two options: either +or *setuptools* older than version **46.4.0**, you will not be able to install +*yamlpath* with a single command. In this case, you have two options: either pre-install *ruamel.yaml* before installing *yamlpath* or update `pip` and/or *setuptools* to at least the minimum required versions so `pip` can -auto-determine and install dependencies. This issue is not unique to yamlpath -because Python's ever-growing capabilities simply require periodic updates to -access. +auto-determine and install dependencies. This issue is not unique to +*yamlpath*. -When you cannot update `pip` or *setuptools*, just pre-install *ruamel.yaml* -before yamlpath, like so: +Upgrading `pip` and *setuptools* is trivially simple as long as you have +sufficient access rights to do so on your local machine. Depending on your +situation, you may need to prefix these with `sudo` and/or you may need to +substitute `python3` and `pip3` for `python` and `pip`, or even `python3.7` and +`pip3.7` (or another specific version of Python 3), respectively. To reiterate +that this project requires Python 3, these sample commands will be +demonstrated using such prefixes: ```shell -# In this edge-case, these commands CANNOT be joined, like: -# pip3.6 install ruamel.yaml yamlpath -pip3.6 install ruamel.yaml -pip3.6 install yamlpath +python3.7 -m pip install --upgrade pip +pip3.7 install --upgrade setuptools ``` +When you cannot or will not update `pip` or *setuptools*, just pre-install +*ruamel.yaml* before yamlpath. Each must be installed seperately and in order, +like this (you **cannot** combine these installations into a single command): + +```shell +pip3.7 install ruamel.yaml +pip3.7 install yamlpath +``` + +The downside to choosing this manual installation path is that you may end up +with an incompatible version of *ruamel.yaml*. This will manifest either as an +inability to install *yamlpath* at all, or only certain versions of *yamlpath*, +or *yamlpath* may experience unexpected errors caused by the incompatible code. +For the best experience, you are strongly encouraged to just keep `pip` and +*setuptools* up-to-date, particularly as a routine part of installing any new +Python packages. + ### Installing EYAML (Optional) EYAML support is entirely optional. You do not need EYAML to use YAML Path. From 8a29c367c33b3d532ba4e6edf43e79e41f4bbd61 Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 25 Apr 2021 17:33:10 -0500 Subject: [PATCH 88/90] Internal documentation update --- CHANGES | 10 +- yamlpath/common/anchors.py | 19 +- yamlpath/common/keywordsearches.py | 145 +++++++++++- yamlpath/common/nodes.py | 48 +++- yamlpath/common/parsers.py | 4 +- yamlpath/common/searches.py | 23 +- yamlpath/differ/diffentry.py | 46 +++- yamlpath/differ/differ.py | 170 ++++++++++++-- yamlpath/differ/differconfig.py | 2 +- yamlpath/enums/pathsearchkeywords.py | 4 + yamlpath/exceptions/yamlpathexception.py | 7 +- yamlpath/eyaml/eyamlprocessor.py | 65 +++--- yamlpath/merger/exceptions/mergeexception.py | 6 +- yamlpath/merger/merger.py | 16 +- yamlpath/merger/mergerconfig.py | 28 ++- yamlpath/path/collectorterms.py | 7 +- yamlpath/path/searchkeywordterms.py | 11 +- yamlpath/path/searchterms.py | 6 +- yamlpath/processor.py | 230 +++++++++++++------ yamlpath/types/__init__.py | 1 + yamlpath/types/ancestryentry.py | 8 + yamlpath/types/pathattributes.py | 6 +- yamlpath/types/pathsegment.py | 6 +- yamlpath/wrappers/consoleprinter.py | 2 +- yamlpath/wrappers/nodecoords.py | 33 ++- yamlpath/yamlpath.py | 24 +- 26 files changed, 724 insertions(+), 203 deletions(-) create mode 100644 yamlpath/types/ancestryentry.py diff --git a/CHANGES b/CHANGES index c53dbd59..0092b687 100644 --- a/CHANGES +++ b/CHANGES @@ -36,11 +36,11 @@ Enhancements: human-friendly, the unescaped paths will not be suitable for use as YAML Path input to other YAML Path processors where special symbols require escaping. * [API] The NodeCoords class now tracks ancestry and the last YAML Path segment - responsible for triggering its generation. The ancestry stack was necessary - to support the [parent()] Search Keyword. The responsible YAML Path segment - tracking was necessary to enable Hash/map/dict key renaming via the [name()] - Search Keyword. These must be set when the NodeCoords is generated; it is - not automatic. + responsible for triggering its generation. The ancestry stack -- + List[AncestryEntry] -- was necessary to support the [parent()] Search + Keyword. The responsible YAML Path segment tracking was necessary to enable + Hash/map/dict key renaming via the [name()] Search Keyword. These optional + attributes may be set when the NodeCoords is generated. * [API] YAMLPath instances now have a pop() method. This mutates the YAMLPath by popping off its last segment, returning that segment. diff --git a/yamlpath/common/anchors.py b/yamlpath/common/anchors.py index 0e59dfeb..13973d73 100644 --- a/yamlpath/common/anchors.py +++ b/yamlpath/common/anchors.py @@ -90,8 +90,14 @@ def replace_merge_anchor(data: Any, old_node: Any, repl_node: Any) -> None: data.merge[midx] = (data.merge[midx][0], repl_node) @staticmethod - def combine_merge_anchors(lhs: CommentedMap, rhs: CommentedMap): - """Merge YAML merge keys.""" + def combine_merge_anchors(lhs: CommentedMap, rhs: CommentedMap) -> None: + """ + Merge YAML merge keys. + + Parameters: + 1. lhs (CommentedMap) The map to merge into + 2. rhs (CommentedMap) The map to merge from + """ for mele in rhs.merge: lhs.add_yaml_merge([mele]) @@ -172,7 +178,14 @@ def generate_unique_anchor_name( @staticmethod def get_node_anchor(node: Any) -> Optional[str]: - """Return a node's Anchor/Alias name or None wheh there isn't one.""" + """ + Return a node's Anchor/Alias name or None when there isn't one. + + Parameters: + 1. node (Any) The node to evaluate + + Returns: (str) The node's Anchor/Alias name or None when unset + """ if ( not hasattr(node, "anchor") or node.anchor is None diff --git a/yamlpath/common/keywordsearches.py b/yamlpath/common/keywordsearches.py index 56649260..e9c9bab8 100644 --- a/yamlpath/common/keywordsearches.py +++ b/yamlpath/common/keywordsearches.py @@ -8,7 +8,7 @@ """ from typing import Any, Generator, List -from yamlpath.types import PathSegment +from yamlpath.types import AncestryEntry, PathSegment from yamlpath.enums import PathSearchKeywords, PathSearchMethods from yamlpath.path import SearchKeywordTerms from yamlpath.exceptions import YAMLPathException @@ -24,7 +24,19 @@ def search_matches( terms: SearchKeywordTerms, haystack: Any, yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Perform a keyword search.""" + """ + Perform a keyword search. + + Parameters: + 1. terms (SearchKeywordTerms) The search operation to perform + 2. haystack (Any) The data to evaluate + 3. yaml_path (YAMLPath) YAML Path containing this search keyword + + Keyword Arguments: See each of the called KeywordSearches methods + + Returns: (Generator[NodeCoords, None, None]) Matching data as it is + generated + """ invert: bool = terms.inverted keyword: PathSearchKeywords = terms.keyword parameters: List[str] = terms.parameters @@ -59,11 +71,33 @@ def has_child( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Indicate whether data has a named child.""" + """ + Indicate whether data has a named child. + + Parameters: + 1. data (Any) The data to evaluate + 2. invert (bool) Invert the evaluation + 3. parameters (List[str]) Parsed parameters + 4. yaml_path (YAMLPath) YAML Path begetting this operation + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + + Returns: (Generator[NodeCoords, None, None]) each result as it is + generated + """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There must be exactly one parameter @@ -132,11 +166,32 @@ def name( invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Match only the key-name of the present node.""" + """ + Match only the key-name of the present node. + + Parameters: + 1. invert (bool) Invert the evaluation + 2. parameters (List[str]) Parsed parameters + 3. yaml_path (YAMLPath) YAML Path begetting this operation + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + + Returns: (Generator[NodeCoords, None, None]) each result as it is + generated + """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There are no parameters @@ -164,11 +219,33 @@ def max( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Find whichever nodes/elements have a maximum value.""" + """ + Find whichever nodes/elements have a maximum value. + + Parameters: + 1. data (Any) The data to evaluate + 2. invert (bool) Invert the evaluation + 3. parameters (List[str]) Parsed parameters + 4. yaml_path (YAMLPath) YAML Path begetting this operation + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + + Returns: (Generator[NodeCoords, None, None]) each result as it is + generated + """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There may be 0 or 1 parameters @@ -345,11 +422,33 @@ def min( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Find whichever nodes/elements have a minimum value.""" + """ + Find whichever nodes/elements have a minimum value. + + Parameters: + 1. data (Any) The data to evaluate + 2. invert (bool) Invert the evaluation + 3. parameters (List[str]) Parsed parameters + 4. yaml_path (YAMLPath) YAML Path begetting this operation + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + + Returns: (Generator[NodeCoords, None, None]) each result as it is + generated + """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There may be 0 or 1 parameters @@ -525,11 +624,33 @@ def parent( data: Any, invert: bool, parameters: List[str], yaml_path: YAMLPath, **kwargs: Any ) -> Generator[NodeCoords, None, None]: - """Climb back up N parent levels in the data hierarchy.""" + """ + Climb back up N parent levels in the data hierarchy. + + Parameters: + 1. data (Any) The data to evaluate + 2. invert (bool) Invert the evaluation; not possible for parent() + 3. parameters (List[str]) Parsed parameters + 4. yaml_path (YAMLPath) YAML Path begetting this operation + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + + Returns: (Generator[NodeCoords, None, None]) each result as it is + generated + """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) # There may be 0 or 1 parameters diff --git a/yamlpath/common/nodes.py b/yamlpath/common/nodes.py index 12914cf3..d69caa0c 100644 --- a/yamlpath/common/nodes.py +++ b/yamlpath/common/nodes.py @@ -285,8 +285,9 @@ def wrap_type(value: Any) -> Any: return wrapped_value @staticmethod - def build_next_node(yaml_path: YAMLPath, depth: int, - value: Any = None) -> Any: + def build_next_node( + yaml_path: YAMLPath, depth: int, value: Any = None + ) -> Any: """ Get the best default value for the next entry in a YAML Path. @@ -313,8 +314,9 @@ def build_next_node(yaml_path: YAMLPath, depth: int, return default_value @staticmethod - def append_list_element(data: Any, value: Any = None, - anchor: str = None) -> Any: + def append_list_element( + data: Any, value: Any = None, anchor: str = None + ) -> Any: """ Append a new element to an ruamel.yaml List. @@ -369,7 +371,7 @@ def apply_yaml_tag(node: Any, value_tag: str) -> Any: 3. value_tag (str) Tag to apply (or None to remove) Returns: (Any) the updated node; may be new data, so replace your node - with this returned value! + with this returned value! """ if value_tag is None: return node @@ -393,12 +395,26 @@ def apply_yaml_tag(node: Any, value_tag: str) -> Any: @staticmethod def node_is_leaf(node: Any) -> bool: - """Indicate whether a node is a leaf (Scalar data).""" + """ + Indicate whether a node is a leaf (Scalar data). + + Parameters: + 1. node (Any) The node to evaluate + + Returns: (bool) True = node is a leaf; False, otherwise + """ return not isinstance(node, (dict, list, set)) @staticmethod def node_is_aoh(node: Any) -> bool: - """Indicate whether a node is an Array-of-Hashes (List of Dicts).""" + """ + Indicate whether a node is an Array-of-Hashes (List of Dicts). + + Parameters: + 1. node (Any) The node under evaluation + + Returns: (bool) True = node is a `list` comprised **only** of `dict`s + """ if node is None: return False @@ -413,7 +429,14 @@ def node_is_aoh(node: Any) -> bool: @staticmethod def tagless_elements(data: list) -> list: - """Get a copy of a list with all elements stripped of YAML Tags.""" + """ + Get a copy of a list with all elements stripped of YAML Tags. + + Parameters: + 1. data (list) The list to strip of YAML Tags + + Returns: (list) De-tagged version of `data` + """ detagged = [] for ele in data: if isinstance(ele, TaggedScalar): @@ -424,7 +447,14 @@ def tagless_elements(data: list) -> list: @staticmethod def tagless_value(value: Any) -> Any: - """Get a value in its true data-type, stripped of any YAML Tag.""" + """ + Get a value in its true data-type, stripped of any YAML Tag. + + Parameters: + 1. value (Any) The value to de-tag + + Returns: (Any) The de-tagged value + """ evalue = value if isinstance(value, TaggedScalar): evalue = value.value diff --git a/yamlpath/common/parsers.py b/yamlpath/common/parsers.py index a6c37799..b1feb94d 100644 --- a/yamlpath/common/parsers.py +++ b/yamlpath/common/parsers.py @@ -79,7 +79,7 @@ def get_yaml_data( 3. source (str) The source file or serialized literal to load; can be - for reading from STDIN (implies literal=True) - Keyword Parameters: + Keyword Arguments: * literal (bool) `source` is literal serialized YAML data rather than a file-spec, so load it directly @@ -177,7 +177,7 @@ def get_yaml_multidoc_data( 3. source (str) The source file to load; can be - for reading from STDIN - Keyword Parameters: + Keyword Arguments: * literal (bool) `source` is literal serialized YAML data rather than a file-spec, so load it directly diff --git a/yamlpath/common/searches.py b/yamlpath/common/searches.py index a4ebb012..5f912ae6 100644 --- a/yamlpath/common/searches.py +++ b/yamlpath/common/searches.py @@ -24,7 +24,19 @@ class Searches: def search_matches( method: PathSearchMethods, needle: str, haystack: Any ) -> bool: - """Perform a search.""" + """ + Perform a search comparison. + + NOTE: For less-than, greather-than and related operations, the test is + whether `haystack` is less/greater-than `needle`. + + Parameters: + 1. method (PathSearchMethods) The search method to employ + 2. needle (str) The value to look for. + 3. haystack (Any) The value to look in. + + Returns: (bool) True = comparision passes; False = comparison fails. + """ try: cased_needle = needle lower_needle = str(needle).lower() @@ -169,7 +181,14 @@ def search_anchor( def create_searchterms_from_pathattributes( rhs: PathAttributes ) -> SearchTerms: - """Convert a PathAttributes instance to a SearchTerms instance.""" + """ + Convert a PathAttributes instance to a SearchTerms instance. + + Parameters: + 1. rhs (PathAttributes) PathAttributes instance to convert + + Returns: (SearchTerms) SearchTerms extracted from `rhs` + """ if isinstance(rhs, SearchTerms): newinst: SearchTerms = SearchTerms( rhs.inverted, rhs.method, rhs.attribute, rhs.term diff --git a/yamlpath/differ/diffentry.py b/yamlpath/differ/diffentry.py index 22652cdc..383c6265 100644 --- a/yamlpath/differ/diffentry.py +++ b/yamlpath/differ/diffentry.py @@ -21,7 +21,27 @@ def __init__( self, action: DiffActions, path: YAMLPath, lhs: Any, rhs: Any, **kwargs ): - """Initiate a new DiffEntry.""" + """ + Instantiate a new DiffEntry. + + Parameters: + 1. action (DiffAction) The action taken for one document to become the + next + 2. path (YAMLPath) Location within the LHS document which changes to + becomes the RHS document + 3. lhs (Any) The Left-Hand-Side (original) document + 4. rhs (Any) The Right-Hand-Side (altered) document + + Keyword Arguments: + * lhs_iteration (Any) "Rough" position of the original element within + its document before it was changed + * lhs_parent (Any) Parent of the original data element + * rhs_iteration (Any) "Rough" position of the changed element within + its document, if it existed before the change (otherwise it'll be 0s) + * rhs_parent (Any) Parent of the changed data element + + Returns: N/A + """ self._action: DiffActions = action self._path: YAMLPath = path self._lhs: Any = lhs @@ -30,8 +50,24 @@ def __init__( self._set_index(lhs, rhs, **kwargs) self._verbose = False - def _set_index(self, lhs: Any, rhs: Any, **kwargs) -> Any: - """Build the sortable index for this entry.""" + def _set_index(self, lhs: Any, rhs: Any, **kwargs) -> None: + """ + Build the sortable index for this entry. + + Parameters: + 1. lhs (Any) The Left-Hand-Side (original) document + 2. rhs (Any) The Right-Hand-Side (altered) document + + Keyword Arguments: + * lhs_iteration (Any) "Rough" position of the original element within + its document before it was changed + * lhs_parent (Any) Parent of the original data element + * rhs_iteration (Any) "Rough" position of the changed element within + its document, if it existed before the change (otherwise it'll be 0s) + * rhs_parent (Any) Parent of the changed data element + + Returns: N/A + """ lhs_lc = DiffEntry._get_index(lhs, kwargs.pop("lhs_parent", None)) rhs_lc = DiffEntry._get_index(rhs, kwargs.pop("rhs_parent", None)) lhs_iteration = kwargs.pop("lhs_iteration", 0) @@ -136,8 +172,8 @@ def _present_data(cls, data: Any, prefix: str) -> str: formatted_data = json_safe_data if isinstance(json_safe_data, str): formatted_data = json_safe_data.strip() - json_data = json.dumps(formatted_data).replace( - "\\n", "\n{} ".format(prefix)) + json_data = json.dumps( + formatted_data).replace("\\n", "\n{} ".format(prefix)) data_tag = "" if isinstance(data, TaggedScalar) and data.tag.value: data_tag = "{} ".format(data.tag.value) diff --git a/yamlpath/differ/differ.py b/yamlpath/differ/differ.py index 49ac55f4..74b68ffb 100644 --- a/yamlpath/differ/differ.py +++ b/yamlpath/differ/differ.py @@ -30,6 +30,10 @@ def __init__( 1. logger (ConsolePrinter) Instance of ConsoleWriter or subclass 2. document (Any) The basis document + Keyword Arguments: + * ignore_eyaml_values (bool) Do not decrypt encrypted YAML value for + comparison + Returns: N/A Raises: N/A @@ -46,20 +50,41 @@ def __init__( else EYAMLProcessor(logger, document, **kwargs)) def compare_to(self, document: Any) -> None: - """Perform the diff calculation.""" + """ + Perform the diff calculation. + + Parameers: + 1. document (Any) The document to compare against + + Returns: N/A + """ self._diffs.clear() self.config.prepare(document) self._diff_between(YAMLPath(), self._data, document) def get_report(self) -> Generator[DiffEntry, None, None]: - """Get the diff report.""" + """ + Get the diff report. + + Parameters: N/A + + Returns: (Generator[DiffEntry, None, None]) Sorted DiffEntry records + """ for entry in sorted( self._diffs, key=lambda e: [int(i) for i in e.index.split('.')] ): yield entry - def _purge_document(self, path: YAMLPath, data: Any): - """Delete every node in the document.""" + def _purge_document(self, path: YAMLPath, data: Any) -> None: + """ + Record changes necessary to delete every node in the document. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. data (Any) The DOM element under evaluation + + Returns: N/A + """ if isinstance(data, CommentedMap): lhs_iteration = -1 for key, val in data.items(): @@ -84,7 +109,15 @@ def _purge_document(self, path: YAMLPath, data: Any): ) def _add_everything(self, path: YAMLPath, data: Any) -> None: - """Add every node in the document.""" + """ + Record changes necessary to add every node in the document. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. data (Any) The DOM element under evaluation + + Returns: N/A + """ if isinstance(data, CommentedMap): rhs_iteration = -1 for key, val in data.items(): @@ -111,7 +144,18 @@ def _add_everything(self, path: YAMLPath, data: Any) -> None: def _diff_scalars( self, path: YAMLPath, lhs: Any, rhs: Any, **kwargs ) -> None: - """Diff two Scalar values.""" + """ + Diff two Scalar values. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Keyword Arguments: See `DiffEntry` + + Returns: N/A + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_scalars: ", @@ -145,7 +189,14 @@ def _diff_scalars( def _diff_dicts( self, path: YAMLPath, lhs: CommentedMap, rhs: CommentedMap ) -> None: - """Diff two dicts.""" + """ + Diff two dicts. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_dicts: ", @@ -223,7 +274,20 @@ def _diff_dicts( def _diff_synced_lists( self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq ) -> None: - """Diff two synchronized lists.""" + """ + Diff two synchronized lists. + + A "synchronized" list -- in this context -- is one in which all + elements that are identical to those of its exemplar list are + (re)positioned to identical index. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Returns: N/A + """ self.logger.debug("Differ::_diff_synced_lists: Starting...") self.logger.debug( "Synchronizing LHS Array elements at YAML Path, {}:" @@ -286,7 +350,19 @@ def _diff_arrays_of_scalars( self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq, node_coord: NodeCoords, **kwargs ) -> None: - """Diff two lists of scalars.""" + """ + Diff two lists of scalars. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + 4. node_coord (NodeCoords) The node being evaluated + + Keyword Parameers: + * diff_deeply (bool) True = Deeply traverse complex elements; False = + compare complex elements as-is + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_arrays_of_scalars: ", @@ -336,7 +412,17 @@ def _diff_arrays_of_hashes( self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq, node_coord: NodeCoords ) -> None: - """Diff two lists-of-dictionaries.""" + """ + Diff two lists-of-dictionaries. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + 4. node_coord (NodeCoords) The node being evaluated + + Returns: N/A + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_arrays_of_hashes: ", @@ -418,7 +504,20 @@ def _diff_arrays_of_hashes( def _diff_lists( self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq, **kwargs ) -> None: - """Diff two lists.""" + """ + Diff two lists. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Keyword Arguments: + * rhs_parent (Any) Parent data node of rhs + * parentref (Any) Reference indicating rhs within rhs_parent + + Returns: N/A + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_lists: ", @@ -442,7 +541,16 @@ def _diff_lists( def _diff_between( self, path: YAMLPath, lhs: Any, rhs: Any, **kwargs ) -> None: - """Calculate the differences between two document nodes.""" + """ + Calculate the differences between two document nodes. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Keyword Arguments: See _diff_lists() and _diff_scalars() + """ self.logger.debug( "Comparing LHS:", prefix="Differ::_diff_between: ", @@ -481,7 +589,17 @@ def synchronize_lists_by_value( ) -> List[Tuple[ Optional[int], Optional[Any], Optional[int], Optional[Any] ]]: - """Synchronize two lists by value.""" + """ + Synchronize two lists by value. + + Parameters: + 1. lhs (Any) The left-hand-side (original) document + 2. rhs (Any) The right-hand-side (altered) document + + Returns: (List[Tuple[ + Optional[int], Optional[Any], Optional[int], Optional[Any] + ]]) List with LHS and RHS elements at identical elements + """ # Build a parallel index array to track the original RHS element # indexes of any surviving elements. rhs_reduced = [] @@ -517,7 +635,19 @@ def synchronize_lods_by_key( ) -> List[Tuple[ Optional[int], Optional[Any], Optional[int], Optional[Any] ]]: - """Synchronize two lists-of-dictionaries by identity key.""" + """ + Synchronize two lists-of-dictionaries by identity key. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Returns: (List[Tuple[ + Optional[int], Optional[Any], Optional[int], Optional[Any] + ]]) List of identical LHS and RHS elements in the same element + positions + """ # Build a parallel index array to track the original RHS element # indexes of any surviving elements. key_attr: str = "" @@ -591,7 +721,17 @@ def synchronize_lods_by_key( @classmethod def _get_key_indicies(cls, data: CommentedMap) -> Dict[Any, int]: - """Get a dictionary mapping of keys to relative positions.""" + """ + Get a dictionary mapping of keys to relative positions. + + Parameters: + 1. path (YAMLPath) YAML Path to the document element under evaluation + 2. lhs (Any) The left-hand-side (original) document + 3. rhs (Any) The right-hand-side (altered) document + + Returns: (Dict[Any, int]) Dictionary indicating the element position + of each key + """ key_map = {} if isinstance(data, CommentedMap): for idx, key in enumerate(data.keys()): diff --git a/yamlpath/differ/differconfig.py b/yamlpath/differ/differconfig.py index fda1b295..ab8249a2 100644 --- a/yamlpath/differ/differconfig.py +++ b/yamlpath/differ/differconfig.py @@ -1,5 +1,5 @@ """ -Config file processor for the Differ. +Implements DifferConfig. Copyright 2020 William W. Kimball, Jr. MBA MSIS """ diff --git a/yamlpath/enums/pathsearchkeywords.py b/yamlpath/enums/pathsearchkeywords.py index 42d7fe9f..de9af170 100644 --- a/yamlpath/enums/pathsearchkeywords.py +++ b/yamlpath/enums/pathsearchkeywords.py @@ -15,20 +15,24 @@ class PathSearchKeywords(Enum): `HAS_CHILD` Matches when the node has a direct child with a given name. + `NAME` Matches only the key-name or element-index of the present node, discarding any and all child node data. Can be used to rename the matched key as long as the new name is unique within the parent, lest the preexisting node be overwritten. Cannot be used to reassign an Array/sequence/list element to another position. + `MAX` Matches whichever node(s) has/have the maximum value for a named child key or the maximum value within an Array/sequence/list. When used against a scalar value, that value is always its own maximum. + `MIN` Matches whichever node(s) has/have the minimum value for a named child key or the minimum value within an Array/sequence/list. When used against a scalar value, that value is always its own minimum. + `PARENT` Access the parent(s) of the present node. """ diff --git a/yamlpath/exceptions/yamlpathexception.py b/yamlpath/exceptions/yamlpathexception.py index 762d0b81..57e496c7 100644 --- a/yamlpath/exceptions/yamlpathexception.py +++ b/yamlpath/exceptions/yamlpathexception.py @@ -1,5 +1,5 @@ """ -Express an issue with a YAML Path. +Implement YAMLPathException. Copyright 2019, 2020 William W. Kimball, Jr. MBA MSIS """ @@ -14,8 +14,9 @@ class YAMLPathException(Exception): YAML node. """ - def __init__(self, user_message: str, yaml_path: str, - segment: Optional[str] = None) -> None: + def __init__( + self, user_message: str, yaml_path: str, segment: Optional[str] = None + ) -> None: """ Initialize this Exception with all pertinent data. diff --git a/yamlpath/eyaml/eyamlprocessor.py b/yamlpath/eyaml/eyamlprocessor.py index 826bd259..3e652a6a 100644 --- a/yamlpath/eyaml/eyamlprocessor.py +++ b/yamlpath/eyaml/eyamlprocessor.py @@ -22,8 +22,9 @@ class EYAMLProcessor(Processor): """Extend Processor to understand EYAML values.""" - def __init__(self, logger: ConsolePrinter, data: Any, - **kwargs: Optional[str]) -> None: + def __init__( + self, logger: ConsolePrinter, data: Any, **kwargs: Optional[str] + ) -> None: """ Instantiate an EYAMLProcessor. @@ -35,15 +36,15 @@ def __init__(self, logger: ConsolePrinter, data: Any, Parameters: 1. logger (ConsolePrinter) Instance of ConsolePrinter or subclass 2. data (Any) Parsed YAML data - 3. **kwargs (Optional[str]) can contain the following keyword - parameters: - * binary (str) The external eyaml command to use when performing - data encryption or decryption; if no path is provided, the - command will be sought on the system PATH. Defaut="eyaml" - * publickey (Optional[str]) Fully-qualified path to the public key - for use with data encryption - * privatekey (Optional[str]) Fully-qualified path to the public key - for use with data decryption + + Keyword Arguments: + * binary (str) The external eyaml command to use when performing + data encryption or decryption; if no path is provided, the + command will be sought on the system PATH. Defaut="eyaml" + * publickey (Optional[str]) Fully-qualified path to the public key + for use with data encryption + * privatekey (Optional[str]) Fully-qualified path to the public key + for use with data decryption Returns: N/A @@ -56,7 +57,7 @@ def __init__(self, logger: ConsolePrinter, data: Any, # pylint: disable=locally-disabled,too-many-branches def _find_eyaml_paths( - self, data: Any, build_path: str = "" + self, data: Any, build_path: str = "" ) -> Generator[YAMLPath, None, None]: """ Find every encrypted value and report each as a YAML Path. @@ -65,8 +66,8 @@ def _find_eyaml_paths( leading to an EYAML value within the evaluated YAML data. Parameters: - 1. data (Any) The parsed YAML data to process - 2. build_path (str) A YAML Path under construction + 1. data (Any) The parsed YAML data to process + 2. build_path (str) A YAML Path under construction Returns: (Generator[Path, None, None]) each YAML Path entry as they are discovered @@ -122,10 +123,10 @@ def decrypt_eyaml(self, value: str) -> str: 1. value (str) The EYAML value to decrypt Returns: (str) The decrypted value or the original value if it was not - actually encrypted. + actually encrypted. Raises: - - `EYAMLCommandException` when the eyaml binary cannot be utilized + - `EYAMLCommandException` when the eyaml binary cannot be utilized """ if not self.is_eyaml_value(value): return value @@ -176,9 +177,10 @@ def decrypt_eyaml(self, value: str) -> str: return retval - def encrypt_eyaml(self, value: str, - output: EYAMLOutputFormats = EYAMLOutputFormats.STRING - ) -> str: + def encrypt_eyaml( + self, value: str, + output: EYAMLOutputFormats = EYAMLOutputFormats.STRING + ) -> str: """ Encrypt a value via EYAML. @@ -187,10 +189,10 @@ def encrypt_eyaml(self, value: str, 2. output (EYAMLOutputFormats) the output format of the encryption Returns: (str) The encrypted result or the original value if it was - already an EYAML encryption. + already an EYAML encryption. Raises: - - `EYAMLCommandException` when the eyaml binary cannot be utilized. + - `EYAMLCommandException` when the eyaml binary cannot be utilized. """ if self.is_eyaml_value(value): return value @@ -248,9 +250,11 @@ def encrypt_eyaml(self, value: str, ) return retval - def set_eyaml_value(self, yaml_path: YAMLPath, value: str, - output: EYAMLOutputFormats = EYAMLOutputFormats.STRING, - mustexist: bool = False) -> None: + def set_eyaml_value( + self, yaml_path: YAMLPath, value: str, + output: EYAMLOutputFormats = EYAMLOutputFormats.STRING, + mustexist: bool = False + ) -> None: """ Encrypt and store a value where specified via YAML Path. @@ -265,7 +269,7 @@ def set_eyaml_value(self, yaml_path: YAMLPath, value: str, Returns: N/A Raises: - - `YAMLPathException` when YAML Path is invalid + - `YAMLPathException` when YAML Path is invalid """ self.logger.verbose( "Encrypting value(s) for {}." @@ -283,9 +287,10 @@ def set_eyaml_value(self, yaml_path: YAMLPath, value: str, value_format=emit_format ) - def get_eyaml_values(self, yaml_path: YAMLPath, mustexist: bool = False, - default_value: str = "" - ) -> Generator[str, None, None]: + def get_eyaml_values( + self, yaml_path: YAMLPath, mustexist: bool = False, + default_value: str = "" + ) -> Generator[str, None, None]: """ Retrieve and decrypt all EYAML nodes identified via a YAML Path. @@ -302,7 +307,7 @@ def get_eyaml_values(self, yaml_path: YAMLPath, mustexist: bool = False, specifies a non-existant node Raises: - - `YAMLPathException` when YAML Path is invalid + - `YAMLPathException` when YAML Path is invalid """ self.logger.verbose( "Decrypting value(s) at {}.".format(yaml_path) @@ -319,7 +324,7 @@ def _can_run_eyaml(self) -> bool: Parameters: N/A Returns: (bool) True when the present eyaml property indicates an - executable; False, otherwise + executable; False, otherwise Raises: N/A """ diff --git a/yamlpath/merger/exceptions/mergeexception.py b/yamlpath/merger/exceptions/mergeexception.py index b2eacfde..aac817fc 100644 --- a/yamlpath/merger/exceptions/mergeexception.py +++ b/yamlpath/merger/exceptions/mergeexception.py @@ -11,8 +11,10 @@ class MergeException(Exception): """Express an issue with a document merge.""" - def __init__(self, user_message: str, - yaml_path: Optional[Union[YAMLPath, str]] = None) -> None: + def __init__( + self, user_message: str, + yaml_path: Optional[Union[YAMLPath, str]] = None + ) -> None: """ Initialize this Exception with all pertinent data. diff --git a/yamlpath/merger/merger.py b/yamlpath/merger/merger.py index cab4ee6d..0c36e90b 100644 --- a/yamlpath/merger/merger.py +++ b/yamlpath/merger/merger.py @@ -28,14 +28,14 @@ class Merger: """Performs YAML document merges.""" - DEPRECATION_WARNING = ("WARNING: Deprecated methods will be removed in" - " the next major release of yamlpath. Please refer" - " to the CHANGES file for more information (and how" - " to get rid of this message).") + DEPRECATION_WARNING = ( + "WARNING: Deprecated methods will be removed in the next major" + " release of yamlpath. Please refer to the CHANGES file for more" + " information (and how to get rid of this message).") depwarn_printed = False def __init__( - self, logger: ConsolePrinter, lhs: Any, config: MergerConfig + self, logger: ConsolePrinter, lhs: Any, config: MergerConfig ) -> None: """ Instantiate this class into an object. @@ -107,7 +107,7 @@ def _merge_dicts( 3. path (YAMLPath) Location within the DOM where this merge is taking place. - Keyword Parameters: + Keyword Arguments: * parent (Any) Parent node of `rhs` * parentref (Any) Child Key or Index of `rhs` within `parent`. @@ -264,7 +264,7 @@ def _merge_simple_lists( 4. node_coord (NodeCoords) The RHS root node, its parent, and reference within its parent; used for config lookups. - Returns: (list) The merged result. + Returns: (CommentedSeq) The merged result. Raises: - `MergeException` when a clean merge is impossible. @@ -409,7 +409,7 @@ def _merge_lists( 2. rhs (CommentedSeq) The list to merge from. 3. path (YAMLPath) Location of the `rhs` source list within its DOM. - Keyword Parameters: + Keyword Arguments: * parent (Any) Parent node of `rhs` * parentref (Any) Child Key or Index of `rhs` within `parent`. diff --git a/yamlpath/merger/mergerconfig.py b/yamlpath/merger/mergerconfig.py index 0ef81c3e..8c9a187e 100644 --- a/yamlpath/merger/mergerconfig.py +++ b/yamlpath/merger/mergerconfig.py @@ -1,5 +1,5 @@ """ -Config file processor for the Merger. +Implement MergerConfig. Copyright 2020 William W. Kimball, Jr. MBA MSIS """ @@ -41,7 +41,14 @@ def __init__(self, logger: ConsolePrinter, args: Namespace) -> None: self._load_config() def anchor_merge_mode(self) -> AnchorConflictResolutions: - """Get Anchor merge mode.""" + """ + Get Anchor merge mode. + + Parameters: N/A + + Returns: (AnchorConflictResolutions) Resolved method for handling + YAML Anchor conflicts. + """ # Precedence: CLI > config[defaults] > default if hasattr(self.args, "anchors") and self.args.anchors: return AnchorConflictResolutions.from_str(self.args.anchors) @@ -178,13 +185,26 @@ def prepare(self, data: Any) -> None: self._prepare_user_rules(proc, merge_path, "keys", self.keys) def get_insertion_point(self) -> YAMLPath: - """Get the YAML Path at which merging shall be performed.""" + """ + Get the YAML Path at which merging shall be performed. + + Parameters: N/A + + Returns: (YAMLPath) User-specified point(s) within the document where + the RHS document is directed to be merged-in. + """ if hasattr(self.args, "mergeat"): return YAMLPath(self.args.mergeat) return YAMLPath("/") def get_document_format(self) -> OutputDocTypes: - """Get the user-desired output format.""" + """ + Get the user-desired output format. + + Paramerers: N/A + + Returns: (OutputDocTypes) The destination document type + """ if hasattr(self.args, "document_format"): return OutputDocTypes.from_str(self.args.document_format) return OutputDocTypes.AUTO diff --git a/yamlpath/path/collectorterms.py b/yamlpath/path/collectorterms.py index 409635bf..f49f77ee 100644 --- a/yamlpath/path/collectorterms.py +++ b/yamlpath/path/collectorterms.py @@ -9,9 +9,10 @@ class CollectorTerms: """YAML Path Collector segment terms.""" - def __init__(self, expression: str, - operation: CollectorOperators = CollectorOperators.NONE - ) -> None: + def __init__( + self, expression: str, + operation: CollectorOperators = CollectorOperators.NONE + ) -> None: """ Instantiate a Collector Term. diff --git a/yamlpath/path/searchkeywordterms.py b/yamlpath/path/searchkeywordterms.py index 829368a3..a80700c5 100644 --- a/yamlpath/path/searchkeywordterms.py +++ b/yamlpath/path/searchkeywordterms.py @@ -1,7 +1,7 @@ """ -YAML path Keyword Search segment terms. +Implement SearchKeywordTerms. -Copyright 2019, 2020 William W. Kimball, Jr. MBA MSIS +Copyright 2021 William W. Kimball, Jr. MBA MSIS """ from typing import List @@ -9,10 +9,11 @@ class SearchKeywordTerms: - """YAML path Keyword Search segment terms.""" + """YAML path Search Keyword segment terms.""" - def __init__(self, inverted: bool, keyword: PathSearchKeywords, - parameters: str) -> None: + def __init__( + self, inverted: bool, keyword: PathSearchKeywords, parameters: str + ) -> None: """ Instantiate a Keyword Search Term segment. diff --git a/yamlpath/path/searchterms.py b/yamlpath/path/searchterms.py index e5cd665e..3b35b001 100644 --- a/yamlpath/path/searchterms.py +++ b/yamlpath/path/searchterms.py @@ -9,8 +9,10 @@ class SearchTerms: """YAML path Search segment terms.""" - def __init__(self, inverted: bool, method: PathSearchMethods, - attribute: str, term: str) -> None: + def __init__( + self, inverted: bool, method: PathSearchMethods, attribute: str, + term: str + ) -> None: """ Instantiate a Search Term. diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 3477d464..56a93183 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -2,13 +2,13 @@ """ YAML Path processor based on ruamel.yaml. -Copyright 2018, 2019, 2020 William W. Kimball, Jr. MBA MSIS +Copyright 2018, 2019, 2020, 2021 William W. Kimball, Jr. MBA MSIS """ from typing import Any, Dict, Generator, List, Union from ruamel.yaml.comments import CommentedMap -from yamlpath.types import PathSegment +from yamlpath.types import AncestryEntry, PathSegment from yamlpath.common import Anchors, KeywordSearches, Nodes, Searches from yamlpath import YAMLPath from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms @@ -41,15 +41,16 @@ def __init__(self, logger: ConsolePrinter, data: Any) -> None: self.logger: ConsolePrinter = logger self.data: Any = data - def get_nodes(self, yaml_path: Union[YAMLPath, str], - **kwargs: Any) -> Generator[Any, None, None]: + def get_nodes( + self, yaml_path: Union[YAMLPath, str], **kwargs: Any + ) -> Generator[Any, None, None]: """ Get nodes at YAML Path in data. Parameters: 1. yaml_path (Union[YAMLPath, str]) The YAML Path to evaluate - Keyword Parameters: + Keyword Arguments: * mustexist (bool) Indicate whether yaml_path must exist in data prior to this query (lest an Exception be raised); default=False @@ -110,8 +111,9 @@ def get_nodes(self, yaml_path: Union[YAMLPath, str], prefix="Processor::get_nodes: ", data=opt_node) yield opt_node - def set_value(self, yaml_path: Union[YAMLPath, str], - value: Any, **kwargs) -> None: + def set_value( + self, yaml_path: Union[YAMLPath, str], value: Any, **kwargs + ) -> None: """ Set the value of zero or more nodes at YAML Path in YAML data. @@ -119,7 +121,7 @@ def set_value(self, yaml_path: Union[YAMLPath, str], 1. yaml_path (Union[Path, str]) The YAML Path to evaluate 2. value (Any) The value to set - Keyword Parameters: + Keyword Arguments: * mustexist (bool) Indicate whether yaml_path must exist in data prior to this query (lest an Exception be raised); default=False @@ -184,8 +186,26 @@ def set_value(self, yaml_path: Union[YAMLPath, str], def _apply_change( self, yaml_path: YAMLPath, node_coord: NodeCoords, value: Any, **kwargs: Any - ): - """Helper for set_value.""" + ) -> None: + """ + Apply a controlled change to the document via gathered NodeCoords. + + Parameters: + 1. yaml_path (YAMLPath) The YAML Path causing this change. + 2. node_coord (NodeCoords) The data node to affect. + 3. value (Any) The value to apply. + + Keyword Arguments: + * value_format (YAMLValueFormats) The demarcation or visual + representation to use when writing the data; + default=YAMLValueFormats.DEFAULT + * tag (str) Custom data-type tag to assign + + Returns: N/A + + Raises: + - YAMLPathException when the attempted change is impossible + """ value_format: YAMLValueFormats = kwargs.pop("value_format", YAMLValueFormats.DEFAULT) tag: str = kwargs.pop("tag", None) @@ -281,7 +301,7 @@ def _get_anchor_node( will result in a YAMLPathException because YAML does not define Aliases for more than one Anchor. - Keyword Parameters: + Keyword Arguments: * anchor_name (str) Alternate name to use for the YAML Anchor and its Aliases. @@ -356,7 +376,7 @@ def alias_nodes( will result in a YAMLPathException because YAML does not define Aliases for more than one Anchor. - Keyword Parameters: + Keyword Arguments: * pathsep (PathSeperators) Forced YAML Path segment seperator; set only when automatic inference fails; default = PathSeperators.AUTO @@ -404,7 +424,18 @@ def alias_gathered_nodes( Assign a YAML Anchor to zero or more YAML Alias nodes. Parameters: - 1. gathered_nodes (List[NodeCoords]) The pre-gathered nodes to assign. + 1. gathered_nodes (List[NodeCoords]) The pre-gathered nodes to assign + 2. anchor_path (Union[YAMLPath, str]) YAML Path to the source Anchor + + Keyword Arguments: + * pathsep (PathSeperators) Forced YAML Path segment seperator; set + only when automatic inference fails; + default = PathSeperators.AUTO + * anchor_name (str) Override automatic anchor name; use this, instead + + Returns: N/A + + Raises: N/A """ pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) anchor_name: str = kwargs.pop("anchor_name", "") @@ -422,7 +453,7 @@ def alias_gathered_nodes( self._alias_nodes(gathered_nodes, anchor_node) def _alias_nodes( - self, gathered_nodes: List[NodeCoords], anchor_node: Any + self, gathered_nodes: List[NodeCoords], anchor_node: Any ) -> None: """ Assign a YAML Anchor to its various YAML Alias nodes. @@ -452,7 +483,7 @@ def tag_nodes( 1. yaml_path (Union[YAMLPath, str]) The YAML Path to evaluate 2. tag (str) The tag to assign - Keyword Parameters: + Keyword Arguments: * pathsep (PathSeperators) Forced YAML Path segment seperator; set only when automatic inference fails; default = PathSeperators.AUTO @@ -460,7 +491,7 @@ def tag_nodes( Returns: N/A Raises: - - `YAMLPathException` when YAML Path is invalid + - `YAMLPathException` when YAML Path is invalid """ pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) @@ -488,7 +519,15 @@ def tag_nodes( def tag_gathered_nodes( self, gathered_nodes: List[NodeCoords], tag: str ) -> None: - """Assign a data-type tag to a set of nodes.""" + """ + Assign a data-type tag to a set of nodes. + + Parameters: + 1. gathered_nodes (List[NodeCoords]) The nodes to affect + 2. tag (str) The tag to assign + + Returns: N/A + """ # A YAML tag must be prefixed via at least one bang (!) if tag and not tag[0] == "!": tag = "!{}".format(tag) @@ -505,15 +544,16 @@ def tag_gathered_nodes( self.data, old_node, node_coord.parent[node_coord.parentref]) - def delete_nodes(self, yaml_path: Union[YAMLPath, str], - **kwargs: Any) -> Generator[NodeCoords, None, None]: + def delete_nodes( + self, yaml_path: Union[YAMLPath, str], **kwargs: Any + ) -> Generator[NodeCoords, None, None]: """ Gather and delete nodes at YAML Path in data. Parameters: 1. yaml_path (Union[YAMLPath, str]) The YAML Path to evaluate - Keyword Parameters: + Keyword Arguments: * pathsep (PathSeperators) Forced YAML Path segment seperator; set only when automatic inference fails; default = PathSeperators.AUTO @@ -567,8 +607,8 @@ def _delete_nodes(self, delete_nodes: List[NodeCoords]) -> None: 1. delete_nodes (List[NodeCoords]) The nodes to delete. Raises: - - `YAMLPathException` when the operation would destroy the entire - document + - `YAMLPathException` when the operation would destroy the entire + document """ for delete_nc in reversed(delete_nodes): node = delete_nc.node @@ -604,10 +644,9 @@ def _delete_nodes(self, delete_nodes: List[NodeCoords]) -> None: ) # pylint: disable=locally-disabled,too-many-branches,too-many-locals - def _get_nodes_by_path_segment(self, data: Any, - yaml_path: YAMLPath, segment_index: int, - **kwargs: Any - ) -> Generator[Any, None, None]: + def _get_nodes_by_path_segment( + self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs: Any + ) -> Generator[Any, None, None]: """ Get nodes identified by their YAML Path segment. @@ -625,7 +664,10 @@ def _get_nodes_by_path_segment(self, data: Any, * parentref (Any) The Index or Key of data within parent * traverse_lists (Boolean) Indicate whether KEY searches against lists are permitted to automatically traverse into the list; Default=True - * ancestry (List[tuple]) + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[Any, None, None]) Each node coordinate or list of node coordinates as they are matched. You must check with isinstance() @@ -633,14 +675,14 @@ def _get_nodes_by_path_segment(self, data: Any, List[NodeCoords]. Raises: - - `NotImplementedError` when the segment indicates an unknown - PathSegmentTypes value. + - `NotImplementedError` when the segment indicates an unknown + PathSegmentTypes value. """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) segments = yaml_path.escaped if not (segments and len(segments) > segment_index): self.logger.debug( @@ -714,8 +756,7 @@ def _get_nodes_by_path_segment(self, data: Any, yield node_coord def _get_nodes_by_key( - self, data: Any, yaml_path: YAMLPath, segment_index: int, - **kwargs: Any + self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs: Any ) -> Generator[NodeCoords, None, None]: """ Get nodes from a Hash by their unique key name. @@ -731,6 +772,10 @@ def _get_nodes_by_key( Keyword Arguments: * traverse_lists (Boolean) Indicate whether KEY searches against lists are permitted to automatically traverse into the list; Default=True + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they are matched @@ -739,7 +784,7 @@ def _get_nodes_by_key( """ traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) pathseg: PathSegment = yaml_path.escaped[segment_index] (_, stripped_attrs) = pathseg @@ -811,7 +856,7 @@ def _get_nodes_by_key( # pylint: disable=locally-disabled,too-many-locals def _get_nodes_by_index( - self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs + self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs ) -> Generator[NodeCoords, None, None]: """ Get nodes from a List by their index. @@ -825,17 +870,24 @@ def _get_nodes_by_index( 2. yaml_path (YAMLPath) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process + Keyword Arguments: + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they - are matched + are matched Raises: N/A """ + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + pathseg: PathSegment = yaml_path.escaped[segment_index] (_, stripped_attrs) = pathseg (_, unstripped_attrs) = yaml_path.unescaped[segment_index] str_stripped = str(stripped_attrs) - translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) self.logger.debug( "Processor::_get_nodes_by_index: Seeking INDEX node at {}." @@ -900,7 +952,7 @@ def _get_nodes_by_index( ancestry + [(data, idx)], pathseg) def _get_nodes_by_anchor( - self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs + self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs ) -> Generator[NodeCoords, None, None]: """ Get nodes matching an Anchor name. @@ -913,15 +965,22 @@ def _get_nodes_by_anchor( 2. yaml_path (YAMLPath) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process + Keyword Arguments: + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they are matched Raises: N/A """ + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + pathseg: PathSegment = yaml_path.escaped[segment_index] (_, stripped_attrs) = pathseg - translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) next_translated_path = translated_path + "[&{}]".format( YAMLPath.escape_path_section( str(stripped_attrs), translated_path.seperator)) @@ -951,8 +1010,8 @@ def _get_nodes_by_anchor( next_ancestry, pathseg) def _get_nodes_by_keyword_search( - self, data: Any, yaml_path: YAMLPath, terms: SearchKeywordTerms, - **kwargs: Any + self, data: Any, yaml_path: YAMLPath, terms: SearchKeywordTerms, + **kwargs: Any ) -> Generator[NodeCoords, None, None]: """ Perform a search identified by a keyword and its parameters. @@ -968,9 +1027,13 @@ def _get_nodes_by_keyword_search( * parentref (Any) The Index or Key of data within parent * traverse_lists (Boolean) Indicate whether searches against lists are permitted to automatically traverse into the list; Default=True + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they - are matched + are matched Raises: N/A """ @@ -990,7 +1053,7 @@ def _get_nodes_by_keyword_search( # pylint: disable=too-many-statements def _get_nodes_by_search( - self, data: Any, terms: SearchTerms, **kwargs: Any + self, data: Any, terms: SearchTerms, **kwargs: Any ) -> Generator[NodeCoords, None, None]: """ Get nodes matching a search expression. @@ -1008,6 +1071,10 @@ def _get_nodes_by_search( * parentref (Any) The Index or Key of data within parent * traverse_lists (Boolean) Indicate whether searches against lists are permitted to automatically traverse into the list; Default=True + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[NodeCoords, None, None]) Each NodeCoords as they are matched @@ -1024,7 +1091,8 @@ def _get_nodes_by_search( traverse_lists: bool = kwargs.pop("traverse_lists", True) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) pathseg: PathSegment = (PathSegmentTypes.SEARCH, terms) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + invert = terms.inverted method = terms.method attr = terms.attribute @@ -1169,8 +1237,8 @@ def _get_nodes_by_search( # pylint: disable=locally-disabled def _get_nodes_by_collector( - self, data: Any, yaml_path: YAMLPath, segment_index: int, - terms: CollectorTerms, **kwargs: Any + self, data: Any, yaml_path: YAMLPath, segment_index: int, + terms: CollectorTerms, **kwargs: Any ) -> Generator[List[NodeCoords], None, None]: """ Generate List of nodes gathered via a Collector. @@ -1189,6 +1257,10 @@ def _get_nodes_by_collector( * parent (ruamel.yaml node) The parent node from which this query originates * parentref (Any) The Index or Key of data within parent + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[List[NodeCoords], None, None]) Each list of NodeCoords as they are matched (the result is always a list) @@ -1202,7 +1274,8 @@ def _get_nodes_by_collector( parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + node_coords: List[NodeCoords] = [] segments = yaml_path.escaped next_segment_idx = segment_index + 1 @@ -1312,9 +1385,9 @@ def _get_nodes_by_collector( yield node_coords # pylint: disable=locally-disabled,too-many-branches - def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, - segment_index: int, **kwargs: Any - ) -> Generator[Any, None, None]: + def _get_nodes_by_traversal( + self, data: Any, yaml_path: YAMLPath, segment_index: int, **kwargs: Any + ) -> Generator[Any, None, None]: """ Deeply traverse the document tree, returning all or filtered nodes. @@ -1323,10 +1396,14 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, 2. yaml_path (yamlpath.Path) The YAML Path being processed 3. segment_index (int) Segment index of the YAML Path to process - Keyword Parameters: + Keyword Arguments: * parent (ruamel.yaml node) The parent node from which this query originates * parentref (Any) The Index or Key of data within parent + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[Any, None, None]) Each node coordinate as they are matched. @@ -1334,7 +1411,8 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + segments = yaml_path.escaped pathseg: PathSegment = segments[segment_index] next_segment_idx: int = segment_index + 1 @@ -1463,9 +1541,9 @@ def _get_nodes_by_traversal(self, data: Any, yaml_path: YAMLPath, data=node_coord) yield node_coord - def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, - depth: int = 0, **kwargs: Any - ) -> Generator[NodeCoords, None, None]: + def _get_required_nodes( + self, data: Any, yaml_path: YAMLPath, depth: int = 0, **kwargs: Any + ) -> Generator[NodeCoords, None, None]: """ Generate pre-existing NodeCoords from YAML data matching a YAML Path. @@ -1477,16 +1555,28 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, originates 5. parentref (Any) Key or Index of data within parent + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation + Returns: (Generator[NodeCoords, None, None]) The requested NodeCoords - as they are matched + as they are matched Raises: N/A """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) - translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) + segments = yaml_path.escaped if segments and len(segments) > depth: pathseg: PathSegment = yaml_path.unescaped[depth] @@ -1559,8 +1649,8 @@ def _get_required_nodes(self, data: Any, yaml_path: YAMLPath, # pylint: disable=locally-disabled,too-many-statements def _get_optional_nodes( - self, data: Any, yaml_path: YAMLPath, value: Any = None, - depth: int = 0, **kwargs: Any + self, data: Any, yaml_path: YAMLPath, value: Any = None, + depth: int = 0, **kwargs: Any ) -> Generator[NodeCoords, None, None]: """ Return zero or more pre-existing NodeCoords matching a YAML Path. @@ -1574,9 +1664,17 @@ def _get_optional_nodes( 3. value (Any) The value to assign to the element 4. depth (int) For recursion, this identifies which segment of yaml_path to evaluate; default=0 - 5. parent (ruamel.yaml node) The parent node from which this query - originates - 6. parentref (Any) Index or Key of data within parent + + Keyword Arguments: + * parent (ruamel.yaml node) The parent node from which this query + originates + * parentref (Any) The Index or Key of data within parent + * relay_segment (PathSegment) YAML Path segment presently under + evaluation + * translated_path (YAMLPath) YAML Path indicating precisely which node + is being evaluated + * ancestry (List[AncestryEntry]) Stack of ancestors preceding the + present node under evaluation Returns: (Generator[NodeCoords, None, None]) The requested NodeCoords as they are matched @@ -1589,9 +1687,9 @@ def _get_optional_nodes( """ parent: Any = kwargs.pop("parent", None) parentref: Any = kwargs.pop("parentref", None) - translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) - ancestry: List[tuple] = kwargs.pop("ancestry", []) relay_segment: PathSegment = kwargs.pop("relay_segment", None) + translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath("")) + ancestry: List[AncestryEntry] = kwargs.pop("ancestry", []) segments = yaml_path.escaped # pylint: disable=locally-disabled,too-many-nested-blocks diff --git a/yamlpath/types/__init__.py b/yamlpath/types/__init__.py index 8cca559a..abf21409 100644 --- a/yamlpath/types/__init__.py +++ b/yamlpath/types/__init__.py @@ -1,3 +1,4 @@ """Make all custom types available.""" +from .ancestryentry import AncestryEntry from .pathattributes import PathAttributes from .pathsegment import PathSegment diff --git a/yamlpath/types/ancestryentry.py b/yamlpath/types/ancestryentry.py new file mode 100644 index 00000000..3ea90450 --- /dev/null +++ b/yamlpath/types/ancestryentry.py @@ -0,0 +1,8 @@ +""" +Defines a custom type for data ancestry (parent, parentref). + +Copyright 2021 William W. Kimball, Jr. MBA MSIS +""" +from typing import Any, Tuple + +AncestryEntry = Tuple[Any, Any] diff --git a/yamlpath/types/pathattributes.py b/yamlpath/types/pathattributes.py index 004aa304..5dc296ff 100644 --- a/yamlpath/types/pathattributes.py +++ b/yamlpath/types/pathattributes.py @@ -1,4 +1,8 @@ -"""Defines a custom type for YAML Path segment attributes.""" +""" +Defines a custom type for YAML Path segment attributes. + +Copyright 2020 William W. Kimball, Jr. MBA MSIS +""" from typing import Union from yamlpath.path import CollectorTerms diff --git a/yamlpath/types/pathsegment.py b/yamlpath/types/pathsegment.py index 1350ea3e..f598cf18 100644 --- a/yamlpath/types/pathsegment.py +++ b/yamlpath/types/pathsegment.py @@ -1,4 +1,8 @@ -"""Defines a custom type for YAML Path segments.""" +""" +Defines a custom type for YAML Path segments. + +Copyright 2020 William W. Kimball, Jr. MBA MSIS +""" from typing import Tuple from yamlpath.enums import PathSegmentTypes diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index b21283ae..27d13555 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -11,7 +11,7 @@ verbose: allows output from ConsolePrinter::verbose(). debug: allows output from ConsolePrinter::debug(). -Copyright 2018, 2019, 2020 William W. Kimball, Jr. MBA MSIS +Copyright 2018, 2019, 2020, 2021 William W. Kimball, Jr. MBA MSIS """ import sys from collections import deque diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index 51c3f7ad..e7fb3220 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -1,4 +1,8 @@ -"""Wrap a node along with its relative coordinates within its DOM.""" +""" +Implement NodeCoords. + +Copyright 2020, 2021 William W. Kimball, Jr. MBA MSIS +""" from typing import Any, List, Optional from yamlpath.types import PathSegment @@ -6,18 +10,24 @@ class NodeCoords: """ - Initialize a new NodeCoords. + Wrap a node's data along with its relative coordinates within its DOM. + + A node's "coordinates" includes these properties: + 1. Reference to the node itself, + 2. Immediate parent node of the wrapped node, + 3. Index or Key of the node within its immediate parent - A node's coordinates track these properties: - 1. Reference-to-the-Node-Itself, - 2. Immediate-Parent-Node-of-the-Node, - 3. Index-or-Key-of-the-Node-Within-Its-Immediate-Parent + Additional, optional data can be wrapped along with the node's coordinates + to facilitate other specific operations upon the node/DOM. See the + `__init__` method for details. """ # pylint: disable=locally-disabled,too-many-arguments def __init__( - self, node: Any, parent: Any, parentref: Any, path: YAMLPath = None, - ancestry: List[tuple] = None, path_segment: PathSegment = None + self, node: Any, parent: Any, parentref: Any, + path: Optional[YAMLPath] = None, + ancestry: Optional[List[tuple]] = None, + path_segment: Optional[PathSegment] = None ) -> None: """ Initialize a new NodeCoords. @@ -29,8 +39,11 @@ def __init__( within `parent` the `node` is located 4. path (YAMLPath) The YAML Path for this node, as reported by its creator process - 5. ancestry (List[tuple]) Tuples in (parent,parentref) form tracking - the hierarchical ancestry of this node through its parent document + 5. ancestry (List[AncestryEntry]) Stack of AncestryEntry (parent, + parentref) tracking the hierarchical ancestry of this node through + its parent document + 6. path_segment (PathSegment) The YAML Path segment which most directly + caused the generation of this NodeCoords Returns: N/A diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index 6ad3e397..d22560d8 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -1,7 +1,7 @@ """ Implement YAML Path. -Copyright 2019, 2020 William W. Kimball, Jr. MBA MSIS +Copyright 2019, 2020, 2021 William W. Kimball, Jr. MBA MSIS """ from collections import deque from typing import Deque, List, Optional, Union @@ -79,6 +79,10 @@ def __eq__(self, other: object) -> bool: """ Indicate equivalence of two YAMLPaths. + The path seperator is ignored for this comparison. This is deliberate + and allows "some.path[1]" == "/some/path[1]" because both forms of the + same path yield exactly the same data. + Parameters: 1. other (object) The other YAMLPath to compare against. @@ -131,7 +135,7 @@ def pop(self) -> PathSegment: """ Pop the last segment off this YAML Path. - This mutates the YAML Path and returns the removed segment tuple. + This mutates the YAML Path and returns the removed segment PathSegment. Returns: (PathSegment) The removed segment """ @@ -160,12 +164,6 @@ def pop(self) -> PathSegment: self.original = path_now[ 0:len(path_now) - len(removable_segment) + 1] - # I cannot come up with a test that would trigger this Exception: - # else: - # raise YAMLPathException( - # "Unable to pop unmatchable segment, {}" - # .format(removable_segment), str(self)) - return popped_segment @property @@ -287,7 +285,7 @@ def _parse_path(self, strip_escapes: bool = True ) -> Deque[PathSegment]: r""" - Parse the YAML Path into its component segments. + Parse the YAML Path into its component PathSegment tuples. Breaks apart a stringified YAML Path into component segments, each identified by its type. See README.md for sample YAML Paths. @@ -297,8 +295,8 @@ def _parse_path(self, only the "escaped" symbol. False = Leave all leading \ symbols intact. - Returns: (deque) an empty queue or a queue of tuples, each identifying - (PathSegmentTypes, segment_attributes). + Returns: (Deque[PathSegment]) an empty queue or a queue of + PathSegments. Raises: - `YAMLPathException` when the YAML Path is invalid @@ -736,7 +734,7 @@ def _parse_path(self, def _expand_splats( yaml_path: str, segment_id: str, segment_type: Optional[PathSegmentTypes] = None - ) -> tuple: + ) -> PathSegment: """ Replace segment IDs with search operators when * is present. @@ -746,7 +744,7 @@ def _expand_splats( 3. segment_type (Optional[PathSegmentTypes]) Pending predetermined type of the segment under evaluation. - Returns: (tuple) Coallesced YAML Path segment. + Returns: (PathSegment) Coallesced YAML Path segment. """ coal_type = segment_type coal_value: Union[str, SearchTerms, None] = segment_id From 332e4216342910330f4e85d0359aa9634cdb762b Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 25 Apr 2021 18:41:09 -0500 Subject: [PATCH 89/90] More consistent use of custom data types --- yamlpath/processor.py | 12 ++++-------- yamlpath/types/pathattributes.py | 2 +- yamlpath/wrappers/nodecoords.py | 8 +++++--- yamlpath/yamlpath.py | 12 ++++++------ 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/yamlpath/processor.py b/yamlpath/processor.py index 56a93183..df125a34 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -8,7 +8,7 @@ from ruamel.yaml.comments import CommentedMap -from yamlpath.types import AncestryEntry, PathSegment +from yamlpath.types import AncestryEntry, PathAttributes, PathSegment from yamlpath.common import Anchors, KeywordSearches, Nodes, Searches from yamlpath import YAMLPath from yamlpath.path import SearchKeywordTerms, SearchTerms, CollectorTerms @@ -798,7 +798,8 @@ def _get_nodes_by_key( next_translated_path = (translated_path + YAMLPath.escape_path_section( str_stripped, translated_path.seperator)) - next_ancestry = ancestry + [(data, stripped_attrs)] + next_ancestry: List[AncestryEntry] = ancestry + [ + (data, stripped_attrs)] if stripped_attrs in data: self.logger.debug( "Processor::_get_nodes_by_key: FOUND key node by name at" @@ -1696,12 +1697,7 @@ def _get_optional_nodes( if segments and len(segments) > depth: pathseg: PathSegment = yaml_path.unescaped[depth] (segment_type, unstripped_attrs) = pathseg - stripped_attrs: Union[ - str, - int, - SearchTerms, - CollectorTerms - ] = segments[depth][1] + stripped_attrs: PathAttributes = segments[depth][1] except_segment = str(unstripped_attrs) self.logger.debug( diff --git a/yamlpath/types/pathattributes.py b/yamlpath/types/pathattributes.py index 5dc296ff..ff0b0fcc 100644 --- a/yamlpath/types/pathattributes.py +++ b/yamlpath/types/pathattributes.py @@ -9,4 +9,4 @@ import yamlpath.path.searchterms as searchterms -PathAttributes = Union[str, CollectorTerms, searchterms.SearchTerms] +PathAttributes = Union[str, int, CollectorTerms, searchterms.SearchTerms, None] diff --git a/yamlpath/wrappers/nodecoords.py b/yamlpath/wrappers/nodecoords.py index e7fb3220..9a3b0447 100644 --- a/yamlpath/wrappers/nodecoords.py +++ b/yamlpath/wrappers/nodecoords.py @@ -5,7 +5,7 @@ """ from typing import Any, List, Optional -from yamlpath.types import PathSegment +from yamlpath.types import AncestryEntry, PathSegment from yamlpath import YAMLPath class NodeCoords: @@ -26,7 +26,7 @@ class NodeCoords: def __init__( self, node: Any, parent: Any, parentref: Any, path: Optional[YAMLPath] = None, - ancestry: Optional[List[tuple]] = None, + ancestry: Optional[List[AncestryEntry]] = None, path_segment: Optional[PathSegment] = None ) -> None: """ @@ -53,7 +53,9 @@ def __init__( self.parent: Any = parent self.parentref: Any = parentref self.path: Optional[YAMLPath] = path - self.ancestry: List[tuple] = [] if ancestry is None else ancestry + self.ancestry: List[AncestryEntry] = ([] + if ancestry is None + else ancestry) self.path_segment: Optional[PathSegment] = path_segment def __str__(self) -> str: diff --git a/yamlpath/yamlpath.py b/yamlpath/yamlpath.py index d22560d8..da0db42a 100644 --- a/yamlpath/yamlpath.py +++ b/yamlpath/yamlpath.py @@ -6,7 +6,7 @@ from collections import deque from typing import Deque, List, Optional, Union -from yamlpath.types import PathSegment +from yamlpath.types import PathAttributes, PathSegment from yamlpath.exceptions import YAMLPathException from yamlpath.enums import ( PathSegmentTypes, @@ -732,8 +732,8 @@ def _parse_path(self, @staticmethod def _expand_splats( - yaml_path: str, segment_id: str, - segment_type: Optional[PathSegmentTypes] = None + yaml_path: str, segment_id: PathAttributes, + segment_type: PathSegmentTypes ) -> PathSegment: """ Replace segment IDs with search operators when * is present. @@ -746,10 +746,10 @@ def _expand_splats( Returns: (PathSegment) Coallesced YAML Path segment. """ - coal_type = segment_type - coal_value: Union[str, SearchTerms, None] = segment_id + coal_type: PathSegmentTypes = segment_type + coal_value: PathAttributes = segment_id - if '*' in segment_id: + if isinstance(segment_id, str) and '*' in segment_id: splat_count = segment_id.count("*") splat_pos = segment_id.index("*") segment_len = len(segment_id) From 1df907dce4c6d6437cec559ff5e450210d1d1b9d Mon Sep 17 00:00:00 2001 From: William Kimball <30981667+wwkimball@users.noreply.github.com> Date: Sun, 25 Apr 2021 19:09:16 -0500 Subject: [PATCH 90/90] Mention Search Keywords --- README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4b4f50f3..5b3c89e4 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,18 @@ YAML Path understands these segment types: * When another segment follows, it matches every node within the remainder of the document's tree for which the following (and subsequent) segments match: `/shows/**/name/Star*` +* Search Keywords: Advanced search capabilities not otherwise possible using + other YAML Path segments. Taking the form of `[KEYWORD(PARAMETERS)]`, these + keywords are + [deeply explored on the Wiki](https://github.com/wwkimball/yamlpath/wiki/Search-Keywords) + and include: + * `[has_child(NAME)]`: Match nodes having a named child key + * `[max([NAME])]`: Match nodes having the maximum value + * `[min([NAME])]`: Match nodes having the minimum value + * `[name()]`: Match only the name of the present node, discarding all + children + * `[parent([STEPS])]`, Step up 1-N levels in the document from the present + node * Collectors: Placing any portion of the YAML Path within parenthesis defines a virtual list collector, like `(YAML Path)`; concatenation and exclusion operators are supported -- `+` and `-`, respectively -- along with nesting, @@ -627,9 +639,9 @@ optional arguments: ```text usage: yaml-paths [-h] [-V] -s EXPRESSION [-c EXPRESSION] [-m] [-L] [-F] [-X] - [-P] [-t ['.', '/', 'auto', 'dot', 'fslash']] [-i | -k | -K] - [-a] [-A | -Y | -y | -l] [-e] [-x EYAML] [-r PRIVATEKEY] - [-u PUBLICKEY] [-S] [-d | -v | -q] + [-P] [-n] [-t ['.', '/', 'auto', 'dot', 'fslash']] + [-i | -k | -K] [-a] [-A | -Y | -y | -l] [-e] [-x EYAML] + [-r PRIVATEKEY] [-u PUBLICKEY] [-S] [-d | -v | -q] [YAML_FILE [YAML_FILE ...]] Returns zero or more YAML Paths indicating where in given YAML/JSON/Compatible @@ -674,6 +686,11 @@ result printing options: or to indicate whether a file has any matches without printing them all, perhaps especially with --noexpression) + -n, --noescape omit escape characters from special characters in + printed YAML Paths; this is unsafe for feeding the + resulting YAML Paths into other YAML Path commands + because the symbols that would be escaped have special + meaning to YAML Path processors key name searching options: -i, --ignorekeynames (default) do not search key names @@ -717,7 +734,9 @@ EYAML options: A search or exception EXPRESSION takes the form of a YAML Path search operator -- %, $, =, ^, >, <, >=, <=, =~, or ! -- followed by the search term, omitting the left-hand operand. For more information about YAML Paths, please visit -https://github.com/wwkimball/yamlpath. +https://github.com/wwkimball/yamlpath/wiki. To report issues with this tool or +to request enhancements, please visit +https://github.com/wwkimball/yamlpath/issues. ``` * [yaml-set](yamlpath/commands/yaml_set.py) @@ -1201,7 +1220,7 @@ from yamlpath.exceptions import YAMLPathException yaml_path = YAMLPath("see.documentation.above.for.many.samples") try: - for node_coordinate in processor.get_nodes(yaml_path): + for node_coordinate in processor.get_nodes(yaml_path, mustexist=True): log.debug("Got {} from '{}'.".format(node_coordinate, yaml_path)) # Do something with each node_coordinate.node (the actual data) except YAMLPathException as ex: