Merge pull request #14 from wwkimball/development

Prepare for release 1.2.0

CHANGES

@@ -1,3 +1,41 @@
+1.2.0
+Enhancements:
+* A new search operator, :, now enables capturing slices of Arrays (by 0-based
+  element number) and Hashes (by alphanumeric key-name).  This looks like:
+  "some::array[2:15]" or "some::hash[beta:gamma]".
+* yaml-get now returns JSON instead of "pretty Python" data objects when the
+  search returns complex data types (Arrays and Hashes).  This change makes the
+  result more portable to non-Python consumers and ensures the result will be
+  one per line.
+* The separator used for identifying Hash sub-keys can now be customized.  If
+  you prefer your paths to look like "/hash/sub/key" rather than "hash.sub.key",
+  you can now have it your way.  For now, only . and / are allowed.  The
+  seperator can be either strictly specified or automatically inferred by
+  whether the first character of a given YAML Path is /.  Command-line tools
+  like yaml-get and yaml-set have a new --pathsep argument for this; the default
+  is "auto" and can be set to "fslash" (/) or "dot" (.).
+
+Bug Fixes:
+* EYAML on Windows now works when a batch file is used to wrap the Ruby `eyaml`
+  command.
+
+Known Issues:
+* Escape symbols in YAML Paths parse correctly and will be properly processed,
+  resulting in retriving or setting the expected data.  However, the parsed
+  path cannot be stringified back to its original form (with escape symbols).
+  This issue affects only logging/printing of the post-parsed path.  A unit test
+  has been created to track this issue, but it is marked xfail until such time
+  as someone is willing to tackle this (very) low priority issue.  Until then,
+  developers should try to print the pre-parsed version of their paths rather
+  than rely exclusively on Parser.str_path().  Further, don't do this:
+    1. Accept or take a string path that has escaped characters.
+    2. Parse that path.
+    3. Stringify the parsed path.
+    4. Parse the stringified, parsed path.  This is silly, anyway because you
+       already have the first (good) parsed result at step 2.
+    5. Try to use this parsed-stringified-parsed path result for anything.
+       Instead, only use the first parsed result that you got at step 2.
+
 1.1.2
 Bug fixes:
 * When the YAML Path is fully quoted -- a known side-effect of using Jenkins and

README.md

@@ -48,6 +48,7 @@ Hash members, this YAML Path solution grew to include new syntax for:
 * Array elements
 * Anchors by name
 * Search expressions for single or multiple matches
+* Forward-slash notation
 
 To illustrate some of these concepts, consider this sample YAML data:
 
@@ -83,34 +84,43 @@ sensitive::accounts:
 
 This YAML data sample contains these single-result YAML Paths:
 
-1. `aliases[&commonUsername]`
-2. `aliases[&commonPassword]`
-3. `configuration::application.'general.settings'.slash\\key`
-4. `configuration::application.'general.settings'.'a.dotted.subkey'[0]`
-5. `configuration::application.'general.settings'.'a.dotted.subkey'[1]`
-6. `configuration::application.'general.settings'.'a.dotted.subkey'[2]`
-7. `sensitive::accounts.database.app_user`
-8. `sensitive::accounts.database.app_pass`
-9. `sensitive::accounts.application.db.users[0].name`
-10. `sensitive::accounts.application.db.users[0].pass`
-11. `sensitive::accounts.application.db.users[0].access_level`
-12. `sensitive::accounts.application.db.users[1].name`
-13. `sensitive::accounts.application.db.users[1].pass`
-14. `sensitive::accounts.application.db.users[1].access_level`
+Dot Notation                                                         | Forward-Slash Notation
+---------------------------------------------------------------------|------------------------------------------------------------------
+`aliases[&commonUsername]`                                           | `/aliases[&commonUsername]`
+`aliases[&commonPassword]`                                           | `/aliases[&commonPassword]`
+`configuration::application.'general.settings'.slash\\key`           | `/configuration::application/general.settings/slash\\key`
+`configuration::application.'general.settings'.'a.dotted.subkey'[0]` | `/configuration::application/general.settings/a.dotted.subkey[0]`
+`configuration::application.'general.settings'.'a.dotted.subkey'[1]` | `/configuration::application/general.settings/a.dotted.subkey[1]`
+`configuration::application.'general.settings'.'a.dotted.subkey'[2]` | `/configuration::application/general.settings/a.dotted.subkey[2]`
+`sensitive::accounts.database.app_user`                              | `/sensitive::accounts/database/app_user`
+`sensitive::accounts.database.app_pass`                              | `/sensitive::accounts/database/app_pass`
+`sensitive::accounts.application.db.users[0].name`                   | `/sensitive::accounts/application/db/users[0]/name`
+`sensitive::accounts.application.db.users[0].pass`                   | `/sensitive::accounts/application/db/users[0]/pass`
+`sensitive::accounts.application.db.users[0].access_level`           | `/sensitive::accounts/application/db/users[0]/access_level`
+`sensitive::accounts.application.db.users[1].name`                   | `/sensitive::accounts/application/db/users[1]/name`
+`sensitive::accounts.application.db.users[1].pass`                   | `/sensitive::accounts/application/db/users[1]/pass`
+`sensitive::accounts.application.db.users[1].access_level`           | `/sensitive::accounts/application/db/users[1]/access_level`
 
 You could also access some of these sample nodes using search expressions, like:
 
-1. `configuration::application.general\.settings.'a.dotted.subkey'[.=~/^element[1-2]$/]`
-2. `sensitive::accounts.application.db.users[name=admin].access_level`
-3. `sensitive::accounts.application.db.users[access_level<500].name`
+Dot Notation                                                                          | Forward-Slash Notation
+--------------------------------------------------------------------------------------|------------------------------------------------------------------
+`configuration::application.general\.settings.'a.dotted.subkey'[.=~/^element[1-2]$/]` | `/configuration::application/general.settings/a.dotted.subkey[.=~/^element[1-2]$/]`
+`configuration::application.general\.settings.'a.dotted.subkey'[1:2]`                 | `/configuration::application/general.settings/a.dotted.subkey[0:-2]`
+`sensitive::accounts.application.db.users[name=admin].access_level`                   | `/sensitive::accounts/application/db/users[name=admin]/access_level`
+`sensitive::accounts.application.db.users[access_level<500].name`                     | `/sensitive::accounts/application/db/users[access_level<500]/name`
 
 ## Supported YAML Path Forms
 
 YAML Path understands these forms:
 
+* Top-level Array element selection: `[#]` where `#` is the 0-based element number (`#` can also be negative, causing the element to be selected from the end of the Array)
+* Top-level Hash key selection: `key`
 * Dot notation for Hash sub-keys:  `hash.child.key`
 * Demarcation for dotted Hash keys:  `hash.'dotted.child.key'` or `hash."dotted.child.key"`
-* Array element selection:  `array[#]` (where `array` is omitted for top-level Arrays or is the name of the Hash key containing Array data and `#` is the 0-based element number)
+* Array element selection:  `array[#]` where `array` is omitted for top-level Arrays or is the name of the Hash key containing Array data and `#` is the 0-based element number (`#` can also be negative, causing the element to be selected from the end of the Array)
+* Array slicing: `array[start#:stop#]` where `start#` is the first, zero-based element and `stop#` is the last element to select (either or both can be negative, causing the elements to be selected from the end of the Array)
+* Hash slicing: `hash[min:max]` where `min` and `max` are alphanumeric terms between which the Hash's keys are compared
 * Escape symbol recognition:  `hash.dotted\.child\.key` or `keys_with_\\slashes`
 * Top-level (Hash) Anchor lookups: `&anchor_name`
 * Anchor lookups in Arrays:  `array[&anchor_name]`
@@ -123,12 +133,13 @@ YAML Path understands these forms:
   * Greater Than match: `sensitive::accounts.application.db.users[access_level>0].pass`
   * Less Than or Equal match: `sensitive::accounts.application.db.users[access_level<=100].pass`
   * Greater Than or Equal match: `sensitive::accounts.application.db.users[access_level>=0].pass`
-  * Regular Expression matches using any delimiter you choose (other than `/`, if you need something else): `sensitive::accounts.application.db.users[access_level=~/^\D+$/].pass` or `some::hash[containing=~#/path/values#]`
+  * Regular Expression matches using any delimiter you choose (other than `/`, if you need something else): `sensitive::accounts.application.db.users[access_level=~/^\D+$/].pass` or `some::hash[containing=~"/path/values"]`
   * Invert any match with `!`, like: `sensitive::accounts.application.db.users[name!=admin].pass`
   * Demarcate and/or escape expression values, like: `sensitive::accounts.application.db.users[full\ name="Some User\'s Name"].pass`
   * Multi-level matching: `sensitive::accounts.application.db.users[name%admin].pass[encrypted!^ENC\[]`
 * Array element and Hash key-name searches with all of the search methods above via `.` (yields their values, not the keys themselves): `sensitive::accounts.database[.^app_]`
-* Complex combinations: `some::deep.hierarchy[with!=""].'any.valid'[.$yaml][data%structure].or.[!complexity=~/^.{4}$/][2]`
+* Complex combinations: `some::deep.hierarchy[with!=""].'any.valid'[.$yaml][data%structure].or[!complexity=~/^.{4}$/][2]`
+* Forward-slash rather than dot notation:  `/key` up to `/some::deep/hierarchy[with!=""]/any.valid[.$yaml][data%structure]/or[!complexity=~/^.{4}$/][2]`
 
 ## Installing
 

bin/yaml-get

@@ -16,14 +16,15 @@
 ################################################################################
 import sys
 import argparse
-import pprint
+import json
 from os import access, R_OK
 from os.path import isfile
 
 from ruamel.yaml import YAML
 from ruamel.yaml.parser import ParserError
 
 from yamlpath.exceptions import YAMLPathException, EYAMLCommandException
+from yamlpath.enums import PathSeperators
 from yamlpath.parser import Parser
 from yamlpath.eyaml import EYAMLPath
 
@@ -51,6 +52,13 @@ def processcli():
         help="YAML Path to query"
     )
 
+    parser.add_argument("-t", "--pathsep",
+        default="auto",
+        choices=[l.lower() for l in PathSeperators.get_names()],
+        type=str.lower,
+        help="force the separator in YAML_PATH when inference fails"
+    )
+
     eyaml_group = parser.add_argument_group(
         "EYAML options", "Left unset, the EYAML keys will default to your\
          system or user defaults.  Both keys must be set either here or in\
@@ -108,7 +116,7 @@ def main():
     args = processcli()
     log = ConsolePrinter(args)
     validateargs(args, log)
-    parser = Parser(log)
+    parser = Parser(log, pathsep=args.pathsep)
     processor = EYAMLPath(
         log,
         eyaml=args.eyaml,
@@ -157,9 +165,11 @@ def main():
     if not discovered_nodes:
         log.critical("No matches for {}!".format(yaml_path), 3)
 
-    pprinter = pprint.PrettyPrinter(indent=4)
     for node in discovered_nodes:
-        pprinter.pprint(node)
+        if isinstance(node, list) or isinstance(node, dict):
+            print(json.dumps(node))
+        else:
+            print("{}".format(node))
 
 if __name__ == "__main__":
     main()

bin/yaml-set

@@ -28,7 +28,7 @@ from ruamel.yaml.parser import ParserError
 
 from yamlpath.exceptions import YAMLPathException, EYAMLCommandException
 from yamlpath.eyaml import EYAMLPath
-from yamlpath.enums import YAMLValueFormats
+from yamlpath.enums import YAMLValueFormats, PathSeperators
 
 import yamlpath.patches
 from yamlpath.wrappers import ConsolePrinter
@@ -89,6 +89,12 @@ def processcli():
         help="require that the --change YAML_PATH already exist in YAML_FILE")
     parser.add_argument("-b", "--backup", action="store_true",
         help="save a backup YAML_FILE with an extra .bak file-extension")
+    parser.add_argument("-t", "--pathsep",
+        default="auto",
+        choices=[l.lower() for l in PathSeperators.get_names()],
+        type=str.lower,
+        help="force the separator in YAML_PATH when inference fails"
+    )
 
     eyaml_group = parser.add_argument_group(
         "EYAML options", "Left unset, the EYAML keys will default to your\
@@ -176,7 +182,8 @@ def main():
         log,
         eyaml=args.eyaml,
         publickey=args.publickey,
-        privatekey=args.privatekey
+        privatekey=args.privatekey,
+        pathsep=args.pathsep,
     )
     backup_file = args.yaml_file + ".bak"
 

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="yamlpath",
-    version="1.1.2",
+    version="1.2.0",
     description="Generally-useful YAML and EYAML tools employing a human-friendly YAML Path",
     long_description=long_description,
     long_description_content_type="text/markdown",

tests/test_eyamlpath.py

@@ -5,12 +5,13 @@
 
 from ruamel.yaml import YAML
 
+import yamlpath.patches
 from yamlpath.eyaml import EYAMLPath
 from yamlpath.wrappers import ConsolePrinter
 from yamlpath.exceptions import EYAMLCommandException
 
 requireseyaml = pytest.mark.skipif(
-    not EYAMLPath.get_eyaml_executable("eyaml")
+    EYAMLPath.get_eyaml_executable("eyaml") is None
     , reason="The 'eyaml' command must be installed and accessible on the PATH"
         + " to test and use EYAML features.  Try:  'gem install hiera-eyaml'"
         + " after intalling ruby and rubygems."
@@ -170,9 +171,10 @@ def eyamlkeys(tmp_path_factory):
     with open(old_public_key_file, 'w') as key_file:
         key_file.write(old_public_key)
 
+    eyaml_cmd = EYAMLPath.get_eyaml_executable("eyaml")
     run(
-        "eyaml createkeys --pkcs7-private-key={} --pkcs7-public-key={}"
-        .format(new_private_key_file, new_public_key_file).split()
+        "{} createkeys --pkcs7-private-key={} --pkcs7-public-key={}"
+        .format(eyaml_cmd, new_private_key_file, new_public_key_file).split()
     )
 
     return (
@@ -207,6 +209,15 @@ def fake_run(*args, **kwargs):
 
     monkeypatch.setattr(break_module, "run", fake_run)
 
+@pytest.fixture
+def force_no_access(monkeypatch):
+    import yamlpath.eyaml.eyamlpath as break_module
+
+    def fake_access(*args, **kwargs):
+        return False
+
+    monkeypatch.setattr(break_module, "access", fake_access)
+
 @requireseyaml
 @pytest.mark.parametrize("search,compare", [
     ("aliases[&secretIdentity]", "This is not the identity you are looking for."),
@@ -233,12 +244,12 @@ def test_happy_set_eyaml_value(eyamlpath_f, eyamldata, eyamlkeys, search, compar
     eyamlpath_f.set_eyaml_value(eyamldata, search, compare, mustexist=mustexist, output=output)
 
     # Ensure the new value is encrypted
+    encvalue = None
     for encnode in eyamlpath_f.get_nodes(eyamldata, search):
-        assert EYAMLPath.is_eyaml_value(encnode)
+        encvalue = encnode
+        break
 
-    # Ensure the new value decrypts back to the original value
-    for decnode in eyamlpath_f.get_eyaml_values(eyamldata, search, mustexist=True):
-        assert decnode == compare
+    assert EYAMLPath.is_eyaml_value(encvalue)
 
 def test_none_eyaml_value():
     assert False == EYAMLPath.is_eyaml_value(None)
@@ -295,4 +306,6 @@ def test_decrypt_calledprocesserror(eyamlpath_f, force_subprocess_run_cpe):
     with pytest.raises(EYAMLCommandException):
         eyamlpath_f.decrypt_eyaml("ENC[...]")
 
-# 60, 67, 98, 101, 123-128, 138-143, 163, 190-191
+@requireseyaml
+def test_non_executable(eyamlkeys, force_no_access):
+    assert EYAMLPath.get_eyaml_executable(str(eyamlkeys[0])) is None

tests/test_parser.py

@@ -6,6 +6,7 @@
 from yamlpath.parser import Parser
 from yamlpath.exceptions import YAMLPathException
 from yamlpath.wrappers import ConsolePrinter
+from yamlpath.enums import PathSeperators
 
 @pytest.fixture
 def parser():
@@ -17,7 +18,6 @@ def parser():
 def test_empty_str_path(parser):
     assert parser.str_path("") == ""
 
-# Happy searches
 @pytest.mark.parametrize("yaml_path,stringified", [
     ("aliases[&anchor]", "aliases[&anchor]"),
     ("a l i a s e s [ & a n c h o r ]", "aliases[&anchor]"),
@@ -67,15 +67,29 @@ def test_empty_str_path(parser):
     ('&topArrayAnchor[0]', '&topArrayAnchor[0]'),
     ('"&topArrayAnchor[0]"', r'\&topArrayAnchor\[0\]'),
     ('"&subHashAnchor.child1.attr_tst"', r'\&subHashAnchor\.child1\.attr_tst'),
-    ("'&topArrayAnchor[!.=~/[Oo]riginal/]'", r"\&topArrayAnchor\[\!\.=\~/\[Oo\]riginal/\]"),
+    ("'&topArrayAnchor[!.=~/[Oo]riginal/]'", r"\&topArrayAnchor\[!\.=~/\[Oo\]riginal/\]"),
 ])
 def test_happy_str_path_translations(parser, yaml_path, stringified):
     assert parser.str_path(yaml_path) == stringified
 
+# This will be a KNOWN ISSUE for this release.  The fix for this may require a
+# deep rethink of the Parser class.  The issue here is that escaped characters
+# in YAML Paths work perfectly well, but they can't be printed back to the
+# screen in their pre-parsed form.  So, when a user submits a YAML Path of
+# "some\\escaped\\key", all printed forms of the key will become
+# "someescapedkey" even though the path WILL find the requested data.  This is
+# only a stringification (printing) anomoly and hense, it will be LOW PRIORITY,
+# tracked as a KNOWN ISSUE, for now.
+@pytest.mark.xfail
+@pytest.mark.parametrize("yaml_path,stringified", [
+    ('key\\with\\slashes', 'key\\with\\slashes'),
+])
+def test_escaped_translations(parser, yaml_path, stringified):
+    assert parser.str_path(yaml_path) == stringified
+
 def test_happy_parse_path_list_to_deque(parser):
     assert isinstance(parser.parse_path(["item1", "item2"]), deque)
 
-# Unhappy searches
 @pytest.mark.parametrize("yaml_path", [
     ('some[search ^^ "Name "]'),
     ('some[search $$ " Here"]'),
@@ -107,8 +121,36 @@ def test_happy_parse_path_list_to_deque(parser):
     ('some[search = "unterminated demarcation]'),
     ('some[search =~ /unterminated RegEx]'),
     ('some[search ^= "meaningless operator"]'),
+    ('array[4F]'),
     ({}),
 ])
 def test_uphappy_str_path_translations(parser, yaml_path):
     with pytest.raises(YAMLPathException):
         parser.str_path(yaml_path)
+
+@pytest.mark.parametrize("pathsep,yaml_path,stringified", [
+    ('.', "some.hash.key", "some.hash.key"),
+    ('/', "/some/hash/key", "/some/hash/key"),
+    ('.', "/some/hash/key", "some.hash.key"),
+    ('/', "some.hash.key", "/some/hash/key"),
+    ('/', "/&someAnchoredArray[0]", "/&someAnchoredArray[0]"),
+    ('.', "&someAnchoredArray[0]", "&someAnchoredArray[0]"),
+    ('.', "/&someAnchoredArray[0]", "&someAnchoredArray[0]"),
+    ('/', "&someAnchoredArray[0]", "/&someAnchoredArray[0]"),
+])
+def test_pathsep(parser, pathsep, yaml_path, stringified):
+    assert parser.str_path(yaml_path, pathsep=pathsep) == stringified
+
+@pytest.mark.parametrize("pathsep,compare", [
+    (PathSeperators.DOT, PathSeperators.DOT),
+    (PathSeperators.FSLASH, PathSeperators.FSLASH),
+    ('.', PathSeperators.DOT),
+    ('/', PathSeperators.FSLASH),
+])
+def test_pretyped_pathsep(pathsep, compare):
+    parser = Parser(None, pathsep=pathsep)
+    assert compare == parser.pathsep
+
+def test_bad_pathsep():
+    with pytest.raises(YAMLPathException):
+        _ = Parser(None, pathsep="no such seperator!")