diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ceed8a8..e789ca1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ['3.6', '3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/python-publish-to-prod-pypi.yml b/.github/workflows/python-publish-to-prod-pypi.yml index 3cb4dfe..dfa6976 100644 --- a/.github/workflows/python-publish-to-prod-pypi.yml +++ b/.github/workflows/python-publish-to-prod-pypi.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.9' - name: Install Build Tools run: | python -m pip install --upgrade pip diff --git a/.github/workflows/python-publish-to-test-pypi.yml b/.github/workflows/python-publish-to-test-pypi.yml index 5de2389..1885c5b 100644 --- a/.github/workflows/python-publish-to-test-pypi.yml +++ b/.github/workflows/python-publish-to-test-pypi.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ['3.6', '3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v2 @@ -60,7 +60,7 @@ jobs: - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.9' - name: Install Build Tools run: | python -m pip install --upgrade pip diff --git a/.pylintrc b/.pylintrc index c48f2b4..ded117d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -67,8 +67,12 @@ confidence= # false-positives. # WWK-2019-05-30: Disable R0801 because similar command-line tools necessarily # share duplicate code. This is by design so each can stand-alone. +# WWK-2021-10-26: Disable C0209 because it is merely a performance-enhancing +# suggestion, not a breaking issue. I will be interested in revisiting +# this later, but not when I have a critical bug that I'm trying to fix. disable=R0401, - R0801 + R0801, + C0209 # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/CHANGES b/CHANGES index cd13897..60be4e3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +3.6.2: +Bug Fixes: +* The eyaml-rotate-keys command-line tool would generate a stack-dump when the + key for an encrypted value contained dots. The underlying library for this + tool now safely generates the internal YAMLPaths it uses. +* The default encoding when opening files is now set to utf-8 to support + extended character sets on Windows. + 3.6.1: Enhancements: * Enable verified support for ruamel.yaml up to version 0.17.10. diff --git a/README.md b/README.md index eb864c7..06a89f6 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![PyPI version](https://badge.fury.io/py/yamlpath.svg)](https://pypi.org/project/yamlpath/) [![Downloads](https://pepy.tech/badge/yamlpath)](https://pepy.tech/project/yamlpath) [![Coverage Status](https://coveralls.io/repos/github/wwkimball/yamlpath/badge.svg?branch=master)](https://coveralls.io/github/wwkimball/yamlpath?branch=master) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/6bcc1f2767854390923a8d25a2e4a191)](https://www.codacy.com/manual/wwkimball/yamlpath?utm_source=github.com&utm_medium=referral&utm_content=wwkimball/yamlpath&utm_campaign=Badge_Grade) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/b146b032a098413cbc2825cdf73fd7b2)](https://www.codacy.com/gh/wwkimball/yamlpath/dashboard?utm_source=github.com&utm_medium=referral&utm_content=wwkimball/yamlpath&utm_campaign=Badge_Grade) Along with providing a [standard for defining YAML Paths](https://github.com/wwkimball/yamlpath/wiki/Segments-of-a-YAML-Path), diff --git a/setup.py b/setup.py index 299774e..02bd3e4 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ """Build this project.""" from setuptools import find_packages, setup -with open("README.md", "r") as fh: +with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() setup( diff --git a/tests/test_commands_eyaml_rotate_keys.py b/tests/test_commands_eyaml_rotate_keys.py index 372c1f2..7cd91c0 100644 --- a/tests/test_commands_eyaml_rotate_keys.py +++ b/tests/test_commands_eyaml_rotate_keys.py @@ -88,10 +88,12 @@ def test_good_multi_replacements(self, script_runner, tmp_path_factory, old_eyam - &stringStyle ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAIu44u62q5sVfzC7kytLi2Z/EzH2DKr4vDsoqDBeSZ71aRku/uSrjyiO4lyoq9Kva+eBAyjBay5fnqPVBaU3Rud2pdEoZEoyofi02jn4hxUKpAO1W0AUgsQolGe53qOdM4U8RbwnTR0gr3gp2mCd18pH3SRMP9ryrsBAxGzJ6mR3RgdZnlTlqVGXCeWUeVpbH+lcHw3uvd+o/xkvJ/3ypxz+rWILiAZ3QlCirzn/qb2fHuKf3VBh8RVFuQDaM5voajZlgjD6KzNCsbATOqOA6eJI4j0ngPdDlIjGHAnahuyluQ5f5SIaIjLC+ZeCOfIYni0MQ+BHO0JNbccjq2Unb7TBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCYmAI0Ao3Ok1cSmVw0SgQGgCBK62z1r5RfRjf1xKfqDxTsGUHfsUmM3EjGJfnWzCRvuQ==] block: *blockStyle string: *stringStyle + yet_another: + 'more.complex.child': *blockStyle """ simple_file = create_temp_yaml_file(tmp_path_factory, simple_content) anchored_file = create_temp_yaml_file(tmp_path_factory, anchored_content) - + result = script_runner.run( self.command, "--newprivatekey={}".format(new_eyaml_keys[0]), diff --git a/yamlpath/__init__.py b/yamlpath/__init__.py index d01f70b..cab52f6 100644 --- a/yamlpath/__init__.py +++ b/yamlpath/__init__.py @@ -1,6 +1,6 @@ """Core YAML Path classes.""" # Establish the version number common to all components -__version__ = "3.6.1" +__version__ = "3.6.2" from yamlpath.yamlpath import YAMLPath from yamlpath.processor import Processor diff --git a/yamlpath/commands/eyaml_rotate_keys.py b/yamlpath/commands/eyaml_rotate_keys.py index 441ed15..14c8ad9 100644 --- a/yamlpath/commands/eyaml_rotate_keys.py +++ b/yamlpath/commands/eyaml_rotate_keys.py @@ -15,7 +15,7 @@ from ruamel.yaml.scalarstring import FoldedScalarString from yamlpath import __version__ as YAMLPATH_VERSION -from yamlpath.common import Parsers +from yamlpath.common import Anchors, Parsers from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.eyaml import EYAMLProcessor from yamlpath.wrappers import ConsolePrinter @@ -136,14 +136,14 @@ def main(): processor.data = yaml_data for yaml_path in processor.find_eyaml_paths(): # Use ::get_nodes() instead of ::get_eyaml_values() here in order - # to ignore values that have already been decrypted via their + # to ignore values that have already been rotated via their # Anchors. - for node_coordinate in processor.get_nodes(yaml_path): - node = node_coordinate.node + for node_coordinate in processor.get_nodes( + yaml_path, mustexist=True + ): # Ignore values which are Aliases for those already decrypted - anchor_name = ( - node.anchor.value if hasattr(node, "anchor") else None - ) + node = node_coordinate.node + anchor_name = Anchors.get_node_anchor(node) if anchor_name is not None: if anchor_name in seen_anchors: continue @@ -190,7 +190,7 @@ def main(): copy2(yaml_file, backup_file) log.verbose("Writing changed data to {}.".format(yaml_file)) - with open(yaml_file, 'w') as yaml_dump: + with open(yaml_file, 'w', encoding='utf-8') as yaml_dump: yaml.dump(yaml_data, yaml_dump) sys.exit(exit_state) diff --git a/yamlpath/commands/yaml_merge.py b/yamlpath/commands/yaml_merge.py index 1c6abf9..f7d9ab8 100644 --- a/yamlpath/commands/yaml_merge.py +++ b/yamlpath/commands/yaml_merge.py @@ -295,7 +295,7 @@ def write_output_document( dumps.append(doc.data) if args.output: - with open(args.output, 'w') as out_fhnd: + with open(args.output, 'w', encoding='utf-8') as out_fhnd: if document_is_json: if len(dumps) > 1: for dump in dumps: diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index c1a4801..e4fbf5a 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -297,7 +297,7 @@ def validateargs(args, log): def save_to_json_file(args, log, yaml_data): """Save to a JSON file.""" log.verbose("Writing changed data as JSON to {}.".format(args.yaml_file)) - with open(args.yaml_file, 'w') as out_fhnd: + with open(args.yaml_file, 'w', encoding='utf-8') as out_fhnd: json.dump(Parsers.jsonify_yaml_data(yaml_data), out_fhnd) def save_to_yaml_file(args, log, yaml_parser, yaml_data, backup_file): @@ -307,7 +307,7 @@ def save_to_yaml_file(args, log, yaml_parser, yaml_data, backup_file): with open(args.yaml_file, 'rb') as inhnd: copyfileobj(inhnd, tmphnd) - with open(args.yaml_file, 'w') as yaml_dump: + with open(args.yaml_file, 'w', encoding='utf-8') as yaml_dump: try: yaml_parser.dump(yaml_data, yaml_dump) # Tell pycov to ignore this block because it is impossible to @@ -471,7 +471,7 @@ def main(): consumed_stdin = True has_new_value = True elif args.file: - with open(args.file, 'r') as fhnd: + with open(args.file, 'r', encoding='utf-8') as fhnd: new_value = fhnd.read().rstrip() has_new_value = True elif args.null: diff --git a/yamlpath/common/parsers.py b/yamlpath/common/parsers.py index d7924e9..c002c25 100644 --- a/yamlpath/common/parsers.py +++ b/yamlpath/common/parsers.py @@ -119,7 +119,7 @@ def get_yaml_data( if literal: yaml_data = parser.load(source) else: - with open(source, 'r') as fhnd: + with open(source, 'r', encoding='utf-8') as fhnd: yaml_data = parser.load(fhnd) except KeyboardInterrupt: logger.error("Aborting data load due to keyboard interrupt!") @@ -227,7 +227,7 @@ def get_yaml_multidoc_data( for document in parser.load_all(source): yield (document, True) else: - with open(source, 'r') as fhnd: + with open(source, 'r', encoding='utf-8') as fhnd: for document in parser.load_all(fhnd): logger.debug( "Yielding document from {}:" diff --git a/yamlpath/eyaml/eyamlprocessor.py b/yamlpath/eyaml/eyamlprocessor.py index 0ab827b..2d7a43d 100644 --- a/yamlpath/eyaml/eyamlprocessor.py +++ b/yamlpath/eyaml/eyamlprocessor.py @@ -12,8 +12,9 @@ from ruamel.yaml.comments import CommentedSeq, CommentedMap from yamlpath import YAMLPath +from yamlpath.common import Anchors from yamlpath.eyaml.enums import EYAMLOutputFormats -from yamlpath.enums import YAMLValueFormats +from yamlpath.enums import YAMLValueFormats, PathSeperators from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.wrappers import ConsolePrinter from yamlpath import Processor @@ -52,7 +53,7 @@ def __init__( # pylint: disable=locally-disabled,too-many-branches def _find_eyaml_paths( - self, data: Any, build_path: str = "" + self, data: Any, build_path: YAMLPath ) -> Generator[YAMLPath, None, None]: """ Find every encrypted value and report each as a YAML Path. @@ -62,7 +63,7 @@ def _find_eyaml_paths( Parameters: 1. data (Any) The parsed YAML data to process - 2. build_path (str) A YAML Path under construction + 2. build_path (YAMLPath) A YAML Path under construction Returns: (Generator[Path, None, None]) each YAML Path entry as they are discovered @@ -70,27 +71,28 @@ def _find_eyaml_paths( Raises: N/A """ if isinstance(data, CommentedSeq): - build_path += "[" for idx, ele in enumerate(data): - if hasattr(ele, "anchor") and ele.anchor.value is not None: - tmp_path = build_path + "&" + ele.anchor.value + "]" + node_anchor = Anchors.get_node_anchor(ele) + if node_anchor is not None: + escaped_section = YAMLPath.escape_path_section( + node_anchor, PathSeperators.DOT) + tmp_path_segment = f"[&{escaped_section}]" else: - tmp_path = build_path + str(idx) + "]" + tmp_path_segment = f"[{idx}]" + tmp_path = build_path + tmp_path_segment if self.is_eyaml_value(ele): - yield YAMLPath(tmp_path) + yield tmp_path else: for subpath in self._find_eyaml_paths(ele, tmp_path): yield subpath elif isinstance(data, CommentedMap): - if build_path: - build_path += "." - for key, val in data.non_merged_items(): - tmp_path = build_path + str(key) + tmp_path = build_path + YAMLPath.escape_path_section( + key, PathSeperators.DOT) if self.is_eyaml_value(val): - yield YAMLPath(tmp_path) + yield tmp_path else: for subpath in self._find_eyaml_paths(val, tmp_path): yield subpath @@ -107,7 +109,7 @@ def find_eyaml_paths(self) -> Generator[YAMLPath, None, None]: Raises: N/A """ # Initiate the scan from the data root - for path in self._find_eyaml_paths(self.data): + for path in self._find_eyaml_paths(self.data, YAMLPath()): yield path def decrypt_eyaml(self, value: str) -> str: diff --git a/yamlpath/types/pathattributes.py b/yamlpath/types/pathattributes.py index ff0b0fc..c2d97e8 100644 --- a/yamlpath/types/pathattributes.py +++ b/yamlpath/types/pathattributes.py @@ -6,7 +6,7 @@ from typing import Union from yamlpath.path import CollectorTerms -import yamlpath.path.searchterms as searchterms +from yamlpath.path import searchterms PathAttributes = Union[str, int, CollectorTerms, searchterms.SearchTerms, None]