From cfc78c3dc2db3755a5d9a34f1b8e6adfe6419d15 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 26 Feb 2024 10:48:59 +0100 Subject: [PATCH] Prepare for future Python versions Ast. Newer Python will remove Bytes, and Str nodes in favor of Constant where the node value is a str or a Byte. --- lib/python/pyflyby/_parse.py | 50 ++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/lib/python/pyflyby/_parse.py b/lib/python/pyflyby/_parse.py index 7c2dfa9b..263c7717 100644 --- a/lib/python/pyflyby/_parse.py +++ b/lib/python/pyflyby/_parse.py @@ -14,15 +14,11 @@ from textwrap import dedent import types -import six -from six.moves import range - from pyflyby._file import FilePos, FileText, Filename from pyflyby._flags import CompilerFlags from pyflyby._log import logger from pyflyby._util import cached_attribute, cmp -from ast import Bytes from ast import TypeIgnore, AsyncFunctionDef @@ -40,10 +36,38 @@ def _is_comment_or_blank(line): return re.sub("#.*", "", line).rstrip() == "" +def _is_ast_str_or_byte(node) -> bool: + """ + utility function that test if node is an ast.Str|ast.Bytes in Python < 3.12, + and if it is a ast.Constant, with node.value being a str in newer version. + """ + return _is_ast_str(node) or _is_ast_bytes(node) + +def _is_ast_bytes(node) -> bool: + """ + utility function that test if node is an ast.Str in Python < 3.12, + and if it is a ast.Constant, with node.value being a str in newer version. + """ + if sys.version_info < (3,12): + return isinstance(node, ast.Bytes) + else: + return (isinstance(node, ast.Constant) and isinstance(node.value , bytes)) + + +def _is_ast_str(node) -> bool: + """ + utility function that test if node is an ast.Str in Python < 3.12, + and if it is a ast.Constant, with node.value being a str in newer version. + """ + if sys.version_info < (3,12): + return isinstance(node, ast.Str) + else: + return (isinstance(node, ast.Constant) and isinstance(node.value , str)) + def _ast_str_literal_value(node): - if isinstance(node, (ast.Str, Bytes)): + if _is_ast_str_or_byte(node): return node.s - if isinstance(node, ast.Expr) and isinstance(node.value, (ast.Str, Bytes)): + if isinstance(node, ast.Expr) and _is_ast_str_or_byte(node.value): return node.value.s else: return None @@ -234,7 +258,7 @@ def _test_parse_string_literal(text, flags): except SyntaxError: return None body = module_node.body - if not isinstance(body, (ast.Str, Bytes)): + if not _is_ast_str_or_byte(body): return None return body.s @@ -407,7 +431,7 @@ def _annotate_ast_startpos(ast_node, parent_ast_node, minpos, text, flags): # # To fix that, we copy start_lineno and start_colno from the Str # node once we've corrected the values. - assert not isinstance(ast_node, (ast.Str, Bytes)) + assert not _is_ast_str_or_byte(ast_node) assert leftstr_node.lineno == ast_node.lineno assert leftstr_node.col_offset == -1 ast_node.startpos = leftstr_node.startpos @@ -420,7 +444,7 @@ def _annotate_ast_startpos(ast_node, parent_ast_node, minpos, text, flags): ast_node.endpos = ast_node.value.startpos return True - if not isinstance(ast_node, (ast.Str, Bytes)): + if not _is_ast_str_or_byte(ast_node): raise ValueError( "got a non-string col_offset=-1: %s" % (ast.dump(ast_node))) # The ``lineno`` attribute gives the ending line number of the multiline @@ -666,7 +690,7 @@ def _ast_node_is_in_docstring_position(ast_node): :return: Whether this string ast node is in docstring position. """ - if not isinstance(ast_node, (ast.Str, Bytes)): + if not _is_ast_str_or_byte(ast_node): raise TypeError expr_node = ast_node.context.parent if not isinstance(expr_node, ast.Expr): @@ -732,7 +756,7 @@ def __new__(cls, arg, filename=None, startpos=None, flags=None): return arg arg = arg.block # Fall through - if isinstance(arg, (PythonBlock, FileText, str, six.text_type)): + if isinstance(arg, (PythonBlock, FileText, str)): block = PythonBlock(arg, filename=filename, startpos=startpos, flags=flags) statements = block.statements @@ -904,7 +928,7 @@ def __new__(cls, arg, filename=None, startpos=None, flags=None, flags = CompilerFlags(flags, arg.flags) arg = arg.text # Fall through - if isinstance(arg, (FileText, Filename, str, six.text_type)): + if isinstance(arg, (FileText, Filename, str)): return cls.from_text( arg, filename=filename, startpos=startpos, flags=flags, auto_flags=auto_flags) @@ -1235,7 +1259,7 @@ def string_literals(self): Iterable of ``ast.Str`` or ``ast.Bytes`` nodes """ for node in _walk_ast_nodes_in_order(self.annotated_ast_node): - if isinstance(node, (ast.Str, Bytes)): + if _is_ast_str_or_byte(node): assert hasattr(node, 'startpos') yield node