diff --git a/rope/refactor/occurrences.py b/rope/refactor/occurrences.py index 6bc3555bf..ed53b76e6 100644 --- a/rope/refactor/occurrences.py +++ b/rope/refactor/occurrences.py @@ -36,7 +36,8 @@ """ import contextlib -import re +import io +import tokenize from typing import Iterator from rope.base import ( @@ -311,14 +312,6 @@ class _TextualFinder: def __init__(self, name, docs=False): self.name = name self.docs = docs - self.comment_pattern = _TextualFinder.any("comment", [r"#[^\n]*"]) - self.string_pattern = _TextualFinder.any( - "string", [codeanalyze.get_string_pattern()] - ) - self.f_string_pattern = _TextualFinder.any( - "fstring", [codeanalyze.get_formatted_string_pattern()] - ) - self.pattern = self._get_occurrence_pattern(self.name) def find_offsets(self, source: str) -> Iterator[int]: if not self._fast_file_query(source): @@ -326,26 +319,42 @@ def find_offsets(self, source: str) -> Iterator[int]: if self.docs: searcher = self._normal_search else: - searcher = self._re_search + searcher = self._token_search yield from searcher(source) - def _re_search(self, source: str) -> Iterator[int]: - for match in self.pattern.finditer(source): - if match.groupdict()["occurrence"]: - yield match.start("occurrence") - elif match.groupdict()["fstring"]: - f_string = match.groupdict()["fstring"] - for offset in self._search_in_f_string(f_string): - yield match.start("fstring") + offset - - def _search_in_f_string(self, f_string: str) -> Iterator[int]: - tree = ast.parse(f_string) - for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id == self.name: - yield node.col_offset - elif isinstance(node, ast.Attribute) and node.attr == self.name: - assert node.end_col_offset is not None - yield node.end_col_offset - len(self.name) + def _token_search(self, source: str) -> Iterator[int]: + """Yield the offsets of every NAME token equal to the searched name. + + Complete string tokens expose replacement expressions through Python's + native AST; expression tokens retain their source coordinates. + """ + line_starts = [0] + for line in source.splitlines(keepends=True): + line_starts.append(line_starts[-1] + len(line)) + for item in tokenize.generate_tokens(io.StringIO(source).readline): + row, column = item.start + if item.type == tokenize.NAME and item.string == self.name: + yield line_starts[row - 1] + column + elif item.type == tokenize.STRING and item.string.lower().startswith( + ("f", "rf") + ): + module = ast.parse(item.string) + positions = codeanalyze.ASTLinesAdapter(item.string) + for offset in self._formatted_offsets( + item.string, module.body[0].value, positions + ): + yield line_starts[row - 1] + column + offset + + def _formatted_offsets(self, source, node, positions) -> Iterator[int]: + for value in node.values: + if not isinstance(value, ast.FormattedValue): + continue + start, end = positions[value.value] + # AST spans omit the grouping that permits continuation indentation. + for offset in self._token_search("(" + source[start:end] + ")"): + yield start + offset - 1 + if value.format_spec is not None: + yield from self._formatted_offsets(source, value.format_spec, positions) def _normal_search(self, source: str) -> Iterator[int]: current = 0 @@ -372,27 +381,6 @@ def _get_source(self, resource, pymodule): else: return pymodule.source_code - def _get_occurrence_pattern(self, name): - # A name directly followed by a quote is a string prefix, e.g. the - # ``f`` in ``f"..."``, never a reference to the name. - occurrence_pattern = _TextualFinder.any( - "occurrence", ["\\b" + name + "\\b(?![\"'])"] - ) - pattern = re.compile( - occurrence_pattern - + "|" - + self.comment_pattern - + "|" - + self.string_pattern - + "|" - + self.f_string_pattern - ) - return pattern - - @staticmethod - def any(name, list_): - return "(?P<%s>" % name + "|".join(list_) + ")" - class _OccurrenceToolsCreator: def __init__(self, project, resource=None, pymodule=None, docs=False): diff --git a/ropetest/refactor/renametest.py b/ropetest/refactor/renametest.py index fb0270122..f7ebbafd7 100644 --- a/ropetest/refactor/renametest.py +++ b/ropetest/refactor/renametest.py @@ -331,6 +331,30 @@ def test_renaming_occurrence_in_nested_f_string(self): refactored = self._local_rename(code, 2, "new_var") self.assertEqual(expected, refactored) + def test_renaming_occurrence_in_f_string_with_reused_quotes(self): + code = dedent('''\ + a_var = {"key": 20} + a_string=f"{a_var['key']} and {a_var['key']:>2}" + ''') + expected = dedent('''\ + new_var = {"key": 20} + a_string=f"{new_var['key']} and {new_var['key']:>2}" + ''') + refactored = self._local_rename(code, 2, "new_var") + self.assertEqual(expected, refactored) + + def test_renaming_occurrence_in_f_string_format_spec(self): + code = dedent("""\ + a_var = 4 + a_string=f"{a_var:>{a_var}}" + """) + expected = dedent("""\ + new_var = 4 + a_string=f"{new_var:>{new_var}}" + """) + refactored = self._local_rename(code, 2, "new_var") + self.assertEqual(expected, refactored) + def test_renaming_attribute_occurrences_in_f_string(self): code = dedent("""\ class MyClass: